@xsft/panels 0.0.0-stage → 0.1.0

package/README.md

@@ -1,3 +1,51 @@
-# Temporary Holding Version
+# @xsft/panels
 
-This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
+`panels run` runs a panel from cross.software/panels against a target. Each judge is a `claude` or `codex` session that scores one or more criteria of the panel. You need Node 20 or newer and one of the two harnesses.
+
+```text
+panels run <panel> [target...] [flags]
+
+<panel>           owner/name, or owner/name/vN to pin version N
+[target...]       free words, joined with single spaces into one target
+
+--size <n>        number of judges                   default: the number of criteria
+--harness <name>  claude | codex                     default: claude if installed, else codex
+--brief <file>    the brief                          default: none
+--sources <path>  a doc folder, a repository, or an app URL; repeat the flag for each
+--                end of the flags; each later word joins the target
+
+exit 0  pass: every criterion scored 4
+exit 1  fail: every judge answered, and one or more criteria scored below 4
+exit 2  error: bad flags, no target, panel not found, harness missing, or a judge failed
+```
+
+CAUTION: A judge with web fetch can send what it reads to a URL. Do not run a panel on a repository that you do not trust.
+
+```sh
+npx @xsft/panels run xsft/marketing-site https://example.com --brief docs/brief.md --harness claude
+npx @xsft/panels run xsft/marketing-site/v6 . at http://localhost:4321 --sources docs --harness codex
+```
+
+Run the command in the folder of the product. Each judge starts in that folder, so `.`, a branch, and relative paths resolve there. The CLI does not test, fetch, or change the target. Four judges run at a time. A judge gets 20 minutes for each of its criteria.
+
+## Run folder
+
+Each run writes to `.panels/<owner>-<name>/<UTC time>/`:
+
+- `panel.json`: the panel as the api gave it.
+- `report.md` and `report.json`: the scores, the evidence, the fixes, and the errors. `schema/report.json` is the schema of `report.json`.
+- `judges/<k>/`: the prompt, the output schema, and the raw answer of judge k.
+
+The last line on stdout is the path of `report.json`.
+
+## Skipped
+
+- Run upload to cross.software, and `--no-upload`.
+- `--version`: `owner/name/vN` pins a version.
+- `--out`: the run folder is fixed.
+- `--json`: stdout ends with the path of `report.json`.
+- `--concurrency`: fixed at 4.
+- `--model`: each harness uses its default.
+- A panel URL as `<panel>`, a `PANELS_API` override, and a panel cache.
+- A login: an owner cannot run an unpublished panel.
+- A limit on `--size`, cost totals, and `cap` and `gap` fields.

package/bin/panels.js

@@ -0,0 +1,78 @@
+#!/usr/bin/env node
+import { existsSync, mkdirSync, writeFileSync } from 'node:fs';
+import { delimiter, join } from 'node:path';
+import { parse, SYNOPSIS } from '../lib/args.js';
+import { runJudge } from '../lib/judge.js';
+import { fetchPanel } from '../lib/panel.js';
+import { buildReport, renderMarkdown } from '../lib/report.js';
+import { split } from '../lib/split.js';
+
+// More parallel sessions on one account get 429 answers from the model api.
+const PARALLEL_JUDGES = 4;
+
+const onPath = (name) => process.env.PATH.split(delimiter).some((dir) => existsSync(join(dir, name)));
+
+async function pool(items, size, fn) {
+  const out = [];
+  let next = 0;
+  const worker = async () => {
+    while (next < items.length) {
+      const i = next++;
+      out[i] = await fn(items[i], i);
+    }
+  };
+  await Promise.all(Array.from({ length: Math.min(size, items.length) }, worker));
+  return out;
+}
+
+async function main() {
+  const args = parse(process.argv.slice(2));
+  if (args.help) {
+    console.log(SYNOPSIS);
+    return 0;
+  }
+  if (args.usage) {
+    console.error(SYNOPSIS);
+    return 2;
+  }
+  if (args.error) {
+    console.error(`panels: ${args.error}`);
+    return 2;
+  }
+  const harness = args.harness ?? (onPath('claude') ? 'claude' : 'codex');
+  if (!onPath(harness)) {
+    console.error(`panels: harness missing: ${harness} is not on PATH`);
+    return 2;
+  }
+  const fetched = await fetchPanel(args.panel);
+  if (fetched.error) {
+    console.error(`panels: ${fetched.error}`);
+    return 2;
+  }
+  const { panel } = fetched;
+  const cwd = process.cwd();
+  const judges = split(panel.criteria.length, args.size ?? panel.criteria.length);
+  const runDir = join(cwd, '.panels', `${panel.owner}-${panel.name}`, new Date().toISOString().replaceAll(':', '-'));
+  mkdirSync(runDir, { recursive: true });
+  writeFileSync(join(runDir, 'panel.json'), JSON.stringify(panel, null, 2));
+  const outcomes = await pool(judges, PARALLEL_JUDGES, (indexes, j) => runJudge({
+    dir: join(runDir, 'judges', String(j + 1)),
+    criteria: indexes.map((i) => ({ rank: i + 1, ...panel.criteria[i] })),
+    harness,
+    cwd,
+    target: args.target,
+    brief: args.brief,
+    sources: args.sources,
+  }));
+  const report = buildReport({ panel, target: args.target, cwd, harness, judges, outcomes });
+  const markdown = renderMarkdown(report);
+  const reportPath = join(runDir, 'report.json');
+  writeFileSync(join(runDir, 'report.md'), markdown);
+  writeFileSync(reportPath, `${JSON.stringify(report, null, 2)}\n`);
+  console.log(markdown);
+  console.log(reportPath);
+  return { pass: 0, fail: 1, error: 2 }[report.status];
+}
+
+// Node exits 1 on a crash, and 1 means "fail" to an agent loop, so a crash prints its stack and exits 2.
+main().then((code) => { process.exitCode = code; }, (error) => { console.error(error); process.exitCode = 2; });

package/lib/args.js

@@ -0,0 +1,52 @@
+import { parseArgs } from 'node:util';
+
+export const SYNOPSIS = `panels run <panel> [target...] [flags]
+
+<panel>           owner/name, or owner/name/vN to pin version N
+[target...]       free words, joined with single spaces into one target
+
+--size <n>        number of judges                   default: the number of criteria
+--harness <name>  claude | codex                     default: claude if installed, else codex
+--brief <file>    the brief                          default: none
+--sources <path>  a doc folder, a repository, or an app URL; repeat the flag for each
+--                end of the flags; each later word joins the target
+
+exit 0  pass: every criterion scored 4
+exit 1  fail: every judge answered, and one or more criteria scored below 4
+exit 2  error: bad flags, no target, panel not found, harness missing, or a judge failed`;
+
+const options = {
+  size: { type: 'string' },
+  harness: { type: 'string' },
+  brief: { type: 'string' },
+  sources: { type: 'string', multiple: true },
+  help: { type: 'boolean' },
+};
+
+const PANEL = /^([a-z0-9-]+)\/([a-z0-9-]+)(?:\/v([1-9]\d*))?$/;
+
+/** Returns the run options, { help: true }, { usage: true }, or { error } with a one-line message. */
+export function parse(argv) {
+  // strict is off so that an unknown flag comes back as a token and gets the one-line message below.
+  const { values, positionals, tokens } = parseArgs({ args: argv, options, strict: false, allowPositionals: true, tokens: true });
+  for (const token of tokens.filter((t) => t.kind === 'option')) {
+    if (!Object.hasOwn(options, token.name)) return { error: `unknown flag ${token.rawName}` };
+    if (token.name !== 'help' && token.value === undefined) return { error: `${token.rawName} needs a value` };
+  }
+  if (values.help) return { help: true };
+  const [command, panel, ...words] = positionals;
+  if (command !== 'run' || panel === undefined) return { usage: true };
+  const match = PANEL.exec(panel);
+  if (!match) return { error: `bad panel ${panel}: use owner/name or owner/name/vN` };
+  if (words.length === 0) return { error: 'no target: type what the panel reviews after the panel name, such as a URL, a path, or a branch' };
+  if (values.size !== undefined && !/^[1-9]\d*$/.test(values.size)) return { error: `--size needs a whole number of 1 or more, not ${values.size}` };
+  if (values.harness !== undefined && !['claude', 'codex'].includes(values.harness)) return { error: `--harness needs claude or codex, not ${values.harness}` };
+  return {
+    panel: { owner: match[1], name: match[2], version: match[3] && Number(match[3]) },
+    target: words.join(' '),
+    size: values.size && Number(values.size),
+    harness: values.harness,
+    brief: values.brief,
+    sources: values.sources ?? [],
+  };
+}

package/lib/judge.js

@@ -0,0 +1,145 @@
+import { spawn } from 'node:child_process';
+import { closeSync, mkdirSync, openSync, readFileSync, writeFileSync } from 'node:fs';
+import { dirname, join, relative, resolve, sep } from 'node:path';
+
+const { judgeResult } = JSON.parse(readFileSync(new URL('../schema/report.json', import.meta.url), 'utf8')).$defs;
+const SYSTEM_PROMPT = readFileSync(new URL('../prompts/judge-system-prompt.txt', import.meta.url), 'utf8');
+const MINUTES_PER_CRITERION = 20;
+const TOOLS = 'Read,Glob,Grep,Bash,WebFetch,WebSearch';
+// With --permission-mode dontAsk, Claude refuses every tool call that this list does not allow, so Bash runs git reads only.
+const ALLOWED_TOOLS = 'Read,Glob,Grep,WebFetch,WebSearch,Bash(git log:*),Bash(git show:*),Bash(git diff:*),Bash(git status:*),Bash(git ls-files:*),Bash(git rev-parse:*),Bash(git branch --list:*)';
+
+export const bandToScore = (band) => (band <= 3 ? 1 : band <= 6 ? 2 : band <= 8 ? 3 : 4);
+
+/**
+ * The user message of a judge. criteria items are { rank, name, description, body }. Each part sits
+ * in a tag, because a body is markdown with headings of its own.
+ */
+export function userMessage(criteria, { target, brief, sources }) {
+  const parts = criteria.map((c) => `<criterion rank="${c.rank}" name="${c.name}">\nDescription: ${c.description}\n\n${c.body}\n</criterion>`);
+  parts.push(`<target>\n${target}\n</target>`, `<brief>\n${brief ?? 'none supplied'}\n</brief>`, `<sources>\n${sources.length ? sources.join('\n') : 'none'}\n</sources>`);
+  return `${parts.join('\n\n')}\n`;
+}
+
+/** The output schema of a judge: { results }, one judgeResult for each of its criteria, with rank and name fixed. */
+export function judgeSchema(criteria) {
+  const items = criteria.map((c) => ({
+    ...judgeResult,
+    properties: { ...judgeResult.properties, rank: { type: 'integer', const: c.rank }, criterion: { type: 'string', const: c.name } },
+  }));
+  return {
+    type: 'object',
+    additionalProperties: false,
+    required: ['results'],
+    properties: { results: { type: 'array', minItems: items.length, maxItems: items.length, items: items.length === 1 ? items[0] : { anyOf: items } } },
+  };
+}
+
+/** Returns the reasons the answer of a judge is not valid, or an empty array. */
+export function check(answer, criteria) {
+  if (!Array.isArray(answer?.results)) return ['The answer must be a JSON object with a results array.'];
+  if (answer.results.length !== criteria.length) return [`results must hold ${criteria.length} items, one for each criterion, in rank order.`];
+  return criteria.flatMap((c, i) => {
+    const r = answer.results[i];
+    const missing = judgeResult.required.filter((key) => !(key in Object(r)));
+    if (missing.length) return [`Item ${i + 1} lacks ${missing.join(', ')}.`];
+    const errors = [];
+    if (r.rank !== c.rank || r.criterion !== c.name) errors.push(`Item ${i + 1} must have rank ${c.rank} and criterion "${c.name}".`);
+    if (!Number.isInteger(r.band) || r.band < 1 || r.band > 10) errors.push(`Rank ${c.rank}: band must be a whole number from 1 to 10.`);
+    else if (r.score !== bandToScore(r.band)) errors.push(`Rank ${c.rank}: band ${r.band} maps to score ${bandToScore(r.band)}, not ${r.score}.`);
+    if (r.score < 4 && !(Array.isArray(r.fixes) && r.fixes.length)) errors.push(`Rank ${c.rank}: score ${r.score} is below 4, so fixes must list each fix that score 4 needs.`);
+    return errors;
+  });
+}
+
+/** The folders of the brief and the path sources that are outside cwd. A Claude judge cannot read them without --add-dir. */
+export function addDirs(cwd, brief, sources) {
+  const paths = [brief && dirname(resolve(cwd, brief)), ...sources.filter((s) => !/^https?:\/\//.test(s)).map((s) => resolve(cwd, s))];
+  return [...new Set(paths.filter(Boolean))].filter((p) => relative(cwd, p).split(sep)[0] === '..');
+}
+
+function exec(command, args, { cwd, stdin, timeout }) {
+  return new Promise((done) => {
+    const child = spawn(command, args, { cwd, timeout, stdio: [stdin, 'pipe', 'pipe'] });
+    let out = '';
+    let err = '';
+    child.stdout.on('data', (d) => { out += d; });
+    child.stderr.on('data', (d) => { err += d; });
+    child.on('close', (code, signal) => {
+      if (signal) done({ error: `${command} stopped at the time limit of ${timeout / 60_000} minutes` });
+      else if (code) done({ error: `${command} exited ${code}: ${(err || out).trim().slice(-1000)}` });
+      else done({ out });
+    });
+  });
+}
+
+async function askClaude(j, prompt, session) {
+  const args = [
+    '-p', '--safe-mode',
+    '--tools', TOOLS,
+    '--allowedTools', ALLOWED_TOOLS,
+    '--permission-mode', 'dontAsk', '--permission-prompts', 'none',
+    ...j.addDirs.flatMap((d) => ['--add-dir', d]),
+    '--system-prompt', SYSTEM_PROMPT,
+    '--json-schema', JSON.stringify(j.schema),
+    '--output-format', 'json',
+    ...(session ? ['--resume', session] : []),
+    prompt,
+  ];
+  const run = await exec(j.command, args, { cwd: j.cwd, stdin: 'ignore', timeout: j.timeout });
+  if (run.error) return run;
+  writeFileSync(join(j.dir, 'raw.json'), run.out);
+  // With verbose on in the user settings, the output is an array of messages and not one result object.
+  const result = [JSON.parse(run.out)].flat().find((m) => m.type === 'result');
+  if (result.is_error) return { error: `claude answered an error: ${result.result}` };
+  return { answer: result.structured_output, session: result.session_id };
+}
+
+// --disable apps: the apps feature gives Codex the connectors of the ChatGPT account, and some of them write
+// (chatgpt_space_create_page, chatgpt_space_trash_page). --ignore-user-config does not turn them off.
+async function askCodex(j, prompt, thread) {
+  const raw = join(j.dir, 'raw.json');
+  const instructions = `developer_instructions=${JSON.stringify(SYSTEM_PROMPT)}`;
+  const output = ['--output-schema', join(j.dir, 'schema.json'), '-o', raw, '--json'];
+  const args = thread
+    ? ['exec', 'resume', thread, prompt, '--skip-git-repo-check', '--ignore-user-config', '-c', 'sandbox_mode="read-only"', '--disable', 'multi_agent', '--disable', 'apps', '-c', instructions, ...output]
+    : ['exec', '-', '-C', j.cwd, '--skip-git-repo-check', '--ignore-user-config', '-s', 'read-only', '--disable', 'multi_agent', '--disable', 'apps', '-c', instructions, ...output];
+  const stdin = thread ? 'ignore' : openSync(join(j.dir, 'prompt.md'));
+  const run = await exec(j.command, args, { cwd: j.cwd, stdin, timeout: j.timeout });
+  if (!thread) closeSync(stdin);
+  if (run.error) return run;
+  const started = run.out.split('\n').find((line) => line.includes('"thread.started"'));
+  return { answer: parseAnswer(readFileSync(raw, 'utf8')), session: thread ?? JSON.parse(started).thread_id };
+}
+
+function parseAnswer(text) {
+  try {
+    return JSON.parse(text);
+  } catch {
+    // check() rejects undefined, so text that is not JSON gets the one re-ask.
+    return undefined;
+  }
+}
+
+/**
+ * Runs one judge to the end: the first answer, and one re-ask when check() rejects it.
+ * Returns { results } or { error }. command is the harness binary; the tests pass a stub.
+ */
+export async function runJudge({ dir, criteria, harness, cwd, target, brief, sources }, command = harness) {
+  mkdirSync(dir, { recursive: true });
+  const prompt = userMessage(criteria, { target, brief, sources });
+  const schema = judgeSchema(criteria);
+  writeFileSync(join(dir, 'prompt.md'), prompt);
+  writeFileSync(join(dir, 'schema.json'), JSON.stringify(schema, null, 2));
+  const j = { dir, cwd, schema, command, addDirs: addDirs(cwd, brief, sources), timeout: criteria.length * MINUTES_PER_CRITERION * 60_000 };
+  const ask = harness === 'claude' ? askClaude : askCodex;
+  let reply = await ask(j, prompt);
+  if (reply.error) return reply;
+  let errors = check(reply.answer, criteria);
+  if (errors.length) {
+    reply = await ask(j, `Your answer failed these checks. Return the whole JSON object again, with each one fixed:\n${errors.map((e) => `- ${e}`).join('\n')}`, reply.session);
+    if (reply.error) return reply;
+    errors = check(reply.answer, criteria);
+  }
+  return errors.length ? { error: `invalid answer after one re-ask: ${errors.join(' ')}` } : { results: reply.answer.results };
+}

package/lib/panel.js

@@ -0,0 +1,9 @@
+const API = 'https://cross.software/panels/api/panels';
+
+/** Reads a panel from the public api. An unpublished panel also answers 404. */
+export async function fetchPanel({ owner, name, version }) {
+  const res = await fetch(`${API}/${owner}/${name}${version ? `/v/${version}` : ''}`);
+  if (res.status === 404) return { error: 'panel not found' };
+  if (!res.ok) return { error: `panel fetch failed: ${res.status}` };
+  return { panel: await res.json() };
+}

package/lib/report.js

@@ -0,0 +1,43 @@
+/**
+ * Builds report.json. judges holds the criterion indexes of each judge, and outcomes the
+ * { results } or { error } of each judge, in the same order. The lowest score of a criterion
+ * counts, and a criterion of a failed judge gets score null.
+ */
+export function buildReport({ panel, target, cwd, harness, judges, outcomes }) {
+  const errors = outcomes.flatMap((o, j) => (o.error ? [{ judge: j + 1, message: o.error }] : []));
+  const failed = new Set(errors.flatMap((e) => judges[e.judge - 1]));
+  const answers = judges.flatMap((indexes, j) => (outcomes[j].results ?? []).map((result, x) => ({ i: indexes[x], judge: j + 1, result })));
+  const criteria = panel.criteria.map((c, i) => {
+    const mine = answers.filter((a) => a.i === i).map(({ judge, result }) => ({ judge, result }));
+    return { rank: i + 1, name: c.name, score: failed.has(i) ? null : Math.min(...mine.map((a) => a.result.score)), judges: mine };
+  });
+  const status = criteria.some((c) => c.score === null) ? 'error' : criteria.every((c) => c.score === 4) ? 'pass' : 'fail';
+  return { status, panel: { owner: panel.owner, name: panel.name, version: panel.version }, target, cwd, harness, size: judges.length, criteria, errors };
+}
+
+const cell = (text) => text.replaceAll('|', '\\|').replaceAll('\n', ' ');
+
+export function renderMarkdown(report) {
+  const { status, panel, target, criteria, errors } = report;
+  const lines = [
+    `${status.toUpperCase()}  ${panel.owner}/${panel.name} v${panel.version} on ${JSON.stringify(target)}  (${criteria.filter((c) => c.score === 4).length} of ${criteria.length} criteria at 4)`,
+    '',
+    '| # | Criterion | Score | Verdict |',
+    '|---|---|---|---|',
+  ];
+  for (const c of criteria) {
+    const lowest = c.judges.find((a) => a.result.score === c.score);
+    lines.push(`| ${c.rank} | ${cell(c.name)} | ${c.score ?? 'error'} | ${cell(lowest?.result.verdict ?? '')} |`);
+  }
+  const list = (title, items) => items.length && lines.push(title, ...items.map((item) => `- ${item}`));
+  for (const c of criteria) {
+    for (const { result } of c.judges) {
+      lines.push('', `## ${c.rank}. ${c.name}: ${result.score} (band ${result.band})`);
+      list('Evidence', result.evidence.map((e) => `${e.where}: ${e.quote ?? e.claim}`));
+      list('Fixes', result.fixes.map((f) => `${f.problem} ${f.change} (${f.where})`));
+      list('Not checked', result.notChecked.map((n) => `${n.test}: ${n.reason}`));
+    }
+  }
+  if (errors.length) lines.push('', '## Errors', ...errors.map((e) => `- judge ${e.judge}: ${e.message}`));
+  return `${lines.join('\n')}\n`;
+}

package/lib/split.js

@@ -0,0 +1,14 @@
+/** Returns, for each judge, the indexes of the criteria it scores. n is the number of criteria. */
+export function split(n, size) {
+  if (size <= n) {
+    const judges = [];
+    let start = 0;
+    for (let j = 0; j < size; j++) {
+      const length = Math.floor(n / size) + (j < n % size ? 1 : 0);
+      judges.push(Array.from({ length }, (_, x) => start + x));
+      start += length;
+    }
+    return judges;
+  }
+  return Array.from({ length: n }, (_, i) => Array(Math.floor(size / n) + (i < size % n ? 1 : 0)).fill([i])).flat();
+}

package/package.json

@@ -1,6 +1,12 @@
 {
   "name": "@xsft/panels",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
-}
+  "version": "0.1.0",
+  "private": false,
+  "description": "Run a panel of AI judges from cross.software/panels against a target.",
+  "type": "module",
+  "bin": { "panels": "bin/panels.js" },
+  "files": ["bin", "lib", "prompts", "schema", "README.md"],
+  "license": "UNLICENSED",
+  "engines": { "node": ">=20" },
+  "scripts": { "test": "node --test test/*.test.js" }
+}

package/prompts/judge-system-prompt.txt

@@ -0,0 +1,91 @@
+You are one judge in a run of a panel. A panel is a named list of criteria. You score one or more criteria against a target, and you name the fixes for each criterion that does not pass.
+
+YOUR INPUTS
+
+The user message gives you:
+- Each criterion: its rank, its name, its one-line description, and its body. Follow the body as written.
+- The target: the text that the engineer typed after the panel name, unchanged. It can be a URL, a path, a branch, a pull request, or a sentence.
+- The brief: a file path, or "none supplied".
+- The sources: paths or URLs, or "none".
+
+Your working directory is the folder where the engineer ran the command.
+
+THE TARGET
+
+Find out what the target names, and reach it with your tools: read files, run git read commands, open URLs. When you cannot reach the target, or cannot do a test of the body with your tools, add the test to notChecked with the reason. Then score as the body says for that case. Never score what you did not see.
+
+ORDER
+
+Score your criteria one at a time, in rank order. Before you read the brief or the sources, do every step that one of your bodies puts before them.
+
+HOW TO SCORE
+
+1. Follow the body to a level from 1 to 10. The body calls this level a score. Here it is the band.
+2. Apply each cap of the body, such as "cannot support a score above 8". When two caps apply, use the lower cap.
+3. When the evidence fits two bands, use the lower band.
+4. Convert the band to the score with this map:
+   band 1 to 3  = score 1
+   band 4 to 6  = score 2
+   band 7 or 8  = score 3
+   band 9 or 10 = score 4
+5. A criterion passes only at score 4.
+
+EVIDENCE
+
+- Each claim needs evidence that the engineer can check: a URL, a file and line, a quote, or a command and its output.
+- Give no credit for intent. A promise in a comment, a doc, or the brief is not done work.
+- No evidence for a band means the lower band.
+- At band 9, name the one specific gap of the body in the evidence.
+
+RECORDS AND FIXES
+
+- Put each record and table that the body tells you to write in records, with the columns of the body.
+- Below score 4, list each fix that score 4 needs: the problem, the change, and the place.
+
+PROHIBITIONS
+
+- Do not change the body, the bands, the caps, or the map.
+- Do not score a fault that the body gives to another criterion.
+- Do not write or change files. Name each fix, and do not make it.
+- Do not submit forms, send messages, or post data.
+- Do not read the .panels folder. It holds earlier reports.
+
+OUTPUT
+
+Return one JSON object and nothing else. Give one item in results for each of your criteria, in rank order.
+
+{
+  "results": [{
+    "rank": integer,
+    "criterion": string,
+    "band": integer 1 to 10,
+    "score": integer 1 to 4,
+    "verdict": string, one sentence of 30 words or less,
+    "records": [{ "name": string, "columns": [string], "rows": [[string]] }],
+    "evidence": [{ "claim": string, "where": string, "quote": string or null }],
+    "notChecked": [{ "test": string, "reason": string }],
+    "fixes": [{ "problem": string, "change": string, "where": string }]
+  }]
+}
+
+Example of one item:
+
+{
+  "rank": 5,
+  "criterion": "Original design assets",
+  "band": 5,
+  "score": 2,
+  "verdict": "The hero shows the real app, and the six feature icons are Lucide icons.",
+  "records": [{
+    "name": "asset table",
+    "columns": ["asset", "provenance", "product test", "verdict"],
+    "rows": [["feature icons (6)", "class \"lucide\"", "generic symbols", "generic"]]
+  }],
+  "evidence": [{ "claim": "The icons come from Lucide.", "where": "src/components/Features.tsx:3", "quote": "import { Zap } from \"lucide-react\"" }],
+  "notChecked": [{ "test": "reverse image search of hero.webp", "reason": "the target is on localhost" }],
+  "fixes": [{
+    "problem": "Library icons carry the feature grid.",
+    "change": "Draw each icon as an object of this product.",
+    "where": "src/components/Features.tsx, home section #features"
+  }]
+}

package/schema/report.json

@@ -0,0 +1,125 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "type": "object",
+  "additionalProperties": false,
+  "required": ["status", "panel", "target", "cwd", "harness", "size", "criteria", "errors"],
+  "properties": {
+    "status": { "enum": ["pass", "fail", "error"] },
+    "panel": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["owner", "name", "version"],
+      "properties": {
+        "owner": { "type": "string" },
+        "name": { "type": "string" },
+        "version": { "type": "integer" }
+      }
+    },
+    "target": { "type": "string" },
+    "cwd": { "type": "string" },
+    "harness": { "enum": ["claude", "codex"] },
+    "size": { "type": "integer", "minimum": 1 },
+    "criteria": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "additionalProperties": false,
+        "required": ["rank", "name", "score", "judges"],
+        "properties": {
+          "rank": { "type": "integer" },
+          "name": { "type": "string" },
+          "score": { "type": ["integer", "null"], "minimum": 1, "maximum": 4 },
+          "judges": {
+            "type": "array",
+            "items": {
+              "type": "object",
+              "additionalProperties": false,
+              "required": ["judge", "result"],
+              "properties": {
+                "judge": { "type": "integer" },
+                "result": { "$ref": "#/$defs/judgeResult" }
+              }
+            }
+          }
+        }
+      }
+    },
+    "errors": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "additionalProperties": false,
+        "required": ["judge", "message"],
+        "properties": {
+          "judge": { "type": "integer" },
+          "message": { "type": "string" }
+        }
+      }
+    }
+  },
+  "$defs": {
+    "judgeResult": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["rank", "criterion", "band", "score", "verdict", "records", "evidence", "notChecked", "fixes"],
+      "properties": {
+        "rank": { "type": "integer" },
+        "criterion": { "type": "string" },
+        "band": { "type": "integer", "minimum": 1, "maximum": 10 },
+        "score": { "type": "integer", "minimum": 1, "maximum": 4 },
+        "verdict": { "type": "string" },
+        "records": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "additionalProperties": false,
+            "required": ["name", "columns", "rows"],
+            "properties": {
+              "name": { "type": "string" },
+              "columns": { "type": "array", "items": { "type": "string" } },
+              "rows": { "type": "array", "items": { "type": "array", "items": { "type": "string" } } }
+            }
+          }
+        },
+        "evidence": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "additionalProperties": false,
+            "required": ["claim", "where", "quote"],
+            "properties": {
+              "claim": { "type": "string" },
+              "where": { "type": "string" },
+              "quote": { "type": ["string", "null"] }
+            }
+          }
+        },
+        "notChecked": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "additionalProperties": false,
+            "required": ["test", "reason"],
+            "properties": {
+              "test": { "type": "string" },
+              "reason": { "type": "string" }
+            }
+          }
+        },
+        "fixes": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "additionalProperties": false,
+            "required": ["problem", "change", "where"],
+            "properties": {
+              "problem": { "type": "string" },
+              "change": { "type": "string" },
+              "where": { "type": "string" }
+            }
+          }
+        }
+      }
+    }
+  }
+}