@josueavalosjim/taste-check 0.6.0 → 0.7.0

package/README.md

@@ -290,6 +290,35 @@ inside a tool is somebody else's taste with the tool's authority behind it.
 Checklist lines are list items in your file; a heading or a paragraph is prose
 and is not judged.
 
+### Letting an agent carry the call
+
+A shell command is one way to reach a model and a poor one for an agent, which
+is already a model and can spawn a genuinely fresh context of its own instead
+of shelling out to a second copy of itself. So the judge splits in two:
+
+```bash
+taste-check judge --emit              # the prompt, and nothing else happens
+taste-check judge --verdict reply.json  # or - for stdin
+```
+
+`--emit` prints the prompt and stops. It still refuses when there are no
+screenshots or no checklist, because a prompt for nothing gets a confident
+verdict about nothing. `--verdict` checks the reply against the checklist
+exactly as the shell route does: every line answered once, no invented lines,
+valid verdicts. Taking the agent route does not buy a softer grading.
+
+`judge.command` is optional when you use this. There is a skill for it:
+
+```bash
+taste-check judge --skill    # prints the path to SKILL.md
+```
+
+Copy it wherever your agent keeps skills. It carries the mechanism, which is
+that the agent must not be the judge: it is reading the session that built the
+thing, and the reasoning that justified each choice is still sitting there
+ready to justify it again. It carries no design rules, for the same reason
+nothing else here does.
+
 ### What the exit code means here
 
 A verdict is an opinion, so a `fail` prints as a note and the command exits 0.
@@ -328,7 +357,7 @@ from anywhere.
 | `judge.checklist` | Your checklist file. List items are judged, prose is not. |
 | `judge.shots` | Screenshots to hand the judge. Matching nothing is a failure. |
 | `judge.shotCommand` | Optional command run first to produce those screenshots. |
-| `judge.command` | The model command. Prompt on stdin, image paths as arguments. |
+| `judge.command` | The model command. Prompt on stdin, image paths as arguments. Optional if an agent carries the call. |
 | `judge.failOn` | `"never"` (default) or `"fail"`. Whether a verdict blocks. |
 | `runtime.url` | The page to measure. A `file://` URL works. |
 | `runtime.endpoint` | An existing CDP endpoint. Given one, taste-check connects rather than launching, and never closes a browser it did not start. |
@@ -453,8 +482,6 @@ Not built. Written down so the shape is clear.
 
 **YAML configs**, once there is a reason to take on a parser.
 
-**A way to run the judge from an agent skill**, not only from a shell.
-
 **`lab()` and `lch()`**, which need the D50 white point and a chromatic
 adaptation step that `oklch()` does not. Completeness rather than reach, so
 it sits behind the others. Worth doing the same way when it happens: derive it,

package/bin/taste-check.mjs

@@ -8,9 +8,10 @@
  * not a quiet skip.
  */
 import { readFileSync } from 'node:fs';
+import { fileURLToPath } from 'node:url';
 
 import { load } from '../src/config.mjs';
-import { judge, run, runtime } from '../src/index.mjs';
+import { gradeVerdict, judge, prepareJudge, run, runtime } from '../src/index.mjs';
 import { failed, toJson, toText } from '../src/report.mjs';
 import { toSarif } from '../src/sarif.mjs';
 
@@ -23,6 +24,10 @@ const USAGE = `taste-check
 Options:
   -c, --config <path>   Config file (default: tastecheck.config.json)
       --only <name>     Run one check: contrast or treatments
+      --emit            judge only: print the prompt and stop, for an agent
+                        to carry to a model itself
+      --verdict <path>  judge only: grade a reply from a file, or - for stdin
+      --skill           Print the path to the bundled agent skill
       --format <kind>   text (default), json, or sarif
       --json            Alias for --format json
   -h, --help            This
@@ -43,6 +48,11 @@ is already up. It measures what is actually painted, compositing every
 background layer behind an element rather than stopping at the first opaque
 one, and it can put the page into a state first.
 
+An agent can carry the model call instead of a shell command. --emit prints
+the prompt and the images; the agent asks a fresh context and pipes the JSON
+back to --verdict -, which checks it against the checklist the same way. Run
+taste-check judge --skill for the bundled skill that wires this up.
+
 The judge is a separate command because it runs a model, and a model's
 verdict is not reproducible. Its verdicts print as notes and do not affect
 the exit code unless judge.failOn is set to "fail". Whether the judge ran
@@ -50,7 +60,15 @@ at all is a different question: no screenshots, a command that failed, or
 a reply that skipped a checklist line all exit 1 either way.`;
 
 function parseArgs(argv) {
-  const options = { config: 'tastecheck.config.json', only: null, format: 'text', command: 'check' };
+  const options = {
+    config: 'tastecheck.config.json',
+    only: null,
+    format: 'text',
+    command: 'check',
+    emit: false,
+    verdict: null,
+    skill: false,
+  };
   // One positional, and only in first position, so a stray argument is an
   // error rather than something silently ignored.
   if (argv[0] === 'judge' || argv[0] === 'runtime') {
@@ -61,7 +79,11 @@ function parseArgs(argv) {
     const arg = argv[i];
     const next = () => {
       const value = argv[i + 1];
-      if (value === undefined || value.startsWith('-')) throw new Error(`${arg} needs a value`);
+      // A bare "-" is a value, not a flag: it is the conventional name for
+      // stdin and --verdict takes it.
+      if (value === undefined || (value !== '-' && value.startsWith('-'))) {
+        throw new Error(`${arg} needs a value`);
+      }
       i += 1;
       return value;
     };
@@ -74,13 +96,15 @@ function parseArgs(argv) {
         throw new Error(`--only takes "contrast" or "treatments", not "${options.only}"`);
       }
     } else if (arg === '--json') options.format = 'json';
+    else if (arg === '--emit') options.emit = true;
+    else if (arg === '--skill') options.skill = true;
+    else if (arg === '--verdict') options.verdict = next();
     else if (arg === '--format') {
       options.format = next();
       if (!['text', 'json', 'sarif'].includes(options.format)) {
         throw new Error(`--format takes text, json or sarif, not "${options.format}"`);
       }
-    }
-    else if (!arg.startsWith('-')) throw new Error(`unknown command "${arg}"`);
+    } else if (!arg.startsWith('-')) throw new Error(`unknown command "${arg}"`);
     else throw new Error(`unknown option "${arg}"`);
   }
   return options;
@@ -108,6 +132,19 @@ if (options.version) {
   process.exit(0);
 }
 
+// Printing the bundled skill needs no config: it is the same file every time.
+if (options.skill) {
+  console.log(fileURLToPath(new URL('../skills/taste-check-judge/SKILL.md', import.meta.url)));
+  process.exit(0);
+}
+
+for (const flag of ['emit', 'verdict']) {
+  if (options[flag] && options.command !== 'judge') {
+    die(`--${flag} applies to \`taste-check judge\`, not to ${options.command}`);
+  }
+}
+if (options.emit && options.verdict) die('--emit prints a prompt and --verdict grades a reply, so not both');
+
 const loaded = load(options.config);
 if (!loaded.ok) {
   console.error(`taste-check: ${options.config} could not be used:\n`);
@@ -123,8 +160,34 @@ if (options.command !== 'check' && !loaded.config[options.command]) {
   die(`${options.config} defines no "${options.command}" block.`);
 }
 
+// --emit stops before any model is involved, so it has no findings to report
+// and no exit code to earn. It still refuses to hand back a prompt when the
+// screenshots or the checklist are missing.
+if (options.emit) {
+  const prepared = prepareJudge(loaded.config.judge, loaded.dir);
+  if (!prepared.ok) {
+    console.log(toText([prepared.result]));
+    process.exit(1);
+  }
+  console.log(prepared.prompt);
+  process.exit(0);
+}
+
 let results;
-if (options.command === 'judge') results = judge(loaded.config, loaded.dir);
+if (options.verdict) {
+  const prepared = prepareJudge(loaded.config.judge, loaded.dir);
+  if (!prepared.ok) {
+    console.log(toText([prepared.result]));
+    process.exit(1);
+  }
+  let reply;
+  try {
+    reply = readFileSync(options.verdict === '-' ? 0 : options.verdict, 'utf8');
+  } catch {
+    die(`cannot read the verdict from ${options.verdict === '-' ? 'stdin' : options.verdict}`);
+  }
+  results = [gradeVerdict(reply, prepared, loaded.config.judge)];
+} else if (options.command === 'judge') results = judge(loaded.config, loaded.dir);
 else if (options.command === 'runtime') results = await runtime(loaded.config, loaded.dir);
 else results = run(loaded.config, loaded.dir, { only: options.only });
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@josueavalosjim/taste-check",
-  "version": "0.6.0",
+  "version": "0.7.0",
   "description": "Design review in CI with a line down the middle: measured checks that gate the build (WCAG contrast from your tokens or from a real rendered page, one-off values in your markup) and a fresh-eyes model judge whose verdicts stay advisory. Zero dependencies. Ships no design rules of its own.",
   "keywords": [
     "accessibility",
@@ -34,6 +34,7 @@
     "bin",
     "src",
     "schema",
+    "skills",
     "README.md",
     "LICENSE"
   ],

package/skills/taste-check-judge/SKILL.md

@@ -0,0 +1,82 @@
+---
+name: taste-check-judge
+description: Run the taste-check fresh-eyes judge on a design, using a fresh subagent as the judge rather than a shell command. Use after building or changing a screen, when asked to review or critique a design, or on any request to run the taste gate. Requires a tastecheck.config.json with a judge block.
+---
+
+# taste-check judge
+
+You are the transport, not the judge.
+
+`taste-check` builds the prompt and checks the answer. You carry the prompt to a
+model and carry the reply back. The model you carry it to must not be you.
+
+## Why not you
+
+You are reading this inside a session that has context: what was built, what it
+was for, which tradeoffs were made and why. That context is exactly what
+disqualifies you from judging the result. The reasoning that justified each
+choice is still sitting here ready to justify it again, and a review that
+reaches for it is not a review.
+
+So the judge is a separate agent with none of it. That is the whole mechanism,
+and skipping it turns this into self-assessment with extra steps.
+
+## Steps
+
+**1. Produce the screenshots.** However this project does it. If the config has
+a `shotCommand`, the next step runs it for you.
+
+**2. Get the prompt.**
+
+```
+npx taste-check judge --emit
+```
+
+It prints the prompt, which already contains the framing and the user's
+checklist. It exits 1 and prints nothing usable if there are no screenshots or
+no checklist, which is deliberate: a judge with nothing to look at must not
+produce a verdict.
+
+**3. Ask a fresh agent.** Spawn a new general-purpose agent on the strongest
+reasoning tier available. Give it:
+
+- the prompt from step 2, verbatim
+- the screenshot files it names
+
+Give it nothing else. No summary of what changed. No statement of intent. No
+prior conversation. Do not mention which lines you expect to fail, do not say
+what you already fixed, and do not add a checklist item of your own. Every one
+of those turns the verdict into your opinion with a second signature on it.
+
+Ask it to reply with the JSON the prompt specifies and nothing else.
+
+**4. Grade the reply.**
+
+```
+npx taste-check judge --verdict reply.json
+```
+
+or pipe it on stdin with `-`. This checks the reply against the checklist:
+every line answered exactly once, no invented lines, valid verdicts. A judge
+that quietly drops the hardest line is the failure this catches.
+
+Exit code 1 means the judge could not run, or a verdict blocked under
+`failOn: "fail"`. Verdicts are otherwise advisory and print as notes.
+
+## Reporting back
+
+Report what the judge said, including the passes. Do not soften a `fail` and do
+not quietly drop an `unsure`.
+
+If you disagree with a verdict, say so as a disagreement and leave the verdict
+standing. "The judge flagged X, I think it is wrong because Y" is useful. Not
+mentioning X is not.
+
+Fix what you can, then run the whole thing again from step 1. A verdict on the
+old screenshots is not a verdict on the new ones.
+
+## What this skill does not contain
+
+Any design rules. The checklist is the user's file, named in their config, and
+if it is empty then this reports nothing and that is correct. A checklist that
+arrived with a tool is somebody else's taste wearing the tool's authority.

package/src/config.mjs

@@ -133,14 +133,16 @@ function validateTreatments(treatments, errors) {
 
 function validateJudge(judge, errors) {
   rejectUnknown(judge, ['checklist', 'shots', 'shotCommand', 'command', 'failOn'], 'judge', errors);
-  for (const key of ['checklist', 'command']) {
-    if (typeof judge[key] !== 'string' || !judge[key]) {
+  if (typeof judge.checklist !== 'string' || !judge.checklist) {
+    errors.push('judge.checklist must be a non-empty string');
+  }
+  // command is optional: an agent carrying the call with --emit and --verdict
+  // never needs one, and demanding a placeholder would be theatre.
+  for (const key of ['command', 'shotCommand']) {
+    if (judge[key] !== undefined && (typeof judge[key] !== 'string' || !judge[key])) {
       errors.push(`judge.${key} must be a non-empty string`);
     }
   }
-  if (judge.shotCommand !== undefined && (typeof judge.shotCommand !== 'string' || !judge.shotCommand)) {
-    errors.push('judge.shotCommand must be a non-empty string');
-  }
   stringArray(judge.shots, 'judge.shots', errors);
   if (judge.failOn !== undefined && judge.failOn !== 'never' && judge.failOn !== 'fail') {
     errors.push(

package/src/index.mjs

@@ -4,7 +4,7 @@
  */
 export { runContrast } from './contrast.mjs';
 export { runTreatments } from './treatments.mjs';
-export { runJudge, buildPrompt, checklistLines, extractJson } from './judge.mjs';
+export { runJudge, prepareJudge, gradeVerdict, buildPrompt, checklistLines, checklistEntries, extractJson } from './judge.mjs';
 export { runRuntime } from './runtime.mjs';
 export { connect, findBrowser } from './cdp.mjs';
 export { load, validate } from './config.mjs';

package/src/judge.mjs

@@ -154,64 +154,98 @@ export function extractJson(stdout) {
 
 const VERDICTS = new Set(['pass', 'fail', 'unsure']);
 
-export function runJudge(config, cwd) {
-  const findings = [];
-  const problems = [];
-  const { checklist: checklistPath, shots = [], shotCommand, command, failOn = 'never' } = config;
+const empty = (problems, failOn) => ({ name: 'judge', findings: [], problems, failOn, summary: '' });
+
+/**
+ * Everything the judge needs before a model is involved: the screenshots, the
+ * checklist, and the prompt built from them.
+ *
+ * Split out so the model call does not have to be a subprocess. A shell
+ * command is one way to reach a model and a poor one for an agent, which is
+ * already a model and can spawn a genuinely fresh context of its own rather
+ * than shelling out to a second copy of itself. This half prepares the call,
+ * `gradeVerdict` below checks the answer, and in between the transport is
+ * somebody else's problem.
+ *
+ * The preconditions are enforced here rather than at grading time. Handing
+ * back a prompt for zero screenshots would produce a confident verdict about
+ * nothing.
+ */
+export function prepareJudge(config, cwd) {
+  const { checklist: checklistPath, shots = [], shotCommand, failOn = 'never' } = config;
 
   if (shotCommand) {
     const made = runCommand(shotCommand, [], '', cwd);
-    if (!made.ok) {
-      problems.push(made.reason);
-      return { name: 'judge', findings, problems, failOn, summary: '' };
-    }
+    if (!made.ok) return { ok: false, result: empty([made.reason], failOn) };
   }
 
   const images = expand(shots, cwd);
   if (!images.length) {
-    problems.push(
-      `no screenshots matched ${shots.map((s) => `"${s}"`).join(', ')}. ` +
-        `A judge with nothing to look at cannot fail, so it does not get to pass either.`,
-    );
-    return { name: 'judge', findings, problems, failOn, summary: '' };
+    return {
+      ok: false,
+      result: empty(
+        [
+          `no screenshots matched ${shots.map((s) => `"${s}"`).join(', ')}. ` +
+            `A judge with nothing to look at cannot fail, so it does not get to pass either.`,
+        ],
+        failOn,
+      ),
+    };
   }
 
   let entries;
   try {
     entries = checklistEntries(readFileSync(resolve(cwd, checklistPath), 'utf8'));
   } catch {
-    problems.push(`cannot read the checklist at ${checklistPath}`);
-    return { name: 'judge', findings, problems, failOn, summary: '' };
+    return { ok: false, result: empty([`cannot read the checklist at ${checklistPath}`], failOn) };
   }
-  const lines = entries.map((e) => e.text);
-  const lineNumbers = new Map(entries.map((e) => [e.text, e.line]));
-  if (!lines.length) {
-    problems.push(
-      `${checklistPath} has no checklist lines in it. Lines to judge are list ` +
-        `items ("- ..." or "1. ..."); everything else is treated as prose.`,
-    );
-    return { name: 'judge', findings, problems, failOn, summary: '' };
+  if (!entries.length) {
+    return {
+      ok: false,
+      result: empty(
+        [
+          `${checklistPath} has no checklist lines in it. Lines to judge are list ` +
+            `items ("- ..." or "1. ..."); everything else is treated as prose.`,
+        ],
+        failOn,
+      ),
+    };
   }
 
-  const reply = runCommand(command, images, buildPrompt(lines, images.map((i) => label(i, cwd))), cwd);
-  if (!reply.ok) {
-    problems.push(reply.reason);
-    return { name: 'judge', findings, problems, failOn, summary: '' };
-  }
+  const relative = images.map((i) => label(i, cwd));
+  return {
+    ok: true,
+    entries,
+    images,
+    relativeImages: relative,
+    prompt: buildPrompt(entries.map((e) => e.text), relative),
+  };
+}
+
+/**
+ * Check a reply against the checklist it was supposed to answer.
+ *
+ * This is the half that makes the whole thing trustworthy, and it does not
+ * care where the reply came from. A judge that quietly drops the hardest line
+ * is the failure mode to guard: every remaining verdict says pass, and the one
+ * nobody answered is the one that mattered.
+ */
+export function gradeVerdict(reply, prepared, config) {
+  const { checklist: checklistPath, failOn = 'never' } = config;
+  const findings = [];
+  const problems = [];
+  const { entries, images } = prepared;
+  const lines = entries.map((e) => e.text);
+  const lineNumbers = new Map(entries.map((e) => [e.text, e.line]));
 
-  const parsed = extractJson(reply.stdout);
+  const parsed = extractJson(reply);
   if (!parsed.ok) {
-    problems.push(`${parsed.reason}. The judge must reply with the documented JSON shape.`);
-    return { name: 'judge', findings, problems, failOn, summary: '' };
+    return empty([`${parsed.reason}. The judge must reply with the documented JSON shape.`], failOn);
   }
   if (!Array.isArray(parsed.value.findings)) {
-    problems.push('the reply has no "findings" array');
-    return { name: 'judge', findings, problems, failOn, summary: '' };
+    return empty(['the reply has no "findings" array'], failOn);
   }
 
-  // Cross-check both directions. A judge that quietly drops the hardest line
-  // is the failure mode to guard: the remaining verdicts all say pass, and
-  // the line nobody answered is the one that mattered.
   const wanted = new Set(lines);
   const answered = new Set();
   for (const f of parsed.value.findings) {
@@ -246,6 +280,27 @@ export function runJudge(config, cwd) {
     failOn,
     summary: `${lines.length} ${lines.length === 1 ? 'line' : 'lines'} against ${images.length} ${
       images.length === 1 ? 'screenshot' : 'screenshots'
-    } (${images.map((i) => label(i, cwd)).join(', ')})`,
+    } (${prepared.relativeImages.join(', ')})`,
   };
 }
+
+/** The whole thing, with a configured command as the transport. */
+export function runJudge(config, cwd) {
+  const { command, failOn = 'never' } = config;
+  const prepared = prepareJudge(config, cwd);
+  if (!prepared.ok) return prepared.result;
+
+  if (!command) {
+    return empty(
+      [
+        'judge.command is not set, so there is nothing to ask. Set it, or use ' +
+          '`taste-check judge --emit` and `--verdict` to let an agent carry the call.',
+      ],
+      failOn,
+    );
+  }
+
+  const reply = runCommand(command, prepared.images, prepared.prompt, cwd);
+  if (!reply.ok) return empty([reply.reason], failOn);
+  return gradeVerdict(reply.stdout, prepared, config);
+}