@josueavalosjim/taste-check 0.5.2 → 0.6.0

package/README.md

@@ -343,13 +343,52 @@ taste-check judge [options]   Ask a fresh-eyes judge about your screenshots
 
   -c, --config <path>   Config file (default: tastecheck.config.json)
       --only <name>     Run one check: contrast or treatments
-      --json            Machine-readable output
+      --format <kind>   text (default), json, or sarif
+      --json            Alias for --format json
       --version         Print the version
 ```
 
 Exit code is 1 if any check fails, 0 if every check ran and passed. The judge
 plays by the rules in its own section above.
 
+## SARIF
+
+```bash
+taste-check --format sarif > taste-check.sarif
+```
+
+Works on any of the three commands, and drops findings into a code scanning tab
+instead of a log nobody opens.
+
+```yaml
+- run: npx taste-check --format sarif > taste-check.sarif
+  continue-on-error: true
+- uses: github/codeql-action/upload-sarif@v3
+  with:
+    sarif_file: taste-check.sarif
+```
+
+`continue-on-error` because the step exits 1 when something fails, and you
+still want the findings uploaded when it does.
+
+Two things make this worth having over a log.
+
+**Every finding points at the line you would edit.** A class points at the
+markup. A contrast failure points at the line in your token file where the
+foreground is declared, which is the half of a pair you usually end up moving.
+A judge verdict points at the line in your checklist. Findings with no file
+behind them, like a browser that would not start, point at the config, because
+that is where the rule was written.
+
+**Fingerprints are keyed to the content of the line, not its number.** Add an
+import at the top of a file and every finding below it stays the same alert
+rather than closing and reopening as a new one. Without that a code scanning
+tab fills with churn and people stop reading it.
+
+Levels follow the same rule the exit code does. Everything that gates the build
+is an `error`. A judge verdict is a `note`, or a `warning` if you set
+`failOn`, because it is an opinion either way.
+
 ## Where it sits next to other tools
 
 This is a small tool with a narrow claim, and several of these are better than
@@ -414,9 +453,6 @@ Not built. Written down so the shape is clear.
 
 **YAML configs**, once there is a reason to take on a parser.
 
-**SARIF output**, so findings land in a code scanning tab rather than only in
-a log.
-
 **A way to run the judge from an agent skill**, not only from a shell.
 
 **`lab()` and `lch()`**, which need the D50 white point and a chromatic

package/bin/taste-check.mjs

@@ -7,9 +7,12 @@
  * exist, or scopes a pair to a theme that is not defined is a failure here,
  * not a quiet skip.
  */
+import { readFileSync } from 'node:fs';
+
 import { load } from '../src/config.mjs';
 import { judge, run, runtime } from '../src/index.mjs';
 import { failed, toJson, toText } from '../src/report.mjs';
+import { toSarif } from '../src/sarif.mjs';
 
 const USAGE = `taste-check
 
@@ -20,12 +23,21 @@ const USAGE = `taste-check
 Options:
   -c, --config <path>   Config file (default: tastecheck.config.json)
       --only <name>     Run one check: contrast or treatments
-      --json            Machine-readable output
+      --format <kind>   text (default), json, or sarif
+      --json            Alias for --format json
   -h, --help            This
       --version         Print the version
 
 Exit code is 1 if any check fails, 0 if every check ran and passed.
 
+--format sarif writes SARIF 2.1.0 on stdout, for a code scanning tab:
+
+  taste-check --format sarif > taste-check.sarif
+
+Every finding points at the line you would edit to change it. A class points
+at the markup, a contrast failure at the line in your token file where the
+foreground is declared, a judge verdict at the line in your checklist.
+
 runtime is a separate command because it needs a browser and a server that
 is already up. It measures what is actually painted, compositing every
 background layer behind an element rather than stopping at the first opaque
@@ -38,7 +50,7 @@ at all is a different question: no screenshots, a command that failed, or
 a reply that skipped a checklist line all exit 1 either way.`;
 
 function parseArgs(argv) {
-  const options = { config: 'tastecheck.config.json', only: null, json: false, command: 'check' };
+  const options = { config: 'tastecheck.config.json', only: null, format: 'text', command: 'check' };
   // One positional, and only in first position, so a stray argument is an
   // error rather than something silently ignored.
   if (argv[0] === 'judge' || argv[0] === 'runtime') {
@@ -61,7 +73,13 @@ function parseArgs(argv) {
       if (options.only !== 'contrast' && options.only !== 'treatments') {
         throw new Error(`--only takes "contrast" or "treatments", not "${options.only}"`);
       }
-    } else if (arg === '--json') options.json = true;
+    } else if (arg === '--json') options.format = 'json';
+    else if (arg === '--format') {
+      options.format = next();
+      if (!['text', 'json', 'sarif'].includes(options.format)) {
+        throw new Error(`--format takes text, json or sarif, not "${options.format}"`);
+      }
+    }
     else if (!arg.startsWith('-')) throw new Error(`unknown command "${arg}"`);
     else throw new Error(`unknown option "${arg}"`);
   }
@@ -85,11 +103,7 @@ if (options.help) {
   process.exit(0);
 }
 if (options.version) {
-  const { version } = JSON.parse(
-    await import('node:fs').then((fs) =>
-      fs.readFileSync(new URL('../package.json', import.meta.url), 'utf8'),
-    ),
-  );
+  const { version } = JSON.parse(readFileSync(new URL('../package.json', import.meta.url), 'utf8'));
   console.log(version);
   process.exit(0);
 }
@@ -118,5 +132,12 @@ if (!results.length) {
   die(`nothing to run. ${options.config} defines no ${options.only ?? 'contrast or treatments'} check.`);
 }
 
-console.log(options.json ? toJson(results) : toText(results));
+if (options.format === 'sarif') {
+  const { version } = JSON.parse(
+    readFileSync(new URL('../package.json', import.meta.url), 'utf8'),
+  );
+  console.log(toSarif(results, { version, configFile: options.config, configDir: loaded.dir }));
+} else {
+  console.log(options.format === 'json' ? toJson(results) : toText(results));
+}
 process.exit(failed(results) ? 1 : 0);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@josueavalosjim/taste-check",
-  "version": "0.5.2",
+  "version": "0.6.0",
   "description": "Design review in CI with a line down the middle: measured checks that gate the build (WCAG contrast from your tokens or from a real rendered page, one-off values in your markup) and a fresh-eyes model judge whose verdicts stay advisory. Zero dependencies. Ships no design rules of its own.",
   "keywords": [
     "accessibility",

package/src/contrast.mjs

@@ -18,7 +18,7 @@
  */
 import { readFileSync } from 'node:fs';
 import { contrastRatio, isOpaque, parseColor } from './color.mjs';
-import { parseDeclarations, resolveScopes, resolveValue, unmatchedScopes } from './css.mjs';
+import { lineAt, parseDeclarations, resolveScopes, resolveValue, unmatchedScopes } from './css.mjs';
 import { expand, label } from './files.mjs';
 
 /** A token name, or a literal colour, resolved to rgba for one theme. */
@@ -28,7 +28,7 @@ function side(spec, table, theme) {
     if (!resolved.ok) return { ok: false, reason: `theme "${theme}": ${resolved.reason}` };
     const color = parseColor(resolved.value);
     if (!color.ok) return { ok: false, reason: `theme "${theme}": ${spec} is ${color.reason}` };
-    return { ok: true, rgba: color.rgba };
+    return { ok: true, rgba: color.rgba, at: resolved.decl };
   }
   const color = parseColor(spec);
   if (!color.ok) return { ok: false, reason: `theme "${theme}": ${color.reason}` };
@@ -49,7 +49,14 @@ export function runContrast(config, cwd) {
   // One declaration list across every token file, in the order they were
   // listed, so a later file overriding an earlier one behaves like a later
   // @import would.
-  const decls = files.flatMap((file) => parseDeclarations(readFileSync(file, 'utf8')));
+  // Each declaration remembers where it was written. A contrast failure is
+  // otherwise a number with nowhere to go, and the line you would edit to fix
+  // it is the line the token is declared on.
+  const decls = files.flatMap((file) => {
+    const source = readFileSync(file, 'utf8');
+    const where = { file: label(file, cwd), };
+    return parseDeclarations(source).map((d) => ({ ...d, ...where, line: lineAt(source, d.index) }));
+  });
 
   for (const theme of themes) {
     const table = resolveScopes(decls, theme.scopes);
@@ -111,6 +118,9 @@ export function runContrast(config, cwd) {
         ratio,
         min: pair.min,
         pass: ratio >= pair.min,
+        // Point at the foreground: it is the half a contrast failure is
+        // usually fixed by moving.
+        at: fg.at ? { file: fg.at.file, line: fg.at.line } : null,
       });
     }
   }

package/src/judge.mjs

@@ -86,10 +86,17 @@ CHECKLIST:
  * returns a verdict on your explanatory paragraph.
  */
 export function checklistLines(text) {
-  return text
-    .split('\n')
-    .filter((l) => /^\s*(?:[-*+]|\d+[.)])\s+\S/.test(l))
-    .map((l) => l.replace(/^\s*(?:[-*+]|\d+[.)])\s+/, '').trim());
+  return checklistEntries(text).map((e) => e.text);
+}
+
+/** The same, keeping the line each one was written on so a finding can point at it. */
+export function checklistEntries(text) {
+  const entries = [];
+  text.split('\n').forEach((raw, i) => {
+    if (!/^\s*(?:[-*+]|\d+[.)])\s+\S/.test(raw)) return;
+    entries.push({ text: raw.replace(/^\s*(?:[-*+]|\d+[.)])\s+/, '').trim(), line: i + 1 });
+  });
+  return entries;
 }
 
 /**
@@ -169,13 +176,15 @@ export function runJudge(config, cwd) {
     return { name: 'judge', findings, problems, failOn, summary: '' };
   }
 
-  let lines;
+  let entries;
   try {
-    lines = checklistLines(readFileSync(resolve(cwd, checklistPath), 'utf8'));
+    entries = checklistEntries(readFileSync(resolve(cwd, checklistPath), 'utf8'));
   } catch {
     problems.push(`cannot read the checklist at ${checklistPath}`);
     return { name: 'judge', findings, problems, failOn, summary: '' };
   }
+  const lines = entries.map((e) => e.text);
+  const lineNumbers = new Map(entries.map((e) => [e.text, e.line]));
   if (!lines.length) {
     problems.push(
       `${checklistPath} has no checklist lines in it. Lines to judge are list ` +
@@ -219,7 +228,12 @@ export function runJudge(config, cwd) {
       continue;
     }
     answered.add(f.line);
-    findings.push({ line: f.line, verdict: f.verdict, why: (f.why ?? '').trim() });
+    findings.push({
+      line: f.line,
+      verdict: f.verdict,
+      why: (f.why ?? '').trim(),
+      at: { file: checklistPath, line: lineNumbers.get(f.line) },
+    });
   }
   for (const line of lines) {
     if (!answered.has(line)) problems.push(`the judge did not answer "${line}"`);

package/src/report.mjs

@@ -28,7 +28,7 @@ function contrastLines(result) {
 
 function treatmentLines(result) {
   return [
-    ...result.failures.map((text) => ({ level: 'fail', text })),
+    ...result.failures.map((f) => ({ level: 'fail', text: `${f.file}:${f.line} ${f.message}` })),
     ...result.problems.map((text) => ({ level: 'error', text })),
   ];
 }

package/src/sarif.mjs

@@ -0,0 +1,236 @@
+/**
+ * SARIF 2.1.0 output, so findings land in a code scanning tab instead of only
+ * in a log nobody opens.
+ *
+ * The part worth getting right is locations. A format conversion that emits
+ * every finding against the config file is technically valid SARIF and useless
+ * in practice: the annotations all pile onto one line and none of them say
+ * where the problem is.
+ *
+ * So every finding points at the line you would edit to change it. An
+ * unapproved class points at the markup. A contrast failure points at the line
+ * in the token file where the foreground is declared, which is the half of a
+ * pair you usually end up moving. A judge verdict points at the line in your
+ * checklist. Where a finding genuinely has no file behind it, it points at the
+ * config, because that is where the rule was written.
+ */
+import { createHash } from 'node:crypto';
+import { readFileSync } from 'node:fs';
+import { relative, resolve, sep } from 'node:path';
+
+import { linesFor } from './report.mjs';
+
+const HELP = 'https://github.com/josueavalosjim/taste-check#readme';
+
+/**
+ * Levels are the same judgement the exit code makes. A judge verdict is
+ * advisory, so it is a note; everything that gates the build is an error.
+ */
+const RULES = [
+  ['contrast/below-floor', 'error', 'A declared pair is under the ratio it was given.'],
+  ['contrast/unmeasurable', 'error', 'A pair could not be measured: a missing token, a dead scope, a translucent background, or a colour that would not parse.'],
+  ['treatments/unapproved-class', 'error', 'A class name that is not on the approved list.'],
+  ['treatments/one-off-value', 'error', 'A literal colour or length hardcoded into an inline style.'],
+  ['treatments/unscannable', 'error', 'No markup matched, so nothing was checked.'],
+  ['runtime/below-floor', 'error', 'A target on the rendered page is under the ratio it was given.'],
+  ['runtime/unmeasurable', 'error', 'A target could not be measured: no element, nothing rendered, an edge with no width, or no browser.'],
+  ['judge/verdict', 'note', 'A checklist line the judge did not pass. Advisory: a model verdict is not reproducible.'],
+  ['judge/did-not-run', 'error', 'The judge could not run, which is a fact rather than an opinion.'],
+];
+
+const index = new Map(RULES.map(([id], i) => [id, i]));
+
+/**
+ * SARIF URIs are relative to the run's root, which for code scanning is the
+ * repository. Paths inside a config are relative to the config file, so they
+ * have to be re-rooted here. Getting this wrong does not fail validation, it
+ * just hangs every annotation on a path that does not exist.
+ */
+function uriFor(file, { configDir, root }) {
+  const absolute = resolve(configDir, file);
+  return relative(root, absolute).split(sep).join('/');
+}
+
+/**
+ * A fingerprint keyed to the content of the line rather than to its number.
+ *
+ * This is what keeps an alert the same alert when the file shifts. Without it
+ * every finding below an added import reads as a new problem and an old one
+ * closed, which turns the code scanning tab into churn and trains people to
+ * ignore it. Hashing the line's text rather than its position is what survives
+ * the shift. The rule id and a key naming what the finding is about are in
+ * there too, because two one-off values on the same line are two findings and
+ * a fingerprint that collides makes them one. The key is an identity, not the
+ * message: rewording a message must not close an alert and open a new one.
+ */
+function fingerprint(uri, line, ruleId, key, ctx) {
+  let text = '';
+  try {
+    text = (readFileSync(resolve(ctx.root, uri), 'utf8').split('\n')[line - 1] ?? '').trim();
+  } catch {
+    /* a file we cannot read still gets a stable fingerprint from its path */
+  }
+  return {
+    primaryLocationLineHash: createHash('sha256')
+      .update(`${ruleId}\u0000${uri}\u0000${text}\u0000${key}`)
+      .digest('hex')
+      .slice(0, 32),
+  };
+}
+
+const location = (file, ctx, line) => ({
+  physicalLocation: {
+    artifactLocation: { uri: uriFor(file, ctx), uriBaseId: '%SRCROOT%' },
+    // A region with no line is invalid, and line 1 is the honest fallback for
+    // "this file, we cannot be more specific".
+    region: { startLine: Math.max(1, line ?? 1) },
+  },
+});
+
+function resultsFor(check, ctx) {
+  const here = location(ctx.configFile, { ...ctx, configDir: ctx.root }, 1);
+  const out = [];
+
+  if (check.name === 'contrast' || check.name === 'runtime') {
+    for (const sample of check.samples ?? []) {
+      if (sample.pass) continue;
+      out.push({
+        ruleId: `${check.name}/below-floor`,
+        key: `${sample.fg}|${sample.bg}|${sample.theme}`,
+        level: 'error',
+        message: {
+          text:
+            `${sample.ratio.toFixed(2)}:1 against a floor of ${sample.min} for ${sample.fg} on ` +
+            `${sample.bg} in ${sample.theme}${sample.note ? `. ${sample.note}` : ''}`,
+        },
+        locations: [sample.at ? location(sample.at.file, ctx, sample.at.line) : here],
+      });
+    }
+    for (const problem of check.problems ?? []) {
+      out.push({
+        ruleId: `${check.name}/unmeasurable`,
+        key: problem,
+        level: 'error',
+        message: { text: problem },
+        locations: [here],
+      });
+    }
+    return out;
+  }
+
+  if (check.name === 'treatments') {
+    for (const failure of check.failures ?? []) {
+      out.push({
+        ruleId: failure.rule,
+        key: `${failure.subject ?? failure.message}`,
+        level: 'error',
+        message: { text: failure.message },
+        locations: [location(failure.file, ctx, failure.line)],
+      });
+    }
+    for (const problem of check.problems ?? []) {
+      out.push({
+        ruleId: 'treatments/unscannable',
+        key: problem,
+        level: 'error',
+        message: { text: problem },
+        locations: [here],
+      });
+    }
+    return out;
+  }
+
+  if (check.name === 'judge') {
+    for (const finding of check.findings ?? []) {
+      if (finding.verdict === 'pass') continue;
+      out.push({
+        ruleId: 'judge/verdict',
+        key: finding.line,
+        // Mirrors the exit code: advisory unless the config opted into blocking.
+        level: check.failOn === 'fail' && finding.verdict === 'fail' ? 'warning' : 'note',
+        message: { text: `${finding.verdict}: ${finding.line}${finding.why ? `. ${finding.why}` : ''}` },
+        locations: [finding.at ? location(finding.at.file, ctx, finding.at.line) : here],
+      });
+    }
+    for (const problem of check.problems ?? []) {
+      out.push({
+        ruleId: 'judge/did-not-run',
+        key: problem,
+        level: 'error',
+        message: { text: problem },
+        locations: [here],
+      });
+    }
+  }
+  return out;
+}
+
+/** Attach a fingerprint to each result, derived from where it points. */
+function withFingerprints(found, ctx) {
+  return found.map((r) => {
+    const place = r.locations[0].physicalLocation;
+    const { key, ...rest } = r;
+    return {
+      ...rest,
+      partialFingerprints: fingerprint(
+        place.artifactLocation.uri,
+        place.region.startLine,
+        r.ruleId,
+        r.key ?? r.message.text,
+        ctx,
+      ),
+    };
+  });
+}
+
+export function toSarif(results, { version, configFile, configDir, root = process.cwd() }) {
+  const ctx = {
+    configFile: configFile ?? 'tastecheck.config.json',
+    configDir: configDir ?? root,
+    root,
+  };
+  const found = results.flatMap((check) => resultsFor(check, ctx));
+  // Only the rules that actually fired, so the tab is not padded with rules
+  // this run had no opinion about.
+  const fired = [...new Set(found.map((r) => r.ruleId))];
+
+  return JSON.stringify(
+    {
+      $schema: 'https://json.schemastore.org/sarif-2.1.0.json',
+      version: '2.1.0',
+      runs: [
+        {
+          tool: {
+            driver: {
+              name: 'taste-check',
+              version,
+              informationUri: HELP,
+              rules: fired.map((id) => {
+                const [, level, description] = RULES[index.get(id)];
+                return {
+                  id,
+                  name: id.replace(/[/-](.)/g, (_, c) => c.toUpperCase()),
+                  shortDescription: { text: description },
+                  helpUri: HELP,
+                  defaultConfiguration: { level },
+                };
+              }),
+            },
+          },
+          results: withFingerprints(found, ctx).map((r) => ({
+            ...r,
+            ruleIndex: fired.indexOf(r.ruleId),
+          })),
+        },
+      ],
+    },
+    null,
+    2,
+  );
+}
+
+/** Every rule this tool can emit, for the docs and for the tests to check. */
+export const ruleIds = () => RULES.map(([id]) => id);
+
+// Re-exported so a caller does not have to know that a note is not a failure.
+export { linesFor };

package/src/treatments.mjs

@@ -202,20 +202,29 @@ export function runTreatments(config, cwd) {
 
   for (const file of files) {
     const source = readFileSync(file, 'utf8');
-    const where = (at) => `${label(file, cwd)}:${lineOf(source, at)}`;
+    const where = (at) => ({ file: label(file, cwd), line: lineOf(source, at) });
 
     for (const tag of openTags(source, elements)) {
       for (const { name, at } of classesOf(tag.attrs)) {
         if (approved.has(name)) continue;
         if (allowPrefixes.some((p) => name.startsWith(p))) continue;
-        failures.push(`${where(tag.start + at)} class "${name}" on <${tag.name}> is not approved`);
+        failures.push({
+          rule: 'treatments/unapproved-class',
+          ...where(tag.start + at),
+          subject: name,
+          message: `class "${name}" on <${tag.name}> is not approved`,
+        });
       }
       for (const { text, at } of inlineValues(tag.attrs)) {
         if (allowedValues.has(text.toLowerCase())) continue;
-        failures.push(
-          `${where(tag.start + at)} inline value "${text}" on <${tag.name}> is a one-off. ` +
+        failures.push({
+          rule: 'treatments/one-off-value',
+          ...where(tag.start + at),
+          subject: text,
+          message:
+            `inline value "${text}" on <${tag.name}> is a one-off. ` +
             `Use a token, or add it to approvedValues.`,
-        );
+        });
       }
     }
   }