@fusion.dev/helix-cli 0.190.0 → 0.191.0

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@fusion.dev/helix-cli",
   "description": "The Helix command line, for a workspace that reaches it through npm, and the visual tests it runs.",
-  "version": "0.190.0",
+  "version": "0.191.0",
   "license": "MIT",
   "author": "Ross Sheppard <ross.sheppard@fusion.dev>",
   "repository": {
@@ -41,11 +41,11 @@
     "visual-regression"
   ],
   "optionalDependencies": {
-    "@fusion.dev/helix-cli-darwin-arm64": "0.190.0",
-    "@fusion.dev/helix-cli-darwin-x64": "0.190.0",
-    "@fusion.dev/helix-cli-linux-arm64": "0.190.0",
-    "@fusion.dev/helix-cli-linux-x64": "0.190.0",
-    "@fusion.dev/helix-cli-win32-x64": "0.190.0"
+    "@fusion.dev/helix-cli-darwin-arm64": "0.191.0",
+    "@fusion.dev/helix-cli-darwin-x64": "0.191.0",
+    "@fusion.dev/helix-cli-linux-arm64": "0.191.0",
+    "@fusion.dev/helix-cli-linux-x64": "0.191.0",
+    "@fusion.dev/helix-cli-win32-x64": "0.191.0"
   },
   "peerDependencies": {
     "playwright": ">=1.50"

package/visual/README.md

@@ -76,7 +76,7 @@ export default defineConfig({
 })
 ```
 
-- **`sources`**: where the entries come from, four kinds:
+- **`sources`**: where the entries come from, five kinds:
   - `html` reads every file its `glob` matches under `root`, described next.
   - `server` pictures a server's `routes`, and every route its `sitemap` lists.
     `/` is named `index`, and a route given as `{ name, path }` takes its name.
@@ -84,6 +84,7 @@ export default defineConfig({
     `include` and `exclude` filter them by tag.
   - `catalog` pictures every `{ id, url }` in a JSON file, each `url` relative to
     the server.
+  - `pairs` compares two renders of each entry with each other, described below.
 
   The three on a server take its `url`, a `start` command that runs only when
   nothing answers there and is stopped when the run ends, and `baselines`, the
@@ -102,6 +103,19 @@ export default defineConfig({
   ends in `/` serves a directory.
 - **`settle.readyProperty`**: a property your elements carry holding a promise
   that resolves once they have rendered. `helix visual` awaits it before each shot.
+  `settle.elements` limits the custom elements a run waits for to one tag
+  prefix, for a framework that writes hyphenated tags it never defines.
+- **`aria`**: keeps each entry's accessibility snapshot, Playwright's
+  `ariaSnapshot()` of its page, beside its baselines as `<stem>.aria.yml`, and
+  compares it as text, so a lost label fails even when no pixel moved. A
+  snapshot leaves out what `mask` names. `{ hide: [...] }` keeps snapshots and
+  leaves out more, for what is read aloud and never drawn, such as a table
+  hidden visually: masked, its box would be painted over whatever it sits
+  under.
+- **`checks.code`**: fails a run on a code block that draws all of its text in
+  one color, as a highlighter that failed leaves it. It reads the page after
+  the first shot and needs no baseline. `selector` names your code blocks and
+  `minimum` the fewest characters a block holds before it is judged.
 - **`diff`**: how a shot is compared, for the whole config or for one source.
   `mode: 'pixel'` counts the pixels past `threshold`, 0.1 unless you set it,
   and fails above `pixels`, 0 unless you set it. `mode: 'perceptual'` scores
@@ -110,7 +124,8 @@ export default defineConfig({
 - **`exceptions`**: each names an `entry`, optionally a `shot`, and a `reason`.
   `unstable: true` skips the shot and names it in every run. `pixels` lets the
   shot differ by that many, and the run fails once the shot stops differing, so
-  an exception that is no longer needed is removed.
+  an exception that is no longer needed is removed. `differs: true` tolerates a
+  known difference between a pair's two sides, and fails once they match.
 - **`inspect`**: `run` is called with the Playwright page after each entry's
   first shot, and `report` once at the end with everything `run` returned.
   `report` returns `{ failed: true }` to fail the run.
@@ -119,7 +134,8 @@ export default defineConfig({
 - **`timings`**: a file to write each entry name's seconds to, which is how you
   measure `shard.cost`.
 - **`mask`**: selectors whose elements are painted over in one fixed color in
-  every picture, for what changes on its own, such as a version number.
+  every picture and left out of every accessibility snapshot, for what changes
+  on its own, such as a version number.
 - **`style`**: a stylesheet applied while each picture is taken, to hold still
   what `mask` cannot, such as a masked element whose width follows its text.
 - **`report`**: where each run writes its report, `.helix/visual` unless you
@@ -128,6 +144,38 @@ export default defineConfig({
   when a run fails.
 - **`fingerprint`**, **`network`**, and **`platform`** are described below.
 
+## Pairs
+
+A `pairs` source compares two renders of each entry with each other, with no
+baseline: React against plain HTML, a server render against a client render, a
+framework wrapper against the element it wraps. Both sides render on one machine
+seconds apart, so the platform cancels out. Pairs run anywhere, and nothing they
+picture is committed.
+
+```ts
+{
+  kind: 'pairs',
+  labels: ['HTML', 'TSX'],
+  name: 'parity',
+  prepare: async ({ changed }) => ({
+    pairs: [{ a: { body: html }, b: { body: '<div id="root"></div>' }, name: 'button' }],
+    serve: { '/bundle/': '/absolute/path/to/bundles/' },
+  }),
+}
+```
+
+- **`prepare`** builds the pairs. It runs in Node, so it can call your bundler
+  or your server renderer, and it receives the git ref `--changed` named. Each
+  side is a `body` the harness wraps, with its own `styles`, `scripts`, `head`,
+  `dir`, and `files`, or a `url`, or an `error` when it could not be built.
+- **`check`** is called on each side's page after its first shot, with the page
+  and every console message it logged, and returns problems that fail the pair.
+- **`errors`** counts console errors: `new`, an error the second side logs and
+  the first does not, is the default, `any` counts all of them, and `ignore`
+  none.
+- **`compare: false`** judges a pair by its `check`, its errors, and its tags
+  alone, with no pictures compared.
+
 ## What every run does
 
 - **Capture holds still.** Before each shot `helix visual` waits for every custom

package/visual/driver.mjs

@@ -11,6 +11,7 @@
  *   stay here, where they run.
  * - `fetch` and `wait` ask a server for a page, or wait for it to answer, so
  *   the binary needs no HTTP of its own.
+ * - `prepare` runs a pairs source's `prepare` and answers with its pairs.
  * - `start` serves the harness origin and launches Chromium.
  * - `capture` pictures entries, answering `shot` for each picture as it is
  *   taken, `item` when an entry is done, and `captured` at the end.
@@ -116,6 +117,8 @@ const state = {
   origins: new Set(),
   /** `/source/name/` to the directory relative requests resolve in, and its renames. */
   pages: new Map(),
+  /** Each pairs source's `prepare` and `check`, by the source's name. */
+  pairs: new Map(),
   /** What `inspect.run` returned for each entry, for `inspect.report`. */
   runs: [],
   serve: [],
@@ -156,7 +159,8 @@ const lookup = (path, document) => {
       if (rest === '') {
         return { html: state.documents.get(document) }
       }
-      return { file: inside(page.dir, page.files[rest] ?? rest) }
+      /* A side with no directory of its own serves nothing beside it. */
+      return { file: page.dir ? inside(page.dir, page.files[rest] ?? rest) : undefined }
     }
     if (at === 0) {
       break
@@ -288,10 +292,92 @@ const within = (seconds, promise, what) => {
   return Promise.race([promise, expired]).finally(() => clearTimeout(timer))
 }
 
+/**
+ * The accessibility snapshot of the page, without what `mask` and `aria.hide`
+ * name.
+ *
+ * **What a run paints over, it does not judge in the snapshot either.** A
+ * masked version number is still text in the accessibility tree, so every
+ * release would change the snapshot that the mask kept out of the picture.
+ * Each masked element is given `aria-hidden` for the snapshot, and the value
+ * it had is put back after.
+ */
+const ariaSnapshot = async (page, mask) => {
+  const MARK = 'data-helix-visual-aria-hidden'
+  for (const selector of mask) {
+    await page.locator(selector).evaluateAll((elements, mark) => {
+      for (const element of elements) {
+        element.setAttribute(mark, element.getAttribute('aria-hidden') ?? '')
+        element.setAttribute('aria-hidden', 'true')
+      }
+    }, MARK)
+  }
+  try {
+    return `${await page.locator('body').ariaSnapshot()}\n`
+  } finally {
+    /* A locator, as above, so an element inside a shadow root is found too. */
+    await page.locator(`[${MARK}]`).evaluateAll((elements, mark) => {
+      for (const element of elements) {
+        const was = element.getAttribute(mark)
+        if (was === '') {
+          element.removeAttribute('aria-hidden')
+        } else {
+          element.setAttribute('aria-hidden', was)
+        }
+        element.removeAttribute(mark)
+      }
+    }, MARK)
+  }
+}
+
+/**
+ * Each code block that draws all of its text in one color, as a sentence.
+ *
+ * **A highlighter that fails leaves nothing to say so.** On October 8, 2026
+ * every code block on Fusion's site lost its colors because two copies of the
+ * highlighting library were installed: the parser tagged tokens with one and
+ * the styles read the other, nothing threw, and the text came out plain. A
+ * block of `minimum` characters or more whose every character resolves to one
+ * color is that fault, and it needs no baseline to see.
+ */
+const codeBlocks = (page, { minimum, selector }) =>
+  page.evaluate(
+    ({ minimum, selector }) => {
+      const found = []
+      for (const block of document.querySelectorAll(selector)) {
+        const colors = new Set()
+        let characters = 0
+        const walker = document.createTreeWalker(block, NodeFilter.SHOW_TEXT)
+        for (let node = walker.nextNode(); node !== null; node = walker.nextNode()) {
+          const text = node.textContent.replace(/\s+/g, '')
+          if (text === '' || node.parentElement === null) {
+            continue
+          }
+          colors.add(getComputedStyle(node.parentElement).color)
+          characters += text.length
+        }
+        if (characters >= minimum && colors.size === 1) {
+          const sample = block.textContent.trim().replace(/\s+/g, ' ').slice(0, 40)
+          found.push(
+            `a code block draws its ${characters} characters in one color, ${[...colors][0]}: "${sample}"`,
+          )
+        }
+      }
+      return found
+    },
+    { minimum, selector },
+  )
+
 /** One entry, every shot of it from one navigation. */
-const take = async (page, item, { elements, readyProperty }, progress) => {
+const take = async (lane, item, { elements, readyProperty }, progress) => {
+  const { page } = lane
   const started = performance.now()
   let error = null
+  let problems = []
+  let findings = []
+  let aria = null
+  lane.errors.length = 0
+  lane.messages.length = 0
   try {
     for (const [index, shot] of item.shots.entries()) {
       progress.stage = index === 0 ? 'loading the page' : `settling for ${shot.name}`
@@ -356,6 +442,24 @@ const take = async (page, item, { elements, readyProperty }, progress) => {
           result: await state.config.inspect.run({ entry: item.entry, page }),
         })
       }
+      /* The page checks and the accessibility snapshot, on the page that was
+         pictured, after its first picture. */
+      if (index === 0 && item.checks?.code) {
+        progress.stage = 'checking its code blocks'
+        findings = findings.concat(await codeBlocks(page, item.checks.code))
+      }
+      if (index === 0 && item.aria) {
+        progress.stage = 'taking its accessibility snapshot'
+        aria = await ariaSnapshot(page, [...state.mask, ...state.hide])
+      }
+      /* A pair's own check, on each side's page after its first picture. */
+      if (index === 0 && item.check) {
+        progress.stage = 'checking the page'
+        const found = await state.pairs
+          .get(item.check)
+          .check({ console: [...lane.messages], entry: item.entry, page })
+        problems = (Array.isArray(found) ? found : found ? [found] : []).map(String)
+      }
     }
   } catch (caught) {
     error = caught instanceof Error ? caught.message.split('\n')[0] : String(caught)
@@ -367,7 +471,16 @@ const take = async (page, item, { elements, readyProperty }, progress) => {
   console.error(
     `helix: ${item.key}: ${error === null ? `${item.shots.length} ${item.shots.length === 1 ? 'shot' : 'shots'}` : 'failed'} in ${seconds.toFixed(1)} s`,
   )
-  say({ error, key: item.key, op: 'item', seconds })
+  say({
+    aria,
+    error,
+    errors: item.errors ? [...lane.errors] : [],
+    findings,
+    key: item.key,
+    op: 'item',
+    problems,
+    seconds,
+  })
 }
 
 /**
@@ -392,7 +505,7 @@ const bounded = async (lane, item, settle) => {
   const expired = new Promise((done) => {
     timer = setTimeout(() => done(true), limit * 1000)
   })
-  const finished = await Promise.race([take(lane.page, item, settle, progress), expired])
+  const finished = await Promise.race([take(lane, item, settle, progress), expired])
   clearTimeout(timer)
   if (finished !== true) {
     return
@@ -403,7 +516,24 @@ const bounded = async (lane, item, settle) => {
   say({ error: message, key: item.key, op: 'item', seconds: limit })
   const context = lane.page.context()
   await lane.page.close().catch(() => {})
-  lane.page = await context.newPage()
+  lane.page = await listened(lane, await context.newPage())
+}
+
+/**
+ * `page`, with every console message, and the errors and uncaught exceptions
+ * apart, recorded on `lane`, which `take` empties before each entry. A pair
+ * names an error one side logged and the other did not, and a pairs source's
+ * `check` is handed the messages, a warning being something only it can judge.
+ */
+const listened = (lane, page) => {
+  page.on('console', (message) => {
+    lane.messages.push({ text: message.text(), type: message.type() })
+    if (message.type() === 'error') {
+      lane.errors.push(message.text())
+    }
+  })
+  page.on('pageerror', (caught) => lane.errors.push(caught.message))
+  return page
 }
 
 /**
@@ -437,7 +567,9 @@ const lanes = async (count) => {
         return state.origins.has(origin) ? route.continue() : route.abort()
       })
     }
-    state.lanes.push({ page: await context.newPage() })
+    const lane = { errors: [], messages: [] }
+    lane.page = listened(lane, await context.newPage())
+    state.lanes.push(lane)
   }
   return state.lanes.slice(0, count)
 }
@@ -481,9 +613,30 @@ const handlers = {
       )
     }
     state.config = config
+    /* A pairs source's functions stay here and are reported as given. */
+    const sources = (config.sources ?? []).map((source) => {
+      if (source?.kind !== 'pairs') {
+        return source
+      }
+      if (typeof source.prepare !== 'function') {
+        throw new Error(
+          `the pairs source \`${source.name}\` needs \`prepare\`, a function that returns its pairs`,
+        )
+      }
+      if (source.check !== undefined && typeof source.check !== 'function') {
+        throw new Error(
+          `the pairs source \`${source.name}\` gives a \`check\` that is not a function`,
+        )
+      }
+      state.pairs.set(source.name, { check: source.check, prepare: source.prepare })
+      return { ...source, check: source.check !== undefined, prepare: true }
+    })
     const { inspect, ...values } = config
     say({
-      config: { ...JSON.parse(JSON.stringify(values)), inspect: inspect !== undefined },
+      config: {
+        ...JSON.parse(JSON.stringify({ ...values, sources })),
+        inspect: inspect !== undefined,
+      },
       op: 'config',
     })
   },
@@ -552,7 +705,26 @@ const handlers = {
     say({ failed: Boolean(reported?.failed), op: 'finished' })
   },
 
-  async start({ mask = [], network, origins = [], serve, style = null, workers }) {
+  async prepare({ changed, source }) {
+    const pair = state.pairs.get(source)
+    if (pair === undefined) {
+      throw new Error(`no pairs source is called \`${source}\``)
+    }
+    const made = await pair.prepare({ changed: changed ?? undefined })
+    const answer = Array.isArray(made) ? { pairs: made } : (made ?? {})
+    if (!Array.isArray(answer.pairs)) {
+      throw new Error(`\`prepare\` for \`${source}\` returned no \`pairs\` array`)
+    }
+    say({
+      note: answer.note ?? null,
+      op: 'prepared',
+      pairs: answer.pairs,
+      serve: answer.serve ?? {},
+    })
+  },
+
+  async start({ hide = [], mask = [], network, origins = [], serve, style = null, workers }) {
+    state.hide = hide
     state.mask = mask
     state.style = style ?? undefined
     state.network = network

package/visual/index.d.ts

@@ -129,8 +129,97 @@ export interface CatalogSource extends Served {
   kind: 'catalog'
 }
 
+/** One side of a pair: a document the harness writes, or a page at an address. */
+export interface Side {
+  /** The markup of the page's body. */
+  body?: string
+  /** The directory relative requests from the page resolve in. Nothing is served beside a side with none. */
+  dir?: string
+  /** Why the side could not be built, such as a server render that threw. The pair fails with it, unpictured. */
+  error?: string
+  /** Relative requests answered by another file in `dir`: `{ 'styles.css': 'preview.css' }`. */
+  files?: Record<string, string>
+  /** Markup placed after the harness style and before the deferred scripts, such as a style of the side's own. */
+  head?: string
+  /** URLs of scripts the page loads, each with `defer`. */
+  scripts?: string[]
+  /** URLs of stylesheets the page links, before the harness style. */
+  styles?: string[]
+  /** A page to load in place of a document, such as a server-rendered route. */
+  url?: string
+}
+
+/** Two sides of one entry, compared with each other. */
+export interface Pair {
+  /** The first side. */
+  a: Side
+  /** The second side. */
+  b: Side
+  /** What the pair is packed into shards and timed by, such as its component family. Its name when not given. */
+  group?: string
+  /** The second half of the pair's id, `<source>/<name>`. May contain `/`. */
+  name: string
+}
+
+/** What a pairs source's `prepare` returns: its pairs, or its pairs with files to serve and a note to print. */
+export type Prepared =
+  | Pair[]
+  | {
+      /** A sentence printed with the run, such as what a scoped run left out. */
+      note?: string
+      /** The pairs. */
+      pairs: Pair[]
+      /** URL paths served from absolute paths on disk, such as a directory of bundles. */
+      serve?: Record<string, string>
+    }
+
+/**
+ * Entries with two sides, compared with each other at every shot and with no baseline: React against
+ * plain HTML, a server render against a client render, a framework wrapper against the element it wraps.
+ * Both sides render on one machine seconds apart, so the platform cancels out and nothing is committed.
+ */
+export interface PairsSource {
+  /**
+   * Called on each side's page after its first shot. Returns problems that fail the pair, or none.
+   * `console` is every message the page logged.
+   */
+  check?: (context: {
+    /** Every console message the page logged, in order. */
+    console: { text: string; type: string }[]
+    /** The pair, and `side`, the label of the side on the page. */
+    entry: Entry & { side: string }
+    /** The Playwright page. */
+    page: Page
+  }) => Promise<string[] | undefined> | string[] | undefined
+  /** Whether the two sides' pictures are compared. `true` when not given. */
+  compare?: boolean
+  /** The diff for this source's shots, in place of the config's. */
+  diff?: Diff
+  /**
+   * Which console errors fail a pair. `new`, an error the second side logs and the first does not, is the
+   * default. `any` counts every error on either side, and `ignore` none.
+   */
+  errors?: 'any' | 'ignore' | 'new'
+  /** The harness's colors and inset, or `false` to picture each side as it is. */
+  harness?: boolean | Harness
+  /** Picks this source out of the others. */
+  kind: 'pairs'
+  /** What the two sides are called in a report. `['a', 'b']` when not given. */
+  labels?: [string, string]
+  /** The first half of each entry's id. One word. */
+  name: string
+  /**
+   * Builds the pairs. `changed` is the git ref `--changed` named, for a source that decides what a change
+   * since it reaches.
+   */
+  prepare: (context: {
+    /** The git ref `--changed` named, when it was given. */
+    changed?: string
+  }) => Promise<Prepared> | Prepared
+}
+
 /** Any source of entries. */
-export type Source = CatalogSource | HtmlSource | ServerSource | StorybookSource
+export type Source = CatalogSource | HtmlSource | PairsSource | ServerSource | StorybookSource
 
 /** An entry as `inspect` sees it. */
 export interface Entry {
@@ -177,6 +266,8 @@ export interface Inspect<Result = unknown> {
 
 /** One exception to the comparison, with the reason it exists. */
 export interface Exception {
+  /** A known difference between a pair's two sides, tolerated however large. It fails the run once the two match on two takes. */
+  differs?: boolean
   /** An entry's id, `styled/aura`, or its name in every source, `aura`. */
   entry: string
   /** What was measured to justify it, said beside it in every run. */
@@ -193,6 +284,22 @@ export interface Exception {
 
 /** A `visual.config.ts`. */
 export interface Config {
+  /**
+   * Whether each entry's accessibility snapshot, Playwright's `ariaSnapshot()` of its page, is kept beside
+   * its baselines as `<stem>.aria.yml` and compared as text. A lost label fails even when no pixel moved.
+   * `{ hide }` keeps it and leaves out the elements its selectors name, as `mask` does, for what is read
+   * aloud and never drawn, which `mask` would paint over by its box.
+   */
+  aria?: boolean | { hide: string[] }
+  /** Checks run on each entry's page after its first shot, which need no baseline. */
+  checks?: {
+    /**
+     * A code block that draws all of its text in one color, as a highlighter that failed leaves it. `true`
+     * reads `pre code` and a `pre` with no `code`; give `selector` for code blocks of your own, and
+     * `minimum`, 20 when not given, for the fewest characters a block holds before it is judged.
+     */
+    code?: boolean | { minimum?: number; selector?: string }
+  }
   /** How a shot is compared with its baseline, unless its source sets its own. */
   diff?: Diff
   /** Exceptions to the comparison, each with its reason. */
@@ -202,8 +309,9 @@ export interface Config {
   /** Checks of your own on each page, after its first picture. */
   inspect?: Inspect
   /**
-   * Selectors whose elements are painted over in one fixed color in every picture, for what changes on its
-   * own and says nothing about the page, such as a version number or a date.
+   * Selectors whose elements are painted over in one fixed color in every picture and left out of every
+   * accessibility snapshot, for what changes on its own and says nothing about the page, such as a version
+   * number or a date.
    */
   mask?: string[]
   /** `origin` refuses every request off the harness origin, and is the default. `open` lets them through. */