@fusion.dev/helix-cli 0.195.0 → 0.197.0

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@fusion.dev/helix-cli",
   "description": "The Helix command line, for a workspace that reaches it through npm, and the visual tests it runs.",
-  "version": "0.195.0",
+  "version": "0.197.0",
   "license": "MIT",
   "author": "Ross Sheppard <ross.sheppard@fusion.dev>",
   "repository": {
@@ -41,16 +41,20 @@
     "visual-regression"
   ],
   "optionalDependencies": {
-    "@fusion.dev/helix-cli-darwin-arm64": "0.195.0",
-    "@fusion.dev/helix-cli-darwin-x64": "0.195.0",
-    "@fusion.dev/helix-cli-linux-arm64": "0.195.0",
-    "@fusion.dev/helix-cli-linux-x64": "0.195.0",
-    "@fusion.dev/helix-cli-win32-x64": "0.195.0"
+    "@fusion.dev/helix-cli-darwin-arm64": "0.197.0",
+    "@fusion.dev/helix-cli-darwin-x64": "0.197.0",
+    "@fusion.dev/helix-cli-linux-arm64": "0.197.0",
+    "@fusion.dev/helix-cli-linux-x64": "0.197.0",
+    "@fusion.dev/helix-cli-win32-x64": "0.197.0"
   },
   "peerDependencies": {
+    "@anthropic-ai/sdk": ">=0.132",
     "playwright": ">=1.50"
   },
   "peerDependenciesMeta": {
+    "@anthropic-ai/sdk": {
+      "optional": true
+    },
     "playwright": {
       "optional": true
     }

package/visual/README.md

@@ -24,8 +24,11 @@ rest of `helix` needs no browser, so Playwright is an optional peer, and
 helix visual test                    compare every shot, and fail on a difference
 helix visual test button             only the entries named button
 helix visual update                  write only the baselines the comparison rejects
+helix visual update --boxes          also write element boxes into baselines that have none
 helix visual test --shard 0 --of 4   the first of four shards, for CI
 helix visual report                  open the report of the last run
+helix visual report --merge a b      the groups of several shards' reports as one
+helix visual report --history runs   what earlier runs' records say
 helix visual compare a.png b.png     compare two PNG files, with no browser
 ```
 
@@ -35,8 +38,20 @@ helix visual compare a.png b.png     compare two PNG files, with no browser
   named `<stem><suffix>.diff.png`. Add `*.diff.png` to your `.gitignore`.
 - **`helix visual update`** pictures every entry and writes a baseline only where the
   comparison would fail. Review the files it writes before you commit them.
+  `--boxes` also writes the element boxes into each matching baseline that has
+  none, inside its own PNG, leaving its pixels as they are, so the run can say
+  how far a changed element moved.
 - **`helix visual report`** opens the report the last run wrote. `--markdown` prints
-  its summary as Markdown instead, for a CI job's summary page.
+  its summary as Markdown instead, for a CI job's summary page. `--merge`
+  takes the reports of a run's shards, each a report directory or its
+  `run.json`, and prints their changes as one set of groups, so a change
+  spread across shards is one group. With a `triage` in the config, a merge
+  asks about each group once, reading the diff since `--changed`.
+  `--history` takes the records of earlier runs, each a `run.json` or a
+  directory holding any number, and names the shots that needed a second
+  take in more than one run or that differed and matched on one commit, and
+  the changes that failed on more than one branch. With `--merge`, each group
+  also names the other branches it failed on lately.
 - **`helix visual compare`** compares two PNG files. `--diff` writes the image of the
   difference, and `--threshold` sets how far apart two colors may be and still
   count as the same pixel, from 0 to 1.
@@ -137,6 +152,27 @@ export default defineConfig({
   covers the region, and the file a manifest says defines it. A baseline
   carries its elements' boxes inside its PNG once a run writes it, and from
   then on the run also says how far each element moved.
+- **`triage`**: a model that explains each group of changes a failed run
+  found, as intended, unintended, or unclear, with a sentence on why. It
+  never approves a baseline: the answers are the order to review in. Off when
+  not given, and then no run makes a network call.
+  - `provider: 'anthropic'`, the default, asks Claude through
+    `@fusion.dev/helix-cli`'s optional peer `@anthropic-ai/sdk`, with the
+    key the SDK finds, `ANTHROPIC_API_KEY` among them. `model` is
+    `claude-opus-5-5` unless you name another.
+  - `provider: 'ollama'` asks a model on an Ollama endpoint, `url`,
+    `http://127.0.0.1:11434` unless you name another, so nothing leaves your
+    machine. `model` names what you pulled.
+  - Only words are sent: the attribution, the accessibility difference, the
+    diff of the attributed files since the run's `--changed` ref, and the
+    names of every file changed since it.
+    `images: true` also sends the baseline and the capture of each group's
+    largest region, cropped and no wider than 512 pixels.
+  - A run split into shards leaves the triage to `helix visual report
+    --merge`, which asks once about a group the shards share.
+  - `groups`, 10 unless you set it, caps the requests a run makes, the
+    largest group first. A run with no differences makes none, and the last
+    line says how many tokens each run read and wrote.
 - **`diff`**: how a shot is compared, for the whole config or for one source.
   `mode: 'pixel'` counts the pixels past `threshold`, 0.1 unless you set it,
   and fails above `pixels`, 0 unless you set it. `mode: 'perceptual'` scores
@@ -210,6 +246,11 @@ picture is committed.
   scheme, and the harness style change between shots on the same page.
 - **A difference is taken again from a fresh page load.** A shot that matches
   on the second take passes, and the run names it.
+- **Changes are grouped by what they happened to.** Each region of a shot
+  that differs is traced to the element under it, and shots whose changes
+  read the same, apart from where they are and each element's `id`, are one
+  group: a token change that moved one kind of element across 300 pictures is
+  one group of 300. No model is involved.
 - **Requests stay on the harness origin.** Every request to another host is
   refused, so a baseline never records what another server returned.
   `network: 'open'` turns that off.
@@ -254,6 +295,50 @@ The Action lives in the Fusion repository, in `helix/visual/action`.
 `helix` and `playwright` name the commands that run each,
 `npx --no-install helix` and `npx --no-install playwright` unless you set them.
 
+- **Every check keeps its run record**, the report's `run.json`, as an
+  artifact named `visual-run-<job>-<index>` for `keep` days, 30 unless you set
+  it. Each record names the commit and the branch it ran on.
+- **`history: 30`** reads the records of the last 30 runs of the workflow and
+  adds what they say to the summary. It reads them with the workflow's token,
+  so the job needs `permissions: actions: read`. Nothing is stored outside
+  your Actions artifacts.
+
+## Review on the pull request
+
+A workflow can let a reviewer retake the entries they have looked at with a
+comment, using only the retake workflow you already run. This one takes
+`/retake styled/badge site/home` from anyone who can push, on a branch of the
+repository, and dispatches the retake there:
+
+```yaml
+name: Retake on request
+on:
+  issue_comment:
+    types: [created]
+permissions:
+  actions: write
+  contents: read
+  pull-requests: write
+jobs:
+  retake:
+    if: ${{ github.event.issue.pull_request && startsWith(github.event.comment.body, '/retake') && contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.comment.author_association) }}
+    runs-on: ubuntu-latest
+    steps:
+      - env:
+          BODY: ${{ github.event.comment.body }}
+          GH_TOKEN: ${{ github.token }}
+          NUMBER: ${{ github.event.issue.number }}
+        run: |
+          head=$(gh api "repos/$GITHUB_REPOSITORY/pulls/$NUMBER" --jq .head.ref)
+          entries=$(head -n 1 <<<"$BODY" | sed 's#^/retake##' | tr -s ' ' '\n' | grep -E '^[a-z0-9-]+(/[a-z0-9-]+)?$' | tr '\n' ' ')
+          gh workflow run visual-baselines.yml --ref "$head" -f entries="$entries"
+```
+
+It dispatches a `visual-baselines.yml` that runs the Action with
+`mode: retake` and passes an `entries` input in `args`. Read the comment from
+the environment, as this does, and never paste it into a command. The retake opens its own pull request into the branch, which is
+where the pictures are looked at before they land.
+
 ## How it works
 
 - **Local-first.** `helix visual` runs on your machine and in your CI. It needs no

package/visual/driver.mjs

@@ -762,6 +762,65 @@ const handlers = {
     say({ browser: `chromium ${state.browser.version()}`, op: 'started' })
   },
 
+  /* **Each group asked in turn, and an answer that fails is one group's
+     failure.** A key that is missing or refused fails every request alike,
+     so the first such failure is the answer for the rest. */
+  async triage({ groups, model, provider, system, url }) {
+    const settings = { model, system, url }
+    const answers = []
+    let stopped = null
+    let client = null
+    if (provider === 'anthropic') {
+      try {
+        client = new (await anthropic())()
+      } catch (caught) {
+        stopped = caught instanceof Error ? caught.message.split('\n')[0] : String(caught)
+      }
+    }
+    for (const group of groups) {
+      if (stopped !== null) {
+        answers.push({ error: stopped })
+        continue
+      }
+      try {
+        const answer =
+          provider === 'anthropic'
+            ? await askClaude(client, settings, group)
+            : await askOllama(settings, group)
+        answers.push(answer)
+      } catch (caught) {
+        /* The API's own sentence when it gave one, without the request id
+           that makes every failure's text unique. */
+        const said = caught?.error?.error?.message
+        const why =
+          typeof said === 'string'
+            ? `${caught.status}: ${said}`
+            : caught instanceof Error
+              ? caught.message.split('\n')[0]
+              : String(caught)
+        answers.push({ error: why })
+        /* A refused key, an endpoint that is down, or the same failure
+           twice running, a balance that has run out among them, is one that
+           every request after it would meet. */
+        const before = answers.at(-2)
+        if (
+          caught?.status === 401 ||
+          caught?.status === 403 ||
+          caught?.code === 'ECONNREFUSED' ||
+          before?.error === why
+        ) {
+          stopped = why
+        }
+      }
+    }
+    /* A failure that stopped every request is said once, for the run. */
+    say({
+      answers,
+      op: 'triaged',
+      stopped: answers.every((one) => one.error === stopped) ? stopped : null,
+    })
+  },
+
   /* Polls until anything answers, a refusal of the connection being the one
      thing that means nothing is listening yet. */
   async wait({ seconds, url }) {
@@ -803,6 +862,109 @@ const playwright = async () => {
   }
 }
 
+/**
+ * The Anthropic SDK, or a failure that says how to install it: an optional
+ * peer for the same reason Playwright is, needed only by a config that asks
+ * for a triage.
+ */
+const anthropic = async () => {
+  try {
+    return (await import('@anthropic-ai/sdk')).default
+  } catch (error) {
+    if (
+      error?.code !== 'ERR_MODULE_NOT_FOUND' ||
+      !String(error.message).includes("'@anthropic-ai/sdk'")
+    ) {
+      throw error
+    }
+    throw new Error(
+      'the config asks for a triage by Claude, through the Anthropic SDK, which is not installed. Install it with `npm install --save-dev @anthropic-ai/sdk`.',
+    )
+  }
+}
+
+/** The answer every triage request is held to. */
+const VERDICT = {
+  additionalProperties: false,
+  properties: {
+    explanation: { type: 'string' },
+    verdict: { enum: ['intended', 'unintended', 'unclear'], type: 'string' },
+  },
+  required: ['verdict', 'explanation'],
+  type: 'object',
+}
+
+/**
+ * One group asked of Claude: the words and any pictures, answered as JSON
+ * held to `VERDICT`. A request a safety classifier declines goes to the
+ * fallback model Anthropic routes it to, and one declined there too is an
+ * answer of its own, not a failure of the run.
+ */
+const askClaude = async (client, { model, system }, { images, text }) => {
+  const response = await client.beta.messages.create({
+    betas: ['server-side-fallback-2026-07-01'],
+    fallbacks: 'default',
+    max_tokens: 16000,
+    messages: [
+      {
+        content: [
+          ...images.map((data) => ({
+            source: { data, media_type: 'image/png', type: 'base64' },
+            type: 'image',
+          })),
+          { text, type: 'text' },
+        ],
+        role: 'user',
+      },
+    ],
+    model,
+    output_config: { effort: 'medium', format: { schema: VERDICT, type: 'json_schema' } },
+    system,
+  })
+  const usage = response.usage ?? {}
+  const input =
+    (usage.input_tokens ?? 0) +
+    (usage.cache_creation_input_tokens ?? 0) +
+    (usage.cache_read_input_tokens ?? 0)
+  const output = usage.output_tokens ?? 0
+  if (response.stop_reason === 'refusal') {
+    return { error: 'the model declined to answer', input, output }
+  }
+  const words = response.content.find((block) => block.type === 'text')?.text ?? ''
+  return { ...JSON.parse(words), input, output }
+}
+
+/**
+ * One group asked of a model on an Ollama endpoint, with the same answer
+ * asked for through its `format`.
+ */
+const askOllama = async ({ model, system, url }, { images, text }) => {
+  const answer = await fetch(`${(url ?? 'http://127.0.0.1:11434').replace(/\/+$/, '')}/api/chat`, {
+    body: JSON.stringify({
+      format: VERDICT,
+      messages: [
+        { content: system, role: 'system' },
+        { content: text, images, role: 'user' },
+      ],
+      model,
+      stream: false,
+    }),
+    headers: { 'content-type': 'application/json' },
+    method: 'POST',
+  })
+  if (!answer.ok) {
+    throw new Error(
+      `the endpoint answered ${answer.status}: ${(await answer.text()).slice(0, 200)}`,
+    )
+  }
+  const body = await answer.json()
+  return {
+    ...JSON.parse(body.message?.content ?? '{}'),
+    input: body.prompt_eval_count ?? 0,
+    output: body.eval_count ?? 0,
+  }
+}
+
 const input = Buffer.alloc(1 << 16)
 const decoder = new StringDecoder('utf8')
 let pending = ''

package/visual/index.d.ts

@@ -392,6 +392,23 @@ export interface Config {
   style?: string
   /** A file to write the seconds each entry name took, relative to the config. */
   timings?: string
+  /**
+   * A model that explains each group of changes a failed run found, as intended, unintended or unclear. It
+   * never approves a baseline. Off when not given, and then no run makes a network call. Only words are sent
+   * unless `images` is `true`.
+   */
+  triage?: {
+    /** How many groups are asked about, the largest first. 10 when not given. */
+    groups?: number
+    /** Whether the baseline and capture of each group's largest region are sent, cropped. */
+    images?: boolean
+    /** The model. `claude-opus-5-5` for `anthropic` when not given; required for `ollama`. */
+    model?: string
+    /** `anthropic`, through the optional peer `@anthropic-ai/sdk`, or `ollama`, on an endpoint of your own. */
+    provider?: 'anthropic' | 'ollama'
+    /** The Ollama endpoint. `http://127.0.0.1:11434` when not given. */
+    url?: string
+  }
 }
 
 /** Returns the config unchanged, typed. */