jekyll-theme-zer0 1.29.0 → 1.30.0

data/scripts/ai/usage_report.rb

@@ -0,0 +1,225 @@
+#!/usr/bin/env ruby
+# =============================================================================
+# usage_report.rb — publish a job's AI usage records (summary, artifact, PR comment)
+# -----------------------------------------------------------------------------
+# The reporting half of AI metering. usage.rb captured one JSONL record per AI
+# call into AI_USAGE_DIR/records.jsonl; this script, run at the end of the
+# job (the claude-run composite calls it automatically), does four things:
+#
+#   1. ATTRIBUTE  — resolve which PR the spend belongs to: an explicit --pr,
+#                   the pull_request event payload, or the pr-result.txt file
+#                   the factory/fleet agents write after opening a PR (that
+#                   run's records become the PR's CREATION cost).
+#   2. CONSUME    — move records.jsonl to a reported-*.jsonl file (so a second
+#                   AI step in the same job never double-reports) and print
+#                   `file=` / `name=` to $GITHUB_OUTPUT for the artifact upload.
+#   3. SUMMARIZE  — append a per-call table to $GITHUB_STEP_SUMMARY.
+#   4. COMMENT    — upsert ONE sticky "AI usage & cost" comment on the PR,
+#                   found by its <!-- lh-ai-usage --> marker (never
+#                   `--edit-last`, which grabs whatever the bot said last).
+#                   The comment embeds its own base64 data blob, so each run
+#                   merges records by id — cumulative, idempotent, and safe to
+#                   re-run. Concurrent jobs can still race the read-merge-write
+#                   (last writer wins for the VIEW); the artifacts + nightly
+#                   ledger remain the source of truth.
+#
+# Every dollar figure is API-equivalent: what the tokens would bill at list
+# prices. Subscription (OAuth) runs cost $0 marginal — the label says so.
+# Best-effort by design: metering must never fail the job. Stdlib + `gh` only.
+#
+#   ruby scripts/ai/usage_report.rb [--pr N] [--pr-result pr-result.txt]
+# =============================================================================
+require 'json'
+require 'time'
+require 'digest'
+require 'securerandom'
+require_relative 'usage'
+
+MARKER    = '<!-- lh-ai-usage -->'.freeze
+DATA_HEAD = '<!-- lh-ai-usage-data:'.freeze
+MAX_BLOB_RECORDS = 150   # older records fold into a rollup so the comment stays < 64KB
+MAX_TABLE_ROWS   = 30
+
+pr_arg = nil
+pr_result_file = 'pr-result.txt'
+args = ARGV.dup
+until args.empty?
+  case (a = args.shift)
+  when '--pr'        then pr_arg = args.shift.to_i
+  when '--pr-result' then pr_result_file = args.shift.to_s
+  end
+end
+
+# --- 1. load this job's records ------------------------------------------------
+src = AIUsage.records_path
+records = File.exist?(src) ? File.read(src, encoding: 'UTF-8').split("\n").map { |l| JSON.parse(l) rescue nil }.compact : []
+if records.empty?
+  warn '[usage_report] no AI usage records this job — nothing to report.'
+  File.open(ENV['GITHUB_OUTPUT'], 'a') { |io| io.puts('file='); io.puts('name=') } if ENV['GITHUB_OUTPUT']
+  exit 0
+end
+
+# --- 2. attribute to a PR --------------------------------------------------------
+pr = nil
+pr_source = nil
+if pr_arg && pr_arg > 0
+  pr = pr_arg
+  pr_source = 'event'
+elsif ENV['GITHUB_EVENT_PATH'] && File.exist?(ENV['GITHUB_EVENT_PATH'])
+  ev = JSON.parse(File.read(ENV['GITHUB_EVENT_PATH'])) rescue {}
+  n = ev.dig('pull_request', 'number')
+  if n
+    pr = n.to_i
+    pr_source = 'event'
+  end
+end
+if pr.nil? && File.exist?(pr_result_file)
+  # The factory/fleet convention: the agent writes the PR/issue URL(s) it opened
+  # to pr-result.txt. The first pull URL is the PR this run CREATED — its spend
+  # is that PR's creation cost.
+  if (m = File.read(pr_result_file, encoding: 'UTF-8')[%r{/pull/(\d+)}, 1])
+    pr = m.to_i
+    pr_source = 'created'
+  end
+end
+records.each { |r| r['pr'] ||= pr; r['pr_source'] ||= pr_source if pr }
+
+# --- 3. consume: move to a reported file, hand the path to the uploader ----------
+reported = File.join(AIUsage.dir, "reported-#{Time.now.utc.strftime('%H%M%S')}-#{SecureRandom.hex(3)}.jsonl")
+File.open(reported, 'w') { |io| records.each { |r| io.puts(JSON.generate(r)) } }
+File.delete(src)
+if ENV['GITHUB_OUTPUT']
+  artifact = "ai-usage-#{ENV['GITHUB_RUN_ID'] || 'local'}-#{(ENV['GITHUB_JOB'] || 'job').gsub(/[^A-Za-z0-9_-]/, '_')}-#{SecureRandom.hex(3)}"
+  File.open(ENV['GITHUB_OUTPUT'], 'a') { |io| io.puts("file=#{reported}"); io.puts("name=#{artifact}") }
+end
+
+fmt_usd = ->(v) { format('$%.4f', v.to_f) }
+fmt_tok = ->(v) { v.to_i >= 10_000 ? "#{(v.to_i / 1000.0).round}k" : v.to_i.to_s }
+
+# --- 4. step summary --------------------------------------------------------------
+if ENV['GITHUB_STEP_SUMMARY']
+  total = records.sum { |r| r['cost_usd'].to_f }
+  lines = []
+  lines << '## 🤖 AI usage (this job)'
+  lines << ''
+  lines << '| role | model | turns | in | out | cache r/w | cost (API-equiv) | via |'
+  lines << '|---|---|---|---|---|---|---|---|'
+  records.each do |r|
+    t = r['tokens'] || {}
+    lines << "| #{r['agent'].to_s.empty? ? '—' : r['agent']} | #{r['model']} | #{r['num_turns'] || '—'} " \
+             "| #{fmt_tok.call(t['input'])} | #{fmt_tok.call(t['output'])} " \
+             "| #{fmt_tok.call(t['cache_read'])}/#{fmt_tok.call(t['cache_creation'])} " \
+             "| #{fmt_usd.call(r['cost_usd'])}#{r['cost_source'] == 'estimated' ? '*' : ''} | #{r['auth']} |"
+  end
+  lines << ''
+  # A failed call bills nothing, so it is invisible in the table above — spell
+  # out what the model refused, right where the operator is already looking.
+  records.select { |r| r['error'] }.each do |r|
+    e = r['error']
+    lines << "> ❌ **#{r['agent'].to_s.empty? ? 'AI call' : r['agent']} failed** " \
+             "(`#{e['subtype'].to_s.empty? ? "exit #{e['exit_code']}" : e['subtype']}`): #{e['message']}"
+    lines << ''
+  end
+  lines << "**Job total: #{fmt_usd.call(total)}** (API-equivalent#{records.any? { |r| r['cost_source'] == 'estimated' } ? '; * = estimated from _data/ai_pricing.yml' : ''})."
+  lines << ''
+  File.open(ENV['GITHUB_STEP_SUMMARY'], 'a') { |io| io.puts(lines.join("\n")) }
+end
+
+# --- 5. sticky PR comment (best-effort) --------------------------------------------
+repo = ENV['GITHUB_REPOSITORY'].to_s
+exit 0 if pr.nil? || repo.empty?
+unless system('gh --version > /dev/null 2>&1') && !(ENV['GH_TOKEN'].to_s + ENV['GITHUB_TOKEN'].to_s).empty?
+  warn '[usage_report] no gh/token — skipping the PR comment (records still in the artifact).'
+  exit 0
+end
+
+def gh_json(args)
+  out = IO.popen(['gh'] + args, err: %i[child out], &:read)
+  return nil unless $?.success?
+  JSON.parse(out)
+rescue StandardError
+  nil
+end
+
+# Find the existing sticky comment by MARKER (any author, any position).
+existing = nil
+page = 1
+loop do
+  batch = gh_json(['api', "repos/#{repo}/issues/#{pr}/comments?per_page=100&page=#{page}"])
+  break unless batch.is_a?(Array)
+  existing = batch.find { |c| c['body'].to_s.start_with?(MARKER) }
+  break if existing || batch.size < 100 || page >= 10
+  page += 1
+end
+
+# Merge this job's records into the comment's embedded blob (dedup by id).
+blob = { 'records' => [], 'folded' => nil }
+if existing && (m = existing['body'].to_s[/#{Regexp.escape(DATA_HEAD)}([A-Za-z0-9+\/=]+) -->/, 1])
+  blob = JSON.parse(m.unpack1('m')) rescue { 'records' => [], 'folded' => nil }
+end
+compact = ->(r) do
+  t = r['tokens'] || {}
+  { 'i' => r['id'], 't' => r['ts'], 'w' => r['workflow'], 'j' => r['job'], 'r' => r['run_id'],
+    'a' => r['agent'], 'm' => r['model'], 'ti' => t['input'].to_i, 'to' => t['output'].to_i,
+    'tr' => t['cache_read'].to_i, 'tc' => t['cache_creation'].to_i,
+    'c' => r['cost_usd'].to_f.round(6), 's' => r['cost_source'], 'au' => r['auth'],
+    'st' => r['status'], 'ps' => r['pr_source'] }
+end
+known = blob['records'].map { |r| r['i'] }
+records.each { |r| blob['records'] << compact.call(r) unless known.include?(r['id']) }
+blob['records'].sort_by! { |r| r['t'].to_s }
+while blob['records'].size > MAX_BLOB_RECORDS
+  old = blob['records'].shift
+  f = blob['folded'] ||= { 'n' => 0, 'c' => 0.0, 'ti' => 0, 'to' => 0 }
+  f['n'] += 1
+  f['c'] = (f['c'] + old['c'].to_f).round(6)
+  f['ti'] += old['ti'].to_i
+  f['to'] += old['to'].to_i
+end
+
+all      = blob['records']
+folded   = blob['folded']
+total    = all.sum { |r| r['c'].to_f } + (folded ? folded['c'].to_f : 0.0)
+creation = all.select { |r| r['ps'] == 'created' }.sum { |r| r['c'].to_f }
+downstream = total - creation
+estimated = all.any? { |r| r['s'] == 'estimated' }
+oauth_only = all.all? { |r| r['au'] == 'oauth' }
+
+body = []
+body << MARKER
+body << '## 🤖 AI usage & cost for this PR'
+body << ''
+body << "**Total: #{fmt_usd.call(total)} API-equivalent** across #{all.size + (folded ? folded['n'] : 0)} AI call(s) — " \
+        "creation #{fmt_usd.call(creation)}, reviews/fixes/checks #{fmt_usd.call(downstream)}."
+body << ''
+body << '| when (UTC) | workflow · job | role | model | out tok | cost |'
+body << '|---|---|---|---|---|---|'
+body << "| _earlier_ | _#{folded['n']} older call(s), folded_ | | | #{fmt_tok.call(folded['to'])} | #{fmt_usd.call(folded['c'])} |" if folded
+all.last(MAX_TABLE_ROWS).each do |r|
+  run_link = r['r'].to_s.empty? ? (r['w'].to_s.empty? ? 'local' : r['w']) : "[#{r['w']} · #{r['j']}](https://github.com/#{repo}/actions/runs/#{r['r']})"
+  body << "| #{r['t'].to_s[5, 11]} | #{run_link} | #{r['a'].to_s.empty? ? '—' : r['a']}#{r['ps'] == 'created' ? ' 🌱' : ''} " \
+          "| #{r['m']} | #{fmt_tok.call(r['to'])} | #{fmt_usd.call(r['c'])}#{r['s'] == 'estimated' ? '*' : ''} |"
+end
+body << "| | _…#{all.size - MAX_TABLE_ROWS} more in the ledger_ | | | | |" if all.size > MAX_TABLE_ROWS
+body << ''
+notes = ['🌱 = the run that opened this PR (creation cost).']
+notes << '\\* = estimated from `_data/ai_pricing.yml` (that path reports tokens, not dollars).' if estimated
+notes << (oauth_only ? 'All calls ran on Claude Code subscription auth (OAuth) — $0 marginal spend; the figure is what these tokens would bill at API list prices.' \
+                     : 'Some calls used a metered API key — those dollars are real.')
+notes << 'Updated automatically after every AI job; full history at [/docs/ai-usage/](https://lifehacker.dev/docs/ai-usage/).'
+body << notes.map { |n| "_#{n}_" }.join(' ')
+body << ''
+body << "#{DATA_HEAD}#{[JSON.generate(blob)].pack('m0')} -->"
+
+payload = JSON.generate('body' => body.join("\n"))
+tmp = File.join(AIUsage.dir, 'comment-payload.json')
+File.write(tmp, payload)
+ok =
+  if existing
+    system('gh', 'api', '-X', 'PATCH', "repos/#{repo}/issues/comments/#{existing['id']}", '--input', tmp, out: File::NULL, err: %i[child out])
+  else
+    system('gh', 'api', "repos/#{repo}/issues/#{pr}/comments", '--input', tmp, out: File::NULL, err: %i[child out])
+  end
+warn(ok ? "[usage_report] PR ##{pr} cost comment #{existing ? 'updated' : 'created'} (total #{fmt_usd.call(total)})." \
+        : "[usage_report] PR ##{pr} comment update failed (non-fatal; records are in the artifact).")
+exit 0

data/scripts/ci/test_visual_evidence_autogen.py

@@ -0,0 +1,341 @@
+#!/usr/bin/env python3
+# Feature: ZER0-085
+"""Unit tests for scripts/ci/visual_evidence_autogen.py — the deterministic half
+of the visual-evidence autogen lane.
+
+What these pin, and why:
+
+* the PLAN — from a PR's changed files alone, does the lane decide to act, and
+  on what? The shape of PR #454 (UI + spec + generator + README-only evidence)
+  must yield exactly one bespoke job; a docs-only PR must yield nothing; the
+  loop guard and the budget must stop it cold;
+* the DECISION — only a well-formed `intentional` verdict on a genuinely failing
+  verify pass may bless baselines (issue #417: a blessed regression and a green
+  check are indistinguishable, so "the check is red" must never be enough);
+* the CONTRACT with the workflows — the evidence gate's UI paths, ci.yml's
+  `styling` filter coverage, the hooks update-snapshots.sh exposes, and the
+  subcommands the workflow calls all exist where this script expects them.
+
+Standard library only; no Docker, no network.
+
+    python3 scripts/ci/test_visual_evidence_autogen.py
+"""
+
+from __future__ import annotations
+
+import re
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+
+import visual_evidence_autogen as vea  # noqa: E402
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+FAILURES: list[str] = []
+PASSED = 0
+
+
+def check(label: str, ok: bool) -> None:
+    global PASSED
+    if ok:
+        PASSED += 1
+        print(f"  ✓ {label}")
+    else:
+        FAILURES.append(label)
+        print(f"  ✗ {label}")
+
+
+class FakeFS:
+    """A head tree: {path: content}; PNGs are any key ending in .png."""
+
+    def __init__(self, files: dict[str, str]):
+        self.files = files
+
+    def exists(self, path: str) -> bool:
+        return path in self.files
+
+    def pngs(self, folder: str) -> list[str]:
+        return sorted(p for p in self.files if p.startswith(folder + "/") and p.endswith(".png"))
+
+    def read(self, path: str) -> str:
+        return self.files[path]
+
+    def generators(self) -> list[str]:
+        return sorted(p for p in self.files if vea.GENERATOR_RE.match(p))
+
+
+GEN_405 = "test/visual/navbar-tiers-evidence.mjs"
+GEN_405_SRC = "import { generateEvidence } from './evidence-kit.mjs';\nawait generateEvidence({\n  slug: 'navbar-tiers-405',\n});\n"
+SHA = "e34e4065b99983b2960015e8e27e2d09fc1bf2c6"
+
+
+def plan_for(changed, files, *, head_subject="feat(navigation): tiers", subjects=(), default_slug="agent-issue-405"):
+    return vea.make_plan(changed=changed, head_sha=SHA, head_subject=head_subject,
+                         branch_subjects=list(subjects), fs=FakeFS(files),
+                         default_slug=default_slug, base_ref="origin/main", head_ref="agent/issue-405")
+
+
+def test_detect_slug() -> None:
+    print("detect_slug")
+    check("reads `slug: '…'` from a kit-based generator", vea.detect_slug(GEN_405_SRC) == "navbar-tiers-405")
+    check("reads OUT = 'test/visual/evidence/<slug>' from a hand-rolled one",
+          vea.detect_slug("const OUT = 'test/visual/evidence/navbar-fit';") == "navbar-fit")
+    check("returns None when nothing names a folder", vea.detect_slug("console.log('hi')") is None)
+
+
+def test_plan_pr_454_shape() -> None:
+    print("plan — PR #454: UI + spec + generator + README-only evidence")
+    changed = ["_sass/core/_navbar.scss", "_includes/core/header.html", "CHANGELOG.md",
+               "test/visual/features/navbar-tiers.spec.js", GEN_405,
+               "test/visual/evidence/navbar-tiers-405/README.md"]
+    files = {GEN_405: GEN_405_SRC, "test/visual/evidence/navbar-tiers-405/README.md": "# pending"}
+    plan = plan_for(changed, files)
+    check("needed", plan["needed"] is True)
+    check("exactly one job", len(plan["jobs"]) == 1)
+    job = plan["jobs"][0] if plan["jobs"] else {}
+    check("it is the bespoke generator for navbar-tiers-405",
+          job.get("kind") == "bespoke" and job.get("generator") == GEN_405 and job.get("slug") == "navbar-tiers-405")
+    check("its folder is the evidence dir", job.get("dir") == "test/visual/evidence/navbar-tiers-405")
+    check("the pixel tier is in scope (sass changed)", plan["snapshots_in_scope"] is True)
+    check("no skip reason", plan["skip_reason"] is None)
+    check("the spec is recorded", plan["specs"] == ["test/visual/features/navbar-tiers.spec.js"])
+
+
+def test_plan_loop_guard_and_budget() -> None:
+    print("plan — loop guard + budget")
+    changed = ["_sass/core/_navbar.scss", "test/visual/evidence/x/01.png"]
+    files = {"test/visual/evidence/x/01.png": ""}
+    plan = plan_for(changed, files, head_subject=f"test(visual): auto-generate evidence {vea.MARKER}")
+    check("an autogen head commit stops the lane", plan["needed"] is False and "loop guard" in plan["skip_reason"])
+    subjects = [f"x {vea.MARKER}"] * vea.MAX_COMMITS + ["feat: something"]
+    plan = plan_for(changed, files, subjects=subjects)
+    check(f"{vea.MAX_COMMITS} prior autogen commits exhaust the budget",
+          plan["needed"] is False and "budget" in plan["skip_reason"])
+    plan = plan_for(changed, files, subjects=[f"x {vea.MARKER}"] * (vea.MAX_COMMITS - 1))
+    check("one under the budget still runs", plan["needed"] is True)
+
+
+def test_plan_generic_fallback() -> None:
+    print("plan — UI change with no evidence at all → generic generator")
+    plan = plan_for(["_layouts/home.html"], {})
+    check("one generic job named after the branch", len(plan["jobs"]) == 1 and plan["jobs"][0]["kind"] == "generic"
+          and plan["jobs"][0]["slug"] == "agent-issue-405" and plan["jobs"][0]["generator"] is None)
+    plan = plan_for(["_layouts/home.html"], {}, default_slug=None)
+    check("…but only when a default slug is known", plan["jobs"] == [] and plan["needed"] is True)
+
+
+def test_plan_respects_author_evidence() -> None:
+    print("plan — author-produced evidence is never regenerated")
+    folder = "test/visual/evidence/hover-flicker"
+    changed = ["_sass/core/_navbar.scss", f"{folder}/01-box-shift.png", f"{folder}/metrics.json"]
+    files = {f"{folder}/01-box-shift.png": "", f"{folder}/metrics.json": "{}"}
+    plan = plan_for(changed, files)
+    check("proof present + no manifest → no job", plan["jobs"] == [])
+    check("…the pixel tier still gets verified", plan["needed"] is True and plan["snapshots_in_scope"] is True)
+    files[f"{folder}/{vea.MANIFEST}"] = '{"rendered_from": "0000000deadbeef"}'
+    plan = plan_for(changed + [f"{folder}/{vea.MANIFEST}"], files)
+    check("autogen evidence from an OLDER head is refreshed", len(plan["jobs"]) == 1 and "rendered from" in plan["jobs"][0]["why"])
+    files[f"{folder}/{vea.MANIFEST}"] = f'{{"rendered_from": "{SHA}"}}'
+    plan = plan_for(changed + [f"{folder}/{vea.MANIFEST}"], files)
+    check("autogen evidence from THIS head is left alone", plan["jobs"] == [])
+
+
+def test_plan_out_of_scope() -> None:
+    print("plan — nothing visual")
+    plan = plan_for(["docs/systems/x.md", "pages/_posts/2026-09-05-x.md", "_data/backlog.yml"], {})
+    check("docs/content/backlog → not needed", plan["needed"] is False and plan["jobs"] == []
+          and plan["snapshots_in_scope"] is False)
+    plan = plan_for(["_data/ui-text.yml"], {})
+    check("a chrome data file alone → pixel tier in scope, no evidence job",
+          plan["needed"] is True and plan["jobs"] == [] and plan["snapshots_in_scope"] is True)
+
+
+def test_styling_matches_ci_filter() -> None:
+    print("contract — styling scope covers every chrome-affecting file lint-workflows.yml pins")
+    for f in ["_data/navigation/main.yml", "_data/ui-text.yml", "_data/i18n/en.yml", "_data/theme_skins.yml",
+              "_includes/components/theme-info.html", "_layouts/default.html", "_sass/main.scss",
+              "test/visual/features/x.spec.js", "test/playwright.config.js", "assets/js/x.js"]:
+        check(f"is_styling({f})", vea.is_styling(f))
+    check("a backlog edit is not styling", not vea.is_styling("_data/backlog.yml"))
+
+
+def test_ui_prefixes_match_gate() -> None:
+    print("contract — UI paths agree with evidence-gate.yml")
+    gate = (REPO_ROOT / ".github/workflows/evidence-gate.yml").read_text(encoding="utf-8")
+    m = re.search(r"grep -E '\^\(([^)]+)\)'", gate)
+    gate_prefixes = tuple(m.group(1).split("|")) if m else ()
+    check("gate regex parsed", bool(gate_prefixes))
+    check("same prefix set", set(gate_prefixes) == set(vea.UI_PREFIXES))
+    check("gate requires generated proof (metrics.json)", "metrics.json" in gate)
+    check("gate points at the autogen", "visual_evidence_autogen.py" in gate)
+
+
+def test_parse_results() -> None:
+    print("parse_playwright_results")
+    results = {"suites": [{"title": "appearance-snapshot.spec.js", "suites": [{"title": "Theme skins", "suites": [
+        {"title": "skin: sunrise", "specs": [{"title": "homepage visual snapshot", "tests": [{"results": [
+            {"status": "failed", "error": {"message": "\x1b[2mexpect(\x1b[22mpage\x1b[2m).\x1b[22mtoHaveScreenshot failed\n\n  1939 pixels (ratio 0.01 of all image pixels) are different."},
+             "attachments": [
+                 {"name": "homepage-sunrise-expected", "path": "/work/test/visual-results/output/a/homepage-sunrise-expected.png"},
+                 {"name": "homepage-sunrise-actual", "path": "/work/test/visual-results/output/a/homepage-sunrise-actual.png"},
+                 {"name": "homepage-sunrise-diff", "path": "/work/test/visual-results/output/a/homepage-sunrise-diff.png"}]}]}]}]},
+        {"title": "skin: air", "specs": [{"title": "homepage visual snapshot", "tests": [{"results": [{"status": "passed"}]}]}]},
+    ]}]}]}
+    parsed = vea.parse_playwright_results(results)
+    check("2 tests, 1 passed", parsed["total"] == 2 and parsed["passed"] == 1)
+    check("one failure", len(parsed["failures"]) == 1)
+    f = parsed["failures"][0] if parsed["failures"] else {}
+    check("skin taken from the describe title", f.get("skin") == "sunrise")
+    check("pixel count parsed through the ANSI noise", f.get("diff_px") == 1939)
+    check("container paths made repo-relative",
+          f.get("diff") == "test/visual-results/output/a/homepage-sunrise-diff.png")
+
+
+def test_decide() -> None:
+    print("decide — only `intentional` on a failing verify may bless")
+    fail = {"snapshots": {"verified": "fail"}}
+    ok = {"snapshots": {"verified": "pass"}}
+    v = lambda kind, summary="looks right": {"schema": "visual-evidence-verdict/v1", "snapshots": {"verdict": kind, "summary": summary}}
+    check("fail + intentional → bless", vea.decide(fail, v("intentional"))[0] is True)
+    check("fail + regression → no", vea.decide(fail, v("regression"))[0] is False)
+    check("fail + unclear → no", vea.decide(fail, v("unclear"))[0] is False)
+    check("fail + no verdict → no (missing)", vea.decide(fail, None) == (False, "missing", vea.decide(fail, None)[2]))
+    check("fail + wrong schema → no", vea.decide(fail, {"snapshots": {"verdict": "intentional"}})[0] is False)
+    check("fail + invented verdict → no", vea.decide(fail, v("ship-it"))[0] is False)
+    check("pass + intentional → nothing to bless", vea.decide(ok, v("intentional")) [1] == "not-applicable")
+    check("error + intentional → nothing to bless", vea.decide({"snapshots": {"verified": "error"}}, v("intentional"))[0] is False)
+
+
+def test_stage_and_messages() -> None:
+    print("stage / commit message / comment")
+    state = {"plan": "", "jobs": [{"slug": "navbar-tiers-405", "kind": "bespoke", "generator": GEN_405,
+                                   "dir": "test/visual/evidence/navbar-tiers-405", "why": "w",
+                                   "files": ["test/visual/evidence/navbar-tiers-405/01-x.png"], "has_proof": True},
+                                  {"slug": "nope", "kind": "generic", "generator": None,
+                                   "dir": "test/visual/evidence/nope", "why": "w", "files": [], "has_proof": False}],
+             "snapshots": {"in_scope": True, "verified": "fail", "total": 9, "passed": 0,
+                           "failures": [{"skin": "air", "diff_px": 1939}], "montage": None},
+             "generators_status": "1"}
+    check("stage: only folders with proof, no snapshots when not blessed",
+          vea.stage_paths(state) == ["test/visual/evidence/navbar-tiers-405"])
+    state["blessed"] = {"baselines": ["test/visual/snapshots/x.png"], "evidence_home": "test/visual/evidence/navbar-tiers-405"}
+    state["decision"] = {"bless": True, "verdict": "intentional", "reason": "subtitle moved to the hero"}
+    check("stage: snapshots included once blessed", vea.SNAPSHOT_DIR in vea.stage_paths(state))
+    msg = vea.render_commit_message(state, "https://example/run/1")
+    check("commit subject carries the loop-guard marker", msg.splitlines()[0].endswith(vea.MARKER))
+    check("commit body carries the trailer", vea.TRAILER in msg and "baselines=yes" in msg)
+    plan = {"head_sha": SHA, "head_ref": "agent/issue-405", "base_ref": "origin/main"}
+    body = vea.render_comment(plan=plan, state=state, verdict=None, repo="bamr87/zer0-mistakes",
+                              sha="abc1234def", run_url="https://example/run/1", pushed=True)
+    check("comment carries the sticky marker", body.startswith(vea.COMMENT_MARKER))
+    check("comment names the blessed verdict and the reviewer duty", "Verdict: intentional" in body and "Files tab" in body)
+    check("comment lists the evidence that could not be generated", "could not be generated" in body)
+    quiet = {"plan": "", "jobs": [], "snapshots": {"in_scope": True, "verified": "pass", "failures": []}}
+    check("nothing noteworthy → empty comment",
+          vea.render_comment(plan=plan, state=quiet, verdict=None, repo="r", sha=None, run_url="", pushed=False) == "")
+
+
+def test_stage_command_output() -> None:
+    print("stage — the command's stdout is what the workflow `mapfile`s")
+    import argparse
+    import contextlib
+    import io
+    import json
+    import tempfile
+    with tempfile.TemporaryDirectory() as tmp:
+        empty = Path(tmp) / "empty.json"
+        empty.write_text(json.dumps({"jobs": [], "snapshots": {"verified": "pass"}}), encoding="utf-8")
+        buf = io.StringIO()
+        with contextlib.redirect_stdout(buf):
+            vea.cmd_stage(argparse.Namespace(state=str(empty)))
+        check("nothing to add → prints nothing, not even a newline (run 33992712400 died on `git add -- \"\"`)",
+              buf.getvalue() == "")
+        some = Path(tmp) / "some.json"
+        some.write_text(json.dumps({"jobs": [{"dir": "test/visual/evidence/x", "has_proof": True}]}), encoding="utf-8")
+        buf = io.StringIO()
+        with contextlib.redirect_stdout(buf):
+            vea.cmd_stage(argparse.Namespace(state=str(some)))
+        check("one path per line when there is something to add", buf.getvalue() == "test/visual/evidence/x\n")
+
+
+def test_lane_tooling_contract() -> None:
+    """The regression that killed the lane's first real run on a real PR.
+
+    visual-evidence-autogen.yml checks out the PR's branch, but the orchestrator
+    lives on main. On #454 — a branch cut before the lane shipped — the job died
+    in 20 seconds with "can't open file scripts/ci/visual_evidence_autogen.py",
+    on the very PR the lane was built to unstick. Every open PR on the day this
+    ships has the same shape, so the fix (restore the lane's tooling from the
+    base branch) is pinned here rather than left to review.
+    """
+    print("contract — the lane's tooling comes from the BASE branch")
+    wf = (REPO_ROOT / ".github/workflows/visual-evidence-autogen.yml").read_text(encoding="utf-8")
+
+    check("the orchestrator itself is restored (the #454 crash)",
+          "scripts/ci/visual_evidence_autogen.py" in vea.LANE_TOOLING)
+    check("update-snapshots.sh is restored (an older copy lacks the hooks and silently no-ops)",
+          "test/update-snapshots.sh" in vea.LANE_TOOLING)
+    check("both shared generators are restored",
+          "test/visual/pr-evidence.mjs" in vea.LANE_TOOLING
+          and "test/visual/snapshot-diff-montage.mjs" in vea.LANE_TOOLING)
+    check("the PR's OWN evidence spec is NOT overwritten",
+          not any(p.endswith("-evidence.mjs") and "pr-evidence" not in p for p in vea.LANE_TOOLING))
+    check("evidence-kit.mjs is left to the PR (a shared library it may extend)",
+          "test/visual/evidence-kit.mjs" not in vea.LANE_TOOLING)
+
+    # The workflow's env list and the module constant must not drift apart.
+    m = re.search(r"LANE_TOOLING: >-\n((?:[ ]{8}\S+\n)+)", wf)
+    check("the workflow declares LANE_TOOLING", bool(m))
+    if m:
+        declared = tuple(m.group(1).split())
+        check("workflow LANE_TOOLING == the module's constant (single source of truth)",
+              declared == vea.LANE_TOOLING)
+
+    check("it restores from the base ref, not the head",
+          'git checkout "origin/${BASE_REF}" -- $LANE_TOOLING' in wf)
+    check("…and unstages at once, so restored files can never be committed",
+          'git reset --quiet HEAD -- $LANE_TOOLING' in wf)
+    # Ordering: the restore must happen before anything invokes the orchestrator.
+    restore_at = wf.find('git checkout "origin/${BASE_REF}" -- $LANE_TOOLING')
+    first_use = wf.find("python3 scripts/ci/visual_evidence_autogen.py plan")
+    check("the restore runs BEFORE the first orchestrator call",
+          restore_at != -1 and first_use != -1 and restore_at < first_use)
+
+
+def test_workflow_wiring() -> None:
+    print("contract — the workflow, the hooks, the hand-offs")
+    wf = REPO_ROOT / ".github/workflows/visual-evidence-autogen.yml"
+    check("workflow exists", wf.exists())
+    text = wf.read_text(encoding="utf-8") if wf.exists() else ""
+    for sub in ("plan", "generate", "decide", "bless", "stage", "commit-message", "comment", "teardown"):
+        check(f"workflow calls `{sub}`", f"visual_evidence_autogen.py {sub}" in text)
+    check("same-repo guard", "head.repo.full_name == github.repository" in text)
+    check("kill switch", "VISUAL_EVIDENCE_AUTOGEN_ENABLED" in text)
+    check("evidence-gate opt-out labels honoured", "skip-evidence" in text and "no-visual-change" in text)
+    check("the reviewer agent is the one invoked", "agent: visual-evidence-reviewer" in text)
+    check("never `git add -A`", not re.search(r"^\s*git add -A", text, re.M) and 'git add -- "${paths[@]}"' in text)
+    check("reviewer agent file exists", (REPO_ROOT / ".claude/agents/visual-evidence-reviewer.md").exists())
+    us = (REPO_ROOT / "test/update-snapshots.sh").read_text(encoding="utf-8")
+    for hook in ("PRE_TEST_SCRIPT", "POST_TEST_SCRIPT", "SKIP_PLAYWRIGHT"):
+        check(f"update-snapshots.sh exposes {hook}", hook in us)
+    for f in (vea.GENERIC_GENERATOR, vea.DIFF_MONTAGE_SCRIPT):
+        check(f"{f} exists", (REPO_ROOT / f).exists())
+    repair = (REPO_ROOT / ".github/workflows/ci-self-repair.yml").read_text(encoding="utf-8")
+    check("ci-self-repair hands a red Visual Snapshots job to this lane", "Visual Snapshots" in repair and "visual-evidence-autogen" in repair)
+
+
+def main() -> int:
+    for t in (test_detect_slug, test_plan_pr_454_shape, test_plan_loop_guard_and_budget, test_plan_generic_fallback,
+              test_plan_respects_author_evidence, test_plan_out_of_scope, test_styling_matches_ci_filter,
+              test_ui_prefixes_match_gate, test_parse_results, test_decide, test_stage_and_messages,
+              test_stage_command_output, test_lane_tooling_contract, test_workflow_wiring):
+        t()
+    print(f"\n{PASSED} passed, {len(FAILURES)} failed")
+    for f in FAILURES:
+        print(f"  FAILED: {f}")
+    return 1 if FAILURES else 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())