browser_review_gate 0.1.0

data/lib/browser_review_gate/github.rb

@@ -0,0 +1,81 @@
+require "json"
+
+module BrowserReviewGate
+  # Everything the gate reads from or writes to a pull request, through the `gh` CLI.
+  class GitHub
+    def initialize(repository: nil, shell: Shell.new)
+      @repository = repository.to_s.empty? ? nil : repository
+      @shell = shell
+    end
+
+    def repository
+      @repository ||= JSON.parse(@shell.call("gh", "repo", "view", "--json", "nameWithOwner")).fetch("nameWithOwner")
+    end
+
+    def pull_request(number)
+      api("repos/#{repository}/pulls/#{number}")
+    end
+
+    # The PR of the current branch, or nil when the branch has none.
+    def pull_request_number_for_branch
+      JSON.parse(@shell.call("gh", "pr", "view", "--json", "number")).fetch("number")
+    rescue Error
+      nil
+    end
+
+    def comments(number)
+      pages("repos/#{repository}/issues/#{number}/comments")
+    end
+
+    def files(number)
+      pages("repos/#{repository}/pulls/#{number}/files")
+    end
+
+    def login
+      @shell.call("gh", "api", "user", "--jq", ".login").strip
+    end
+
+    def create_comment(number, body)
+      api("repos/#{repository}/issues/#{number}/comments", method: "POST", input: { body: body })
+    end
+
+    def update_comment(comment_id, body)
+      api("repos/#{repository}/issues/comments/#{comment_id}", method: "PATCH", input: { body: body })
+    end
+
+    # Adding a label to a PR creates the label in the repository when it does not exist yet.
+    def add_label(number, label)
+      api("repos/#{repository}/issues/#{number}/labels", method: "POST", input: { labels: [ label ] })
+    end
+
+    def remove_label(number, label)
+      @shell.call("gh", "api", "-X", "DELETE", "repos/#{repository}/issues/#{number}/labels/#{label}")
+    end
+
+    def remove_reviewers(number, reviewers:, team_reviewers:)
+      api("repos/#{repository}/pulls/#{number}/requested_reviewers", method: "DELETE",
+        input: { reviewers: reviewers, team_reviewers: team_reviewers })
+    end
+
+    def dispatch_workflow(workflow, number)
+      @shell.call("gh", "workflow", "run", workflow, "-f", "pr_number=#{number}")
+    end
+
+    private
+
+    def api(path, method: "GET", input: nil)
+      argv = [ "gh", "api", "-X", method, path ]
+      argv += [ "--input", "-" ] if input
+      output = @shell.call(*argv, stdin_data: input && JSON.generate(input))
+      output.strip.empty? ? {} : JSON.parse(output)
+    rescue JSON::ParserError
+      raise Error, "gh returned output that is not JSON for #{path}"
+    end
+
+    def pages(path)
+      JSON.parse(@shell.call("gh", "api", "--paginate", "--slurp", "#{path}?per_page=100")).flatten(1)
+    rescue JSON::ParserError
+      raise Error, "gh returned output that is not JSON for #{path}"
+    end
+  end
+end

data/lib/browser_review_gate/hook.rb

@@ -0,0 +1,36 @@
+require "json"
+
+module BrowserReviewGate
+  # Runs after an agent's shell command. When the command opened a PR, the browser run saved for this
+  # commit is published. Never fails the agent: every outcome is reported as a message.
+  class Hook
+    TRIGGER = "gh pr create"
+
+    def initialize(publisher_factory:, shell: Shell.new)
+      @publisher_factory = publisher_factory
+      @shell = shell
+    end
+
+    # `payload` is the raw hook input. Returns a message for the agent, or nil when nothing was done.
+    def call(payload)
+      return unless payload.to_s.include?(TRIGGER)
+
+      sha = @shell.call("git", "rev-parse", "HEAD").strip
+      return unless SavedReport.new(sha: sha, shell: @shell).exist?
+
+      verified = @publisher_factory.call.publish
+      verified ? "Saved browser verification published; human review can be requested." :
+                 "Saved browser results published, but some required cases still need to pass."
+    rescue Error => error
+      "Saved browser verification was not published: #{error.message}"
+    end
+
+    # Claude Code and Codex read a JSON object; other agents show plain text.
+    def self.render(message, agent)
+      return if message.nil?
+      return message unless %w[claude codex].include?(agent)
+
+      JSON.generate(hookSpecificOutput: { hookEventName: "PostToolUse", additionalContext: message })
+    end
+  end
+end

data/lib/browser_review_gate/installer.rb

@@ -0,0 +1,171 @@
+require "erb"
+require "fileutils"
+require "json"
+
+module BrowserReviewGate
+  # Sets a project up in one go: the CI workflow, the settings file, the agent playbook, and the
+  # entry points for every AI coding agent the project uses. Safe to run again after a gem update.
+  class Installer
+    AGENTS = %w[claude cursor codex].freeze
+    MANAGED = "browser_review_gate:managed"
+    PLAYBOOK = "docs/browser-review-gate.md"
+    HOOK = "browser-review-gate hook"
+    BLOCK = /<!-- browser_review_gate:start -->.*?<!-- browser_review_gate:end -->\n?/m
+
+    def initialize(root:, log: $stdout, agents: nil, gem_source: nil, workflow: nil, force: false, dry_run: false)
+      @root = root
+      @log = log
+      @agents = agents
+      @gem_source = gem_source
+      @workflow = workflow
+      @force = force
+      @dry_run = dry_run
+    end
+
+    def install
+      agents = @agents || detect_agents
+      unknown = agents - AGENTS
+      raise Error, "Unknown agents: #{unknown.join(", ")} (known: #{AGENTS.join(", ")})" if unknown.any?
+
+      preflight(agents)
+      create_only(Config::PATH, render("config.yml"))
+      managed(".github/workflows/#{workflow}", render("workflow.yml"))
+      managed(PLAYBOOK, render("playbook.md"))
+      agents.each { |agent| send("install_#{agent}") }
+      @log.puts "", "Agents: #{agents.join(", ")}.", next_steps
+      agents
+    end
+
+    # Which AI coding agents the project is set up for. AGENTS.md is read by most of them, so it is
+    # the fallback when nothing is found.
+    def detect_agents
+      found = []
+      found << "claude" if exist?(".claude") || exist?("CLAUDE.md")
+      found << "cursor" if exist?(".cursor") || exist?(".cursorrules")
+      found << "codex" if exist?(".codex") || exist?("AGENTS.md")
+      found.empty? ? [ "codex" ] : found
+    end
+
+    private
+
+    def config = @config ||= Config.load(@root)
+    def workflow = @workflow || config.workflow
+    def command = config.command
+    def label = config.label
+    def playbook_path = PLAYBOOK
+    def exist?(path) = File.exist?(File.join(@root, path))
+
+    def start_command
+      return "bin/dev" if exist?("bin/dev")
+
+      "npm run dev" if exist?("package.json")
+    end
+
+    def url = exist?("config/application.rb") ? "http://localhost:3000" : nil
+
+    # setup-ruby reads the project's own version file when there is one.
+    def ruby_version = exist?(".ruby-version") || exist?(".tool-versions") ? nil : "3.4"
+
+    # How CI gets the gem: from RubyGems, from a git repository ("git:URL" or "git:URL#REF"), or from a
+    # copy kept in the project itself ("path:DIR", read from the base branch like the rest of the checkout).
+    def install_command
+      return "gem install browser_review_gate -v #{VERSION} --no-document" if @gem_source.nil? || @gem_source == "rubygems"
+      return %(echo "$GITHUB_WORKSPACE/#{@gem_source.delete_prefix("path:")}/exe" >> "$GITHUB_PATH") if @gem_source.match?(%r{\Apath:[\w./-]+\z})
+
+      url, ref = @gem_source.delete_prefix("git:").split("#", 2)
+      raise Error, "--gem-source must be rubygems, git:URL[#REF] or path:DIR" unless @gem_source.start_with?("git:") && !url.to_s.empty?
+
+      %(git clone --quiet --depth 1 --branch "#{ref || "v#{VERSION}"}" "#{url}" "$RUNNER_TEMP/browser_review_gate" && ) +
+        %(echo "$RUNNER_TEMP/browser_review_gate/exe" >> "$GITHUB_PATH")
+    end
+
+    JSON_FILES = { "claude" => ".claude/settings.json", "cursor" => ".cursor/hooks.json", "codex" => ".codex/hooks.json" }.freeze
+
+    # Nothing is written when a settings file that must be merged cannot be read.
+    def preflight(agents)
+      agents.filter_map { |agent| JSON_FILES[agent] }.each do |path|
+        JSON.parse(File.read(File.join(@root, path))) if exist?(path)
+      rescue JSON::ParserError
+        raise Error, "#{path} is not valid JSON; fix it and run install again"
+      end
+    end
+
+    def install_claude
+      managed(".claude/skills/browser-pr-verification/SKILL.md", render("claude_skill.md"))
+      merge_json(".claude/settings.json") do |settings|
+        entries = ((settings["hooks"] ||= {})["PostToolUse"] ||= [])
+        entries << { "matcher" => "Bash", "hooks" => [ { "type" => "command", "command" => "#{HOOK} claude", "timeout" => 60 } ] }
+      end
+    end
+
+    def install_cursor
+      managed(".cursor/commands/browser-pr-verification.md", render("cursor_command.md"))
+      merge_json(".cursor/hooks.json") do |settings|
+        settings["version"] ||= 1
+        ((settings["hooks"] ||= {})["afterShellExecution"] ||= []) << { "command" => "#{HOOK} cursor" }
+      end
+    end
+
+    # Codex runs a project hook only after the person trusts it in /hooks, so the AGENTS.md section also
+    # tells the agent to publish after it opens the PR.
+    def install_codex
+      merge_json(".codex/hooks.json") do |settings|
+        entries = ((settings["hooks"] ||= {})["PostToolUse"] ||= [])
+        entries << { "matcher" => "^Bash$", "hooks" => [ { "type" => "command", "command" => "#{HOOK} codex", "timeout" => 60 } ] }
+      end
+      path = "AGENTS.md"
+      current = exist?(path) ? File.read(File.join(@root, path)) : nil
+      block = render("agents_block.md")
+      updated = current.to_s.match?(BLOCK) ? current.sub(BLOCK) { block } : [ current.to_s.rstrip, block ].reject(&:empty?).join("\n\n")
+      write(path, updated.end_with?("\n") ? updated : "#{updated}\n", current)
+    end
+
+    def render(name)
+      ERB.new(File.read(File.join(__dir__, "templates", "#{name}.erb")), trim_mode: "-").result(binding)
+    end
+
+    def create_only(path, content)
+      return @log.puts("  exists    #{path}") if exist?(path)
+
+      write(path, content, nil)
+    end
+
+    # A file this gem wrote before is updated; a file someone else wrote is left alone without --force.
+    def managed(path, content)
+      current = exist?(path) ? File.read(File.join(@root, path)) : nil
+      return @log.puts("  skipped   #{path} (not written by browser_review_gate; use --force to replace)") if current && !current.include?(MANAGED) && !@force
+
+      write(path, content, current)
+    end
+
+    # Adds the hook entry unless the file already runs the hook.
+    def merge_json(path)
+      current = exist?(path) ? File.read(File.join(@root, path)) : nil
+      return @log.puts("  unchanged #{path}") if current&.include?(HOOK)
+
+      settings = current ? JSON.parse(current) : {}
+      yield settings
+      write(path, "#{JSON.pretty_generate(settings)}\n", current)
+    end
+
+    def write(path, content, current)
+      return @log.puts("  unchanged #{path}") if current == content
+
+      @log.puts "  #{current ? "updated  " : "created  "} #{path}"
+      return if @dry_run
+
+      full_path = File.join(@root, path)
+      FileUtils.mkdir_p(File.dirname(full_path))
+      File.write(full_path, content)
+    end
+
+    def next_steps
+      <<~TEXT
+        Next:
+          1. Add a CLAUDE_CODE_OAUTH_TOKEN or ANTHROPIC_API_KEY secret to the repository.
+          2. Fill start_command, url and sign_in in #{Config::PATH}.
+          3. Commit and merge to the default branch: the workflow runs from there.
+      TEXT
+    end
+  end
+end

data/lib/browser_review_gate/markdown.rb

@@ -0,0 +1,8 @@
+module BrowserReviewGate
+  module Markdown
+    # Comment text comes from a model or a report file, so it must not start markup or ping anyone.
+    def self.escape(value)
+      value.to_s.gsub(/[\\`*_{}\[\]()<>#+.!|@]/) { |character| "\\#{character}" }.gsub(/[\r\n]+/, " ")
+    end
+  end
+end

data/lib/browser_review_gate/model_client.rb

@@ -0,0 +1,70 @@
+require "json"
+require "open3"
+require "tmpdir"
+
+module BrowserReviewGate
+  # One headless Claude Code call with every tool disabled and the answer bound to a JSON schema, so
+  # the model can only answer. The GitHub token is removed from its environment.
+  class ModelClient
+    NPX = %w[npx --yes @anthropic-ai/claude-code@2.1.288].freeze
+    CREDENTIALS = %w[CLAUDE_CODE_OAUTH_TOKEN ANTHROPIC_API_KEY].freeze
+
+    # `runner` takes the environment, argv and stdin, and returns [stdout, stderr, success?].
+    def initialize(command: nil, model: nil, runner: nil)
+      @command = command.to_s.empty? ? default_command : command.split
+      @model = model
+      @runner = runner || method(:run)
+    end
+
+    # Returns the model's answer as JSON text. Raises Error when the call fails or gives no answer.
+    def complete(system:, user:, schema:)
+      argv = @command + [ "--print", "--tools", "", "--setting-sources", "", "--system-prompt", system,
+                          "--json-schema", JSON.generate(schema), "--output-format", "json" ]
+      argv += [ "--model", @model ] if @model
+      stdout, stderr, success = @runner.call(environment, argv, user)
+      result = parse(stdout)
+      answer = result["structured_output"] if success && result["is_error"] == false
+      return JSON.generate(answer) if answer.is_a?(Hash)
+
+      raise Error, "The model gave no answer: #{failure(result, stdout, stderr)}"
+    end
+
+    private
+
+    def parse(stdout)
+      result = JSON.parse(stdout.to_s)
+      result.is_a?(Hash) ? result : {}
+    rescue JSON::ParserError
+      {}
+    end
+
+    # The CLI explains a failure in `result`; a missing credential is the usual cause in CI.
+    def failure(result, stdout, stderr)
+      reason = result["result"].to_s.strip
+      reason = "#{stderr} #{stdout}".strip[0, 300] if reason.empty?
+      missing = CREDENTIALS.all? { |name| ENV[name].to_s.empty? }
+      missing ? "#{reason} (neither #{CREDENTIALS.join(" nor ")} is set)" : reason
+    end
+
+    # An installed `claude` is used as is; CI has none and fetches a pinned version on demand.
+    def default_command
+      installed = ENV["PATH"].to_s.split(File::PATH_SEPARATOR).any? { |dir| File.executable?(File.join(dir, "claude")) }
+      installed ? [ "claude" ] : NPX
+    end
+
+    # Empty credentials are unset: the workflow passes both kinds and only one is configured.
+    def environment
+      unset = %w[GH_TOKEN GITHUB_TOKEN].to_h { |name| [ name, nil ] }
+      CREDENTIALS.each { |name| unset[name] = nil if ENV[name].to_s.empty? }
+      unset
+    end
+
+    # Runs outside the checkout so the CLI loads no project instructions or settings.
+    def run(environment, argv, stdin)
+      stdout, stderr, status = Open3.capture3(environment, *argv, stdin_data: stdin, chdir: Dir.tmpdir)
+      [ stdout, stderr, status.success? ]
+    rescue Errno::ENOENT
+      raise Error, "#{argv.first} is not installed"
+    end
+  end
+end

data/lib/browser_review_gate/publisher.rb

@@ -0,0 +1,136 @@
+require "json"
+
+module BrowserReviewGate
+  # Publishes locally observed browser results to the PR, or saves them until the PR exists.
+  class Publisher
+    # Without `number` the PR of the current branch is used. Without `report_path` the report saved for
+    # the current commit is used.
+    def initialize(github:, number: nil, report_path: nil, config: Config.new, shell: Shell.new)
+      @github = github
+      @number = number
+      @report_path = report_path
+      @config = config
+      @shell = shell
+    end
+
+    # Returns true when the PR is verified, false when the report was published but does not pass.
+    def publish
+      number = @number || @github.pull_request_number_for_branch
+      raise Error, "This branch has no pull request yet; save the run with --save instead" unless number
+
+      pull_request = @github.pull_request(number)
+      local_sha = head_sha
+      ensure_clean_worktree!
+      pr_sha = pull_request.fetch("head").fetch("sha")
+      raise Error, "Local HEAD #{local_sha} does not match the PR head #{pr_sha}" unless pr_sha == local_sha
+
+      saved_report = SavedReport.new(sha: local_sha, shell: @shell)
+      data = load_report(saved_report)
+      ensure_report_sha!(data, local_sha)
+      actor = @github.login
+      data["verified_by"] = "AI browser agent (run by #{actor})"
+
+      comments = @github.comments(number)
+      ensure_browser_run_wanted!(number, comments, local_sha)
+      report = merged_report(data, comments)
+
+      complete = report.passed?
+      labelled = pull_request.fetch("labels").any? { |label| label["name"] == @config.label }
+      @github.remove_label(number, @config.label) if labelled && !complete
+      write_comment(number, comments, actor, body(report, complete))
+      @github.add_label(number, @config.label) if complete
+      saved_report.delete unless @report_path
+      complete
+    rescue KeyError => error
+      raise Error, "Browser verification data is incomplete: #{error.message}"
+    rescue JSON::ParserError => error
+      raise Error, "Browser verification data is not valid JSON: #{error.message}"
+    end
+
+    # Keeps the results of a run made before the PR exists. Returns the path they were saved to.
+    def save
+      local_sha = head_sha
+      ensure_clean_worktree!
+      data = read_report_file
+      ensure_report_sha!(data, local_sha)
+      report = Report.new(data.merge("verified_by" => "pending"))
+      raise Error, report.errors.join("; ") unless report.valid?
+
+      saved_report = SavedReport.new(sha: local_sha, shell: @shell)
+      saved_report.write(data)
+      saved_report.path
+    rescue JSON::ParserError => error
+      raise Error, "Browser verification data is not valid JSON: #{error.message}"
+    end
+
+    private
+
+    def head_sha = @shell.call("git", "rev-parse", "HEAD").strip
+
+    # An untracked file may be code the browser run depended on, so it blocks publishing too.
+    def ensure_clean_worktree!
+      changes = @shell.call("git", "status", "--porcelain").lines.map(&:chomp)
+      return if changes.empty?
+
+      raise Error, "Commit and push changes before publishing browser verification, or remove these files:\n" \
+        "#{changes.join("\n")}"
+    end
+
+    def ensure_report_sha!(data, sha)
+      return if data.is_a?(Hash) && data["tested_sha"] == sha
+
+      raise Error, "Report tested_sha must match the current commit #{sha}"
+    end
+
+    def load_report(saved_report)
+      return read_report_file if @report_path
+      return saved_report.read if saved_report.exist?
+
+      raise Error, "No saved browser verification report for the current commit"
+    end
+
+    def read_report_file
+      raise Error, "A report file is required" unless @report_path
+
+      JSON.parse(File.read(@report_path))
+    rescue SystemCallError => error
+      raise Error, "Cannot read report file #{@report_path}: #{error.message}"
+    end
+
+    # A report made before the PR existed may arrive before the gate has assessed the commit.
+    def ensure_browser_run_wanted!(number, comments, tested_sha)
+      assessment = Assessment.latest(comments)
+      return unless assessment&.for?(tested_sha)
+
+      raise Error, "PR #{number} is not marked as requiring browser testing" unless assessment.required?
+    end
+
+    def merged_report(data, comments)
+      submitted = Report.new(data)
+      raise Error, submitted.errors.join("; ") unless submitted.valid?
+
+      prior = Report.latest_comment(comments)
+      report = Report.merge(prior ? Report.data_from(prior["body"]) : {}, submitted.to_h)
+      raise Error, report.errors.join("; ") unless report.valid?
+
+      report
+    end
+
+    def body(report, complete)
+      return report.markdown if complete
+
+      [ "**Not verified.** Complete every required browser scenario and pass all reported cases before human review.",
+        "", report.markdown ].join("\n")
+    end
+
+    # Only the author's own earlier report comment can be edited in place.
+    def write_comment(number, comments, actor, body)
+      prior = Report.latest_comment(comments)
+      if prior && prior.dig("user", "login") == actor
+        @github.update_comment(prior.fetch("id"), body)
+      else
+        @github.create_comment(number, body)
+      end
+    end
+  end
+end

data/lib/browser_review_gate/report.rb

@@ -0,0 +1,154 @@
+require "json"
+
+module BrowserReviewGate
+  # The browser cases an agent actually ran, with their outcomes. Published as one PR comment.
+  class Report
+    MARKER = /<!--\s*browser-verification-report:([A-Za-z0-9_-]+)\s*-->/
+    SHA_PATTERN = /\A\h{7,40}\z/
+    CASE_ID_PATTERN = /\A[A-Za-z0-9][A-Za-z0-9_.-]{0,63}\z/
+    RESULTS = %w[pass fail].freeze
+
+    attr_reader :errors
+
+    def self.latest_comment(comments)
+      comments.reverse.find { |comment| comment["body"].to_s.match?(MARKER) }
+    end
+
+    # The report data embedded in a comment body; raises Error when the marker is damaged.
+    def self.data_from(body)
+      JSON.parse(Marker.decode(body.to_s.match(MARKER)[1]))
+    rescue ArgumentError, JSON::ParserError
+      raise Error, "Existing browser verification report has a malformed data marker"
+    end
+
+    # The newest passing report on the PR, or nil.
+    def self.latest_passing(comments)
+      comment = latest_comment(comments)
+      report = comment && new(data_from(comment["body"]))
+      report if report&.passed?
+    rescue Error
+      nil
+    end
+
+    # Later results replace earlier ones case by case; cases that were not run again are kept.
+    def self.merge(previous_data, latest_data)
+      previous = new(previous_data)
+      latest = new(latest_data)
+      return latest unless previous.valid? && latest.valid?
+
+      combined = previous.cases.to_h { |test_case| [ test_case["id"], test_case ] }
+      latest.cases.each { |test_case| combined[test_case["id"]] = test_case }
+      new(latest.to_h.merge("cases" => combined.values))
+    end
+
+    def initialize(data)
+      @errors = []
+      @data = normalize(data.is_a?(Hash) ? data : {})
+      attach_defaults_to_cases
+      validate
+    end
+
+    def valid? = errors.empty?
+    def tested_sha = @data["tested_sha"]
+    def verified_by = @data["verified_by"]
+    def cases = @data["cases"].is_a?(Array) ? @data["cases"] : []
+    def excluded = @data["excluded"].is_a?(Array) ? @data["excluded"] : []
+    def to_h = JSON.parse(JSON.generate(@data))
+
+    def passed?
+      valid? && @data["coverage_complete"] && cases.all? { |test_case| test_case["result"] == "pass" }
+    end
+
+    def markdown
+      return "Browser verification report is invalid: #{errors.map { |error| Markdown.escape(error) }.join("; ")}" unless valid?
+
+      coverage = @data["coverage_complete"] ? "Complete scenario run." : "Scenario run is incomplete."
+      lines = [ "## Browser verification", "",
+        "Verified by **#{Markdown.escape(verified_by)}** for commit `#{tested_sha}`.", coverage, "" ]
+      lines.concat(cases.map { |test_case| markdown_case(test_case) })
+      lines.concat([ "", "Not exercised in the browser, by the author's decision:" ] + excluded.map { |entry| markdown_excluded(entry) }) if excluded.any?
+      payload = Marker.encode(JSON.generate(to_h))
+      lines.concat([ "", "<!-- browser-verification-report:#{payload} -->" ]).join("\n")
+    end
+
+    private
+
+    def normalize(value)
+      case value
+      when Hash then value.to_h { |key, nested| [ key.to_s, normalize(nested) ] }
+      when Array then value.map { |entry| normalize(entry) }
+      else value
+      end
+    end
+
+    def attach_defaults_to_cases
+      cases.each do |test_case|
+        next unless test_case.is_a?(Hash)
+
+        test_case["tested_sha"] ||= @data["tested_sha"]
+        test_case["verified_by"] ||= @data["verified_by"]
+      end
+    end
+
+    def validate
+      errors << "tested_sha must be a 7–40 character hexadecimal commit SHA" unless sha?(@data["tested_sha"])
+      errors << "verified_by must be a non-empty string of at most 100 characters" unless text?(@data["verified_by"], 100)
+      errors << "coverage_complete must be true or false" unless [ true, false ].include?(@data["coverage_complete"])
+      validate_cases
+      validate_excluded
+    end
+
+    def validate_cases
+      return errors << "cases must be a non-empty array" unless @data["cases"].is_a?(Array) && cases.any?
+
+      seen_ids = []
+      cases.each_with_index do |test_case, index|
+        next errors << "case #{index + 1} must be an object" unless test_case.is_a?(Hash)
+
+        validate_case(test_case, index + 1, seen_ids)
+      end
+    end
+
+    def validate_case(test_case, position, seen_ids)
+      id = test_case["id"]
+      if !(id.is_a?(String) && CASE_ID_PATTERN.match?(id))
+        errors << "case #{position} id must use letters, numbers, dots, underscores, or hyphens"
+      elsif seen_ids.include?(id)
+        errors << "case id #{id.inspect} appears more than once"
+      else
+        seen_ids << id
+      end
+      errors << "case #{position} name must be a non-empty string of at most 200 characters" unless text?(test_case["name"], 200)
+      errors << "case #{position} result must be pass or fail" unless RESULTS.include?(test_case["result"])
+      errors << "case #{position} tested_sha must be a 7–40 character hexadecimal commit SHA" unless sha?(test_case["tested_sha"])
+      errors << "case #{position} verified_by must be a non-empty string of at most 100 characters" unless text?(test_case["verified_by"], 100)
+      details = test_case["details"]
+      errors << "case #{position} details must be a string of at most 500 characters" unless details.nil? || (details.is_a?(String) && details.length <= 500)
+    end
+
+    # Scenarios the author decided not to run locally, e.g. a sign-in through an external provider.
+    def validate_excluded
+      return if @data["excluded"].nil?
+      return errors << "excluded must be an array" unless @data["excluded"].is_a?(Array)
+
+      excluded.each_with_index do |entry, index|
+        next if entry.is_a?(Hash) && text?(entry["name"], 200) && text?(entry["reason"], 500)
+
+        errors << "excluded #{index + 1} needs a name and a reason"
+      end
+    end
+
+    def sha?(value) = value.is_a?(String) && SHA_PATTERN.match?(value)
+    def text?(value, limit) = value.is_a?(String) && !value.strip.empty? && value.length <= limit
+
+    def markdown_case(test_case)
+      detail = test_case["details"].to_s.empty? ? "" : ": #{Markdown.escape(test_case["details"])}"
+      "- #{test_case["result"].upcase} — #{Markdown.escape(test_case["name"])} (`#{test_case["id"]}`, tested " \
+        "`#{test_case["tested_sha"]}` by #{Markdown.escape(test_case["verified_by"])})#{detail}"
+    end
+
+    def markdown_excluded(entry)
+      "- #{Markdown.escape(entry["name"])}: #{Markdown.escape(entry["reason"])}"
+    end
+  end
+end