browser_review_gate 0.2.2 → 0.4.0

data/lib/browser_review_gate/doctor.rb

@@ -0,0 +1,252 @@
+require "json"
+require "stringio"
+
+module BrowserReviewGate
+  # Checks a project's setup and says what to fix. A check that cannot be verified from here is SKIP,
+  # never PASS. Nothing secret is printed: secrets are checked by name only.
+  class Doctor
+    Check = Struct.new(:name, :status, :message, :fix) do
+      def to_h = { "name" => name, "status" => status, "message" => message, "fix" => fix }.compact
+    end
+
+    SECRETS = %w[CLAUDE_CODE_OAUTH_TOKEN ANTHROPIC_API_KEY OPENAI_API_KEY GEMINI_API_KEY].freeze
+    BROWSER_TOOLS = { "playwright" => "Playwright", "chrome-devtools" => "Chrome DevTools MCP", "browsermcp" => "Browser MCP",
+                      "puppeteer" => "Puppeteer" }.freeze
+    AGENT_FILES = {
+      "claude" => [ ".claude/skills/browser-pr-verification/SKILL.md", ".claude/settings.json" ],
+      "cursor" => [ ".cursor/commands/browser-pr-verification.md", ".cursor/hooks.json" ],
+      "codex" => [ "AGENTS.md", ".codex/hooks.json" ]
+    }.freeze
+
+    # `probe` takes a URL and says whether something answers there.
+    # `header_probe` takes a URL and a header name and returns the header's value there, or nil.
+    def initialize(root:, shell: Shell.new, github: nil, probe: nil, config_loader: nil, header_probe: nil)
+      @header_probe = header_probe || Status.method(:header_at)
+      @root = root
+      @shell = shell
+      @github = github
+      @probe = probe || Status.method(:answers?)
+      @config_loader = config_loader || -> { Config.load(root) }
+    end
+
+    def checks
+      @checks = []
+      ruby
+      gem_executable
+      repository
+      config = configuration
+      gh_ready = github_cli
+      repository_access if gh_ready
+      environments(config) if config
+      agents(config)
+      browser_tools(config)
+      workflow(config) if config
+      required_check(config) if config && gh_ready
+      provider_secrets(config) if config && gh_ready
+      @checks
+    end
+
+    def self.text(checks)
+      width = checks.map { |check| check.name.length }.max.to_i
+      lines = checks.map do |check|
+        line = "#{check.status.ljust(4)}  #{check.name.ljust(width)}  #{check.message}"
+        check.fix ? "#{line}\n#{" " * (width + 8)}fix: #{check.fix}" : line
+      end
+      counts = %w[PASS WARN FAIL SKIP].map { |status| "#{checks.count { |check| check.status == status }} #{status}" }
+      (lines + [ "", counts.join(", ") ]).join("\n")
+    end
+
+    def self.json(checks) = JSON.pretty_generate("checks" => checks.map(&:to_h), "ok" => checks.none? { |check| check.status == "FAIL" })
+
+    private
+
+    def add(name, status, message, fix = nil) = @checks << Check.new(name, status, message, fix)
+    def exist?(path) = File.exist?(File.join(@root, path))
+    def read(path) = exist?(path) ? File.read(File.join(@root, path)) : ""
+
+    def run(*argv)
+      @shell.call(*argv)
+    rescue Error
+      nil
+    end
+
+    def ruby
+      return add("ruby", "PASS", "Ruby #{RUBY_VERSION}") if Gem::Version.new(RUBY_VERSION) >= Gem::Version.new("3.2")
+
+      add("ruby", "FAIL", "Ruby #{RUBY_VERSION} is older than 3.2", "Use Ruby 3.2 or newer")
+    end
+
+    def gem_executable
+      found = ENV["PATH"].to_s.split(File::PATH_SEPARATOR).any? { |dir| File.executable?(File.join(dir, "browser-review-gate")) }
+      return add("gem", "PASS", "browser_review_gate #{VERSION} runs; a browser-review-gate executable is on PATH") if found
+
+      add("gem", "WARN", "browser_review_gate #{VERSION} runs, but browser-review-gate is not on PATH",
+        "bundle add browser_review_gate --group development, or run it with bundle exec")
+    end
+
+    def repository
+      top = run("git", "rev-parse", "--show-toplevel")
+      return add("repository", "PASS", "git repository at #{top.strip}") if top
+
+      add("repository", "FAIL", "not inside a git repository", "Run from the project checkout")
+    end
+
+    def configuration
+      config = @config_loader.call
+      add("settings", exist?(Config::PATH) ? "PASS" : "WARN", exist?(Config::PATH) ? "#{Config::PATH} is valid" : "#{Config::PATH} is missing; defaults apply",
+        exist?(Config::PATH) ? nil : "browser-review-gate install")
+      config
+    rescue Error => error
+      add("settings", "FAIL", error.message, "Fix #{Config::PATH}")
+      nil
+    end
+
+    def github_cli
+      unless run("gh", "--version")
+        add("gh", "FAIL", "the GitHub CLI is not installed", "Install gh: https://cli.github.com")
+        return false
+      end
+      unless run("gh", "auth", "status")
+        add("gh", "FAIL", "the GitHub CLI is not signed in", "gh auth login")
+        return false
+      end
+
+      add("gh", "PASS", "installed and signed in")
+      true
+    end
+
+    def repository_access
+      github = @github || GitHub.new(shell: @shell)
+      name = github.repository
+      push = run("gh", "api", "repos/#{name}", "--jq", ".permissions.push").to_s.strip
+      return add("repository access", "PASS", "#{name}: you can push, so your reports count") if push == "true"
+      return add("repository access", "WARN", "#{name}: you cannot push; reports you publish are ignored", "Ask for write access") if push == "false"
+
+      add("repository access", "SKIP", "#{name}: permissions could not be read")
+    rescue Error => error
+      add("repository access", "FAIL", error.message.lines.first.to_s.strip, "Check that the repository has a GitHub remote")
+    end
+
+    def environments(config)
+      return add("environments", "WARN", "no environment is configured", "Add environments.local.url in #{Config::PATH}") if config.environments.empty?
+
+      config.environments.each do |name, settings|
+        next add("environment #{name}", "WARN", "no url", "Set environments.#{name}.url") unless settings["url"]
+
+        url = Status.healthcheck_url(settings.merge("name" => name))
+        if @probe.call(url)
+          add("environment #{name}", "PASS", "answers at #{url}")
+          revision_header(config, name, settings["url"])
+        else
+          add("environment #{name}", "WARN", "does not answer at #{url}",
+            settings["start_command"] ? "Start it with #{settings["start_command"]} before a run" : "Make it reachable from where the agent runs")
+        end
+      end
+    end
+
+    # With trace checks on, a run can only count when the app names the commit it serves.
+    def revision_header(config, name, url)
+      mode = config.provenance["trace"]
+      return if mode == "off"
+
+      header = config.provenance["revision_header"]
+      served = @header_probe.call(url, header).to_s.strip.downcase
+      head = run("git", "-C", @root, "rev-parse", "HEAD").to_s.strip
+      label = "revision header #{name}"
+      fix = "Add `config.middleware.use BrowserReviewGate::RevisionHeader` to config/environments/development.rb"
+      return add(label, mode == "require" ? "FAIL" : "WARN", "#{url} sends no #{header} header; traces cannot be checked", fix) if served.empty?
+      return add(label, "PASS", "#{header} matches the checked-out commit") if !head.empty? && head.start_with?(served)
+
+      add(label, "WARN", "#{header} is #{served[0, 12]}, the checkout is at #{head[0, 12]}",
+        "Serve the checked-out code (restart the server, or run it from this worktree)")
+    end
+
+    def agents(config)
+      installer = Installer.new(root: @root, log: StringIO.new)
+      installer.detect_agents.each do |agent|
+        entry, hooks = AGENT_FILES.fetch(agent)
+        wired = agent == "codex" ? read(entry).include?("browser_review_gate:start") : exist?(entry)
+        hooked = read(hooks).include?("browser-review-gate hook #{agent}")
+        if wired && hooked
+          add("agent #{agent}", "PASS", "instructions and post-command hook installed")
+        else
+          add("agent #{agent}", "WARN", [ wired ? nil : "#{entry} missing", hooked ? nil : "hook missing in #{hooks}" ].compact.join("; "),
+            "browser-review-gate install --agents #{agent}")
+        end
+      end
+      add("playbook", exist?(Installer::PLAYBOOK) ? "PASS" : "WARN", exist?(Installer::PLAYBOOK) ? Installer::PLAYBOOK : "#{Installer::PLAYBOOK} missing",
+        exist?(Installer::PLAYBOOK) ? nil : "browser-review-gate install")
+      rules = Prompts.new(@root).rules
+      add("project rules", rules ? "PASS" : "WARN", rules ? "#{Prompts::RULES} has rules" : "#{Prompts::RULES} has no rules yet",
+        rules ? nil : "Describe what this application always checks")
+      config
+    end
+
+    # Only what the project's own files show. A tool configured elsewhere cannot be seen from here.
+    def browser_tools(config)
+      files = [ ".mcp.json", ".cursor/mcp.json", ".codex/config.toml", ".claude/settings.json", "package.json" ]
+      text = files.map { |path| read(path) }.join("\n").downcase
+      found = BROWSER_TOOLS.select { |key, _label| text.include?(key) }.values
+      if found.any?
+        add("browser tool", "PASS", "configured in the project: #{found.join(", ")}")
+      else
+        add("browser tool", "SKIP", "none found in the project's files; one may be configured per user",
+          "Give the agent a browser tool, e.g. Playwright MCP or Chrome DevTools MCP")
+      end
+      trace_tool(config, text) if config
+    end
+
+    # With trace checks on, every case needs a Playwright trace or a HAR file. Only Playwright is known
+    # to write both; other tools may write HAR, but that cannot be told from here.
+    def trace_tool(config, text)
+      mode = config.provenance["trace"]
+      return if mode == "off"
+      return add("trace recording", "PASS", "Playwright is configured in the project; it records traces and HAR files") if text.include?("playwright")
+
+      add("trace recording", mode == "require" ? "WARN" : "SKIP",
+        "provenance.trace is #{mode}, and no Playwright is configured in the project's files. Each case needs a Playwright " \
+        "trace (.zip) or a HAR file (.har); a browser tool that cannot save either #{mode == "require" ? "gets every passing case refused" : "gets no check"}",
+        "Give the agent Playwright (e.g. Playwright MCP), or a tool that saves HAR files, or set provenance.trace to warn or off")
+    end
+
+    def workflow(config)
+      path = ".github/workflows/#{config.workflow}"
+      return add("workflow", "FAIL", "#{path} is missing", "browser-review-gate install") unless exist?(path)
+
+      version = read(path)[/browser_review_gate:managed (\S+)/, 1]
+      return add("workflow", "PASS", "#{path} (#{version})") if version == VERSION
+      return add("workflow", "SKIP", "#{path} was not written by the installer; cannot tell its version") unless version
+
+      add("workflow", "WARN", "#{path} was written by #{version}; this is #{VERSION}", "browser-review-gate install")
+    end
+
+    def required_check(config)
+      github = @github || GitHub.new(shell: @shell)
+      branch = run("gh", "api", "repos/#{github.repository}", "--jq", ".default_branch").to_s.strip
+      return add("required check", "SKIP", "the default branch could not be read") if branch.empty?
+
+      contexts = run("gh", "api", "repos/#{github.repository}/branches/#{branch}/protection/required_status_checks", "--jq", ".contexts[]")
+      return add("required check", "SKIP", "branch protection of #{branch} could not be read (needs admin access, or none is set)",
+        "To block merging, make #{config.status_context} a required status check") unless contexts
+      return add("required check", "PASS", "#{config.status_context} is required on #{branch}") if contexts.lines.map(&:strip).include?(config.status_context)
+
+      add("required check", "WARN", "#{config.status_context} is not required on #{branch}; the gate does not block merging",
+        "Make #{config.status_context} a required status check")
+    rescue Error
+      add("required check", "SKIP", "could not be checked")
+    end
+
+    def provider_secrets(config)
+      names = run("gh", "secret", "list", "--json", "name", "--jq", ".[].name")
+      return add("model credentials", "SKIP", "repository secrets could not be listed (needs admin access)") unless names
+
+      present = names.lines.map(&:strip) & SECRETS
+      return add("model credentials", "FAIL", "no model secret in the repository", "Add one of #{SECRETS.join(", ")}") if present.empty?
+      if config.provider.to_s.match?(/\A(openai|gemini)\z/) && config.model.nil?
+        return add("model credentials", "FAIL", "#{config.provider} needs `model`", "Set model in #{Config::PATH}")
+      end
+
+      add("model credentials", "PASS", "secret present: #{present.join(", ")}")
+    end
+  end
+end

data/lib/browser_review_gate/evidence.rb

@@ -0,0 +1,183 @@
+require "digest"
+require "fileutils"
+require "uri"
+
+module BrowserReviewGate
+  # Turns the evidence an agent lists in a submitted report into what is published.
+  #
+  # Files (screenshots, traces) are read from the directory of the report file, checked, hashed and
+  # copied into the git directory; the PR gets their names, sizes and SHA-256 digests, never the
+  # files. A digest shows that a file of that type existed when the run was published. It does not
+  # show that a browser produced it. Console and network entries are the agent's own account; they are
+  # trimmed and scrubbed of anything that looks like a credential.
+  class Evidence
+    MAX_FILE_BYTES = 20 * 1024 * 1024
+    MAX_FILES = 10
+    MAX_ENTRIES = 50
+    MAX_ENTRY = 500
+    KINDS = { "screenshots" => "screenshot", "trace" => "trace" }.freeze
+    # Accepted files by extension, each with the bytes its content must start with.
+    TYPES = {
+      ".png" => [ "image/png", [ "\x89PNG\r\n\x1A\n".b ] ],
+      ".jpg" => [ "image/jpeg", [ "\xFF\xD8\xFF".b ] ],
+      ".jpeg" => [ "image/jpeg", [ "\xFF\xD8\xFF".b ] ],
+      ".webp" => [ "image/webp", [ "RIFF".b ] ],
+      ".zip" => [ "application/zip", [ "PK\x03\x04".b ] ],
+      # A HAR network log: JSON, so it starts with "{" after optional whitespace or a byte order mark.
+      ".har" => [ "application/json", [ "{".b ] ]
+    }.freeze
+    TRACE_TYPES = %w[.zip .har].freeze
+    SECRET = /
+      (?<key>authorization|proxy-authorization|cookie|set-cookie|x-api-key|api[_-]?key|access[_-]?token|refresh[_-]?token|
+        id[_-]?token|token|secret|password|passwd|session(?:[_-]?id)?|csrf[_-]?token|auth)
+      (?<sep>["']?\s*[:=]\s*["']?)(?:bearer\s+|basic\s+)?[^\s"',;&]+
+    /xi
+    BEARER = /\b(bearer|basic)\s+[A-Za-z0-9._~+\/=-]{8,}/i
+    JWT = /\beyJ[A-Za-z0-9_-]{5,}\.[A-Za-z0-9_-]{5,}\.[A-Za-z0-9_-]{5,}/
+    LONG_TOKEN = /\b[A-Za-z0-9_-]{40,}\b/
+
+    # Masks values that look like credentials. Applied to everything an agent writes into a report.
+    def self.scrub(text)
+      text.to_s.gsub(SECRET) { "#{$~[:key]}#{$~[:sep]}[REDACTED]" }.gsub(BEARER) { "#{$1} [REDACTED]" }
+        .gsub(JWT, "[REDACTED]").gsub(LONG_TOKEN, "[REDACTED]")
+    end
+
+    # Origin and path only: query strings and fragments carry tokens too often to publish.
+    def self.safe_url(value)
+      uri = URI.parse(value.to_s)
+      return scrub(value.to_s.split(/[?#]/).first.to_s)[0, 300] unless uri.scheme
+
+      port = uri.port == uri.default_port ? "" : ":#{uri.port}"
+      scrub("#{uri.scheme}://#{uri.host}#{port}#{uri.path}")[0, 300]
+    rescue URI::InvalidURIError
+      "[unparseable URL]"
+    end
+
+    attr_reader :files
+
+    # `base_dir` is where the report file lives; every evidence path is relative to it.
+    def initialize(base_dir:, config: Config.new)
+      @base_dir = base_dir && File.realpath(base_dir)
+      @config = config
+      @files = []
+    rescue SystemCallError
+      raise Error, "The report directory #{base_dir} cannot be read"
+    end
+
+    # Returns the report data with each case's evidence in its published form. Raises Error naming the
+    # case and the problem when a path, file or entry is not acceptable.
+    def process(data)
+      return data unless data.is_a?(Hash) && data["cases"].is_a?(Array)
+
+      @run = data
+      data.merge("cases" => data["cases"].each_with_index.map { |test_case, index| process_case(test_case, index + 1) })
+    end
+
+    # Copies the accepted files under `dir`, by digest, so the run can be looked at later.
+    def keep(dir)
+      files.each do |file|
+        target = File.join(dir, "#{file["sha256"]}#{File.extname(file["name"])}")
+        FileUtils.mkdir_p(dir)
+        FileUtils.cp(file.fetch("source"), target) unless File.exist?(target)
+      end
+    end
+
+    private
+
+    def process_case(test_case, position)
+      return test_case unless test_case.is_a?(Hash) && test_case.key?("evidence")
+
+      evidence = test_case["evidence"]
+      raise Error, "case #{position} evidence must be an object" unless evidence.is_a?(Hash)
+
+      unknown = evidence.keys - %w[screenshots trace console_errors network_errors links]
+      raise Error, "case #{position} evidence has unknown keys: #{unknown.join(", ")}" if unknown.any?
+
+      published = {}
+      listed = Array(evidence["screenshots"]).map { |path| [ "screenshots", path ] }
+      listed << [ "trace", evidence["trace"] ] if evidence["trace"]
+      raise Error, "case #{position} evidence lists more than #{MAX_FILES} files" if listed.size > MAX_FILES
+
+      published["files"] = listed.map { |kind, path| file(kind, path, position) } if listed.any?
+      trace = @files.reverse.find { |file| file["kind"] == "trace" && published.fetch("files", []).any? { |entry| entry["sha256"] == file["sha256"] } }
+      published["trace_check"] = check_trace(trace["source"], test_case) if trace && @config.provenance["trace"] != "off"
+      published["console_errors"] = entries(evidence["console_errors"], position, "console_errors") { |entry| Evidence.scrub(entry) } if evidence.key?("console_errors")
+      published["network_errors"] = entries(evidence["network_errors"], position, "network_errors") { |entry| network(entry) } if evidence.key?("network_errors")
+      published["links"] = links(evidence["links"], position) if evidence.key?("links")
+      test_case.merge("evidence" => published)
+    end
+
+    def file(kind, path, position)
+      raise Error, "case #{position} evidence files need a report file to be relative to" unless @base_dir
+      raise Error, "case #{position} #{kind} path must be a relative path inside the report directory" unless relative?(path)
+
+      full = File.join(@base_dir, path)
+      raise Error, "case #{position} #{kind} file #{path} does not exist" unless File.exist?(full)
+      raise Error, "case #{position} #{kind} file #{path} must be a regular file, not a link" if File.symlink?(full) || !File.file?(full)
+
+      real = File.realpath(full)
+      raise Error, "case #{position} #{kind} file #{path} is outside the report directory" unless real.start_with?("#{@base_dir}/")
+
+      extension = File.extname(real).downcase
+      allowed = kind == "trace" ? TRACE_TYPES : TYPES.keys - TRACE_TYPES
+      raise Error, "case #{position} #{kind} file #{path} must be one of #{allowed.join(", ")}" unless allowed.include?(extension)
+
+      bytes = File.size(real)
+      raise Error, "case #{position} #{kind} file #{path} is larger than #{MAX_FILE_BYTES / 1024 / 1024} MB" if bytes > MAX_FILE_BYTES
+
+      media_type, signatures = TYPES.fetch(extension)
+      head = File.binread(real, 64).to_s.delete_prefix("\xEF\xBB\xBF".b).lstrip
+      raise Error, "case #{position} #{kind} file #{path} is not a #{media_type} file" unless signatures.any? { |signature| head.start_with?(signature) }
+
+      entry = { "kind" => KINDS.fetch(kind), "name" => File.basename(real)[0, 100], "media_type" => media_type,
+                "bytes" => bytes, "sha256" => Digest::SHA256.file(real).hexdigest }
+      @files << entry.merge("source" => real)
+      entry
+    end
+
+    # A trace that cannot be read is a failed check, not an error: the PR shows why.
+    def check_trace(path, test_case)
+      TraceCheck.new(@config).call(path, tested_sha: @run["tested_sha"].to_s, url: test_case["url"],
+        started_at: @run["started_at"], finished_at: @run["finished_at"])
+    rescue Error => error
+      { "status" => "failed", "requests" => 0, "findings" => [ error.message[0, 300] ] }
+    end
+
+    def relative?(path)
+      path.is_a?(String) && !path.empty? && path.length <= 300 && !path.start_with?("/", "~") &&
+        !path.include?("\0") && !path.split(%r{[/\\]}).include?("..")
+    end
+
+    def entries(value, position, key)
+      raise Error, "case #{position} evidence #{key} must be a list" unless value.is_a?(Array)
+      raise Error, "case #{position} evidence #{key} has more than #{MAX_ENTRIES} entries" if value.size > MAX_ENTRIES
+
+      value.map { |entry| yield(entry).to_s[0, MAX_ENTRY] }
+    end
+
+    # A failed request as "METHOD URL -> STATUS", with the URL reduced to origin and path.
+    def network(entry)
+      return Evidence.safe_url(entry) unless entry.is_a?(Hash)
+
+      method = entry["method"].to_s.upcase[/\A[A-Z]{1,10}\z/]
+      status = entry["status"].to_s[/\A\d{1,3}|\A[a-z_:]{1,40}\z/i]
+      [ method, Evidence.safe_url(entry["url"]), status && "-> #{status}" ].compact.join(" ")
+    end
+
+    # Links to evidence kept elsewhere, accepted only on hosts the project listed.
+    def links(value, position)
+      raise Error, "case #{position} evidence links must be a list" unless value.is_a?(Array)
+      raise Error, "case #{position} evidence has more than #{MAX_FILES} links" if value.size > MAX_FILES
+
+      value.map do |link|
+        uri = URI.parse(link.to_s)
+        allowed = uri.scheme == "https" && uri.userinfo.nil? && @config.evidence["link_hosts"].include?(uri.host)
+        raise Error, "case #{position} evidence link #{link.to_s[0, 100]} must be https on a host listed in evidence.link_hosts" unless allowed
+
+        link.to_s[0, 300]
+      rescue URI::InvalidURIError
+        raise Error, "case #{position} evidence link is not a valid URL"
+      end
+    end
+  end
+end

data/lib/browser_review_gate/gate.rb

@@ -39,7 +39,7 @@ module BrowserReviewGate
       return [ :open, "Browser verification waived by #{waived_by}." ] if waived_by
       return [ :closed, "The browser-testing assessment for this commit is not available." ] unless assessment
       return [ :open, "Browser verification is not needed." ] unless assessment.required?
-      return [ :open, "Required browser cases are verified." ] if verified?(pull_request, comments, assessment)
+      return [ :open, "Required browser cases are verified." ] if verified?(number, pull_request, comments, assessment)
 
       [ :closed, "Browser run needed: #{@config.command} #{number}" ]
     end
@@ -48,8 +48,11 @@ module BrowserReviewGate
       pull_request.fetch("labels").any? { |entry| entry["name"] == label }
     end
 
-    def verified?(pull_request, comments, assessment)
-      labelled?(pull_request, @config.label) && assessment.covered_by?(Report.latest(comments))
+    # Only cases tested on a commit of this PR count: a report about other code proves nothing here.
+    def verified?(number, pull_request, comments, assessment)
+      labelled?(pull_request, @config.label) &&
+        assessment.covered_by?(Report.latest(comments, config: @config), commits: @github.pull_request_commits(number),
+          reuse_passing: @config.scenarios["reuse_passing"])
     end
 
     # A waiver counts only from someone other than the author, unless the project allows otherwise.
@@ -146,8 +149,8 @@ module BrowserReviewGate
     def notice(number, assessment, comments)
       return "Review request paused. The browser-testing assessment for this commit is not available. Request review again to retry it." unless assessment
 
-      report = Report.latest_passing(comments)
-      missing = report && !assessment.covered_by?(report) ? "the published browser run does not cover the latest commits" :
+      report = Report.latest(comments, config: @config)
+      missing = report&.passed? && !assessment.covered_by?(report, commits: @github.pull_request_commits(number)) ? "the published browser run does not cover the latest commits" :
                            "no successful local browser run has been published yet"
       "Review request paused. Browser testing is required for this change, and #{missing}. " \
         "Run `#{@config.command} #{number}` in your AI coding agent from the PR branch."

data/lib/browser_review_gate/github.rb

@@ -51,6 +51,13 @@ module BrowserReviewGate
       raise Error, "gh returned output that is not a pull request list"
     end
 
+    # Full SHAs of the PR's commits, oldest first. GitHub lists at most 250.
+    def pull_request_commits(number)
+      pages("repos/#{repository}/pulls/#{number}/commits").map { |commit| commit.fetch("sha") }
+    rescue KeyError
+      raise Error, "gh returned commits without a sha for PR #{number}"
+    end
+
     def files(number)
       pages("repos/#{repository}/pulls/#{number}/files")
     end

data/lib/browser_review_gate/installer.rb

@@ -193,9 +193,10 @@ module BrowserReviewGate
       <<~TEXT
         Next:
           1. Add one secret to the repository: CLAUDE_CODE_OAUTH_TOKEN, ANTHROPIC_API_KEY, OPENAI_API_KEY or GEMINI_API_KEY.
-          2. Fill start_command, url and sign_in in #{Config::PATH}.
+          2. Describe your environments (url, healthcheck, start_command, sign_in) in #{Config::PATH}.
           3. Describe what this application always checks in #{Prompts::RULES}.
           4. Commit and merge to the default branch: the workflow runs from there.
+          5. Run browser-review-gate doctor to check the setup.
       TEXT
     end
   end

data/lib/browser_review_gate/model_api.rb

@@ -24,6 +24,9 @@ module BrowserReviewGate
       raise Error, "Set `model` in #{Config::PATH} to assess with #{provider}" unless @model
     end
 
+    # Token counts of the last call as the provider reported them, or nil. Providers report no cost.
+    attr_reader :last_usage
+
     # Returns the model's answer as JSON text. Raises Error when the call fails or gives no answer.
     def complete(system:, user:, schema:)
       key = ENV[@key_name].to_s
@@ -32,6 +35,7 @@ module BrowserReviewGate
       url, headers, body = send("#{@provider}_request", key, system, user, schema)
       status, text = @transport.call(url, headers, body)
       response = parse(text)
+      @last_usage = usage(response)
       raise Error, "The model gave no answer: HTTP #{status} #{failure(response, text)}" unless status.to_i == 200
 
       answer = parse(send("#{@provider}_answer", response))
@@ -81,6 +85,18 @@ module BrowserReviewGate
       Array(response.dig("candidates", 0, "content", "parts")).filter_map { |part| part["text"] if part.is_a?(Hash) }.join
     end
 
+    def usage(response)
+      input, output = case @provider
+                      when "anthropic" then [ response.dig("usage", "input_tokens"), response.dig("usage", "output_tokens") ]
+                      when "openai" then [ response.dig("usage", "prompt_tokens"), response.dig("usage", "completion_tokens") ]
+                      else [ response.dig("usageMetadata", "promptTokenCount"), response.dig("usageMetadata", "candidatesTokenCount") ]
+                      end
+      return unless input.is_a?(Integer) || output.is_a?(Integer)
+
+      { "provider" => @provider, "model" => (response["model"] || response["modelVersion"] || @model).to_s,
+        "input_tokens" => input, "output_tokens" => output }.compact
+    end
+
     def parse(text)
       data = JSON.parse(text.to_s)
       data.is_a?(Hash) ? data : {}

data/lib/browser_review_gate/model_client.rb

@@ -32,6 +32,9 @@ module BrowserReviewGate
       @runner = runner || method(:run)
     end
 
+    # Token counts and cost of the last call as the CLI reported them, or nil.
+    attr_reader :last_usage
+
     # Returns the model's answer as JSON text. Raises Error when the call fails or gives no answer.
     def complete(system:, user:, schema:)
       argv = @command + [ "--print", "--tools", "", "--setting-sources", "", "--system-prompt", system,
@@ -39,6 +42,7 @@ module BrowserReviewGate
       argv += [ "--model", @model ] if @model
       stdout, stderr, success = @runner.call(environment, argv, user)
       result = parse(stdout)
+      @last_usage = usage(result)
       answer = result["structured_output"] if success && result["is_error"] == false
       return JSON.generate(answer) if answer.is_a?(Hash)
 
@@ -47,6 +51,12 @@ module BrowserReviewGate
 
     private
 
+    def usage(result)
+      tokens = result["usage"].is_a?(Hash) ? result["usage"] : {}
+      { "provider" => "claude-cli", "model" => @model, "input_tokens" => tokens["input_tokens"],
+        "output_tokens" => tokens["output_tokens"], "cost_usd" => result["total_cost_usd"] }.compact.then { |found| found.size > 1 ? found : nil }
+    end
+
     def parse(stdout)
       result = JSON.parse(stdout.to_s)
       result.is_a?(Hash) ? result : {}

data/lib/browser_review_gate/prompts.rb

@@ -30,10 +30,10 @@ module BrowserReviewGate
       text(RULES)
     end
 
-    def assessment
+    def assessment(max_count: 12)
       parts = [ FRAME_HEAD, "## Decision\n\n#{policy}" ]
       parts << "## Project rules\n\n#{PROJECT_RULES_NOTE}\n\n#{rules}" if rules
-      parts << FRAME_TAIL
+      parts << format(FRAME_TAIL, max_count: max_count)
       parts.join("\n\n")
     end
 
@@ -58,9 +58,14 @@ module BrowserReviewGate
 
       When the decision is `required`, `scenarios` lists what a person must do in the browser to exercise
       every behavior this pull request changes, including what the project rules require for the areas
-      it touches. One entry per scenario, at most 12, the fewest that cover the change. `name` says what
-      to do and what must be seen, in one sentence. `id` is a short label such as `SIGN-IN-1`: letters,
-      digits and hyphens.
+      it touches. One entry per scenario, at most %<max_count>d, the fewest that cover the change. `name` says
+      what to do and what must be seen, in one sentence. `id` is a short label such as `SIGN-IN-1`:
+      letters, digits and hyphens. Derive the id from the behavior, not from its position, so the same
+      behavior gets the same id when the pull request changes. `priority` is `critical` when a failure
+      would block people from a core task or expose data, otherwise `normal`. `category` is one short
+      lowercase word for the area, such as `forms`, `navigation` or `permissions`. `why` is one sentence
+      saying which change makes this scenario necessary. Scenarios the project settings make mandatory
+      are added after you answer; do not list them.
 
       When `report_cases` is present, those are the cases of an earlier browser run. If one of them
       already exercises a scenario, give that scenario the `id` of that case, exactly. Give every other