browser_review_gate 0.1.1 → 0.2.0

data/lib/browser_review_gate/hook.rb

@@ -1,27 +1,23 @@
 require "json"
 
 module BrowserReviewGate
-  # Runs around an agent's shell commands.
-  #
-  # After a command that opened a PR, the browser run saved for this commit is published.
-  # Before a command that opens a PR or asks for review, the agent is told when a browser run is still
-  # owed, so the request is not made only to be taken back. It never fails the agent's session.
+  # Runs after an agent's shell command. When the command opened a PR, the browser run saved for this
+  # commit is published. It never holds a command back and never fails the agent: running the
+  # verification is the author's choice, and the gate on review requests lives in CI.
   class Hook
     OPEN_PR = "gh pr create"
-    REVIEW_REQUEST = /gh pr (create|edit)\b.*(--reviewer|--add-reviewer|-r )|requested_reviewers/
-    SKIP = "BROWSER_REVIEW_GATE_SKIP=1"
 
-    def initialize(publisher_factory:, status_factory: nil, config: Config.new, shell: Shell.new)
+    def initialize(publisher_factory:, shell: Shell.new)
       @publisher_factory = publisher_factory
-      @status_factory = status_factory
-      @config = config
       @shell = shell
     end
 
     # `payload` is the raw hook input. Returns a message for the agent, or nil when nothing was done.
     def after(payload)
       return unless payload.to_s.include?(OPEN_PR)
-      return unless saved_run?
+
+      sha = @shell.call("git", "rev-parse", "HEAD").strip
+      return unless SavedReport.new(sha: sha, shell: @shell).exist?
 
       verified = @publisher_factory.call.publish
       verified ? "Saved browser verification published; human review can be requested." :
@@ -30,58 +26,12 @@ module BrowserReviewGate
       "Saved browser verification was not published: #{error.message}"
     end
 
-    # Returns the reason the command should wait, or nil to let it run.
-    def before(payload)
-      text = payload.to_s
-      return if text.include?(SKIP)
-
-      reason = review_request_reason(text) if text.match?(REVIEW_REQUEST)
-      reason ||= open_pr_reason if text.include?(OPEN_PR)
-      reason && "#{reason} If the person asked to go ahead anyway, run the same command prefixed with #{SKIP}."
-    rescue Error
-      nil
-    end
-
-    # Claude Code and Codex read a JSON object; Cursor reads its own shape.
-    def self.render(message, agent, before: false)
+    # Claude Code and Codex read a JSON object; other agents show plain text.
+    def self.render(message, agent)
       return if message.nil?
-      return JSON.generate(permission: "deny", agentMessage: message, userMessage: message) if before && agent == "cursor"
       return message unless %w[claude codex].include?(agent)
-      return JSON.generate(hookSpecificOutput: { hookEventName: "PostToolUse", additionalContext: message }) unless before
-
-      JSON.generate(hookSpecificOutput: { hookEventName: "PreToolUse", permissionDecision: "deny", permissionDecisionReason: message })
-    end
-
-    private
-
-    def saved_run?
-      SavedReport.new(sha: @shell.call("git", "rev-parse", "HEAD").strip, shell: @shell).exist?
-    end
-
-    # No PR exists yet, so CI has not decided; only the changed paths can tell.
-    def open_pr_reason
-      return if saved_run? || changed_files.all? { |path| @config.ignored?(path) }
-
-      "No browser run is saved for this commit, and the change may alter browser behavior. " \
-        "Run #{@config.command} first: the run is published when the PR is opened and review requests stay."
-    end
-
-    def review_request_reason(text)
-      status = @status_factory.call.to_h(text[/gh pr edit\s+(\d+)/, 1]&.to_i)
-      case status["next"]
-      when "publish_saved" then "A browser run is saved but not published. Run `browser-review-gate publish` first, or the review request will be taken back."
-      when "run_and_publish", "ask_to_start_app" then "This PR needs a browser run (#{@config.command} #{status["pull_request"]}); a review request made now will be taken back."
-      end
-    end
 
-    # What the PR would contain. When no base branch can be found the change counts as unknown.
-    def changed_files
-      %w[origin/HEAD origin/main origin/master].each do |base|
-        return @shell.call("git", "diff", "--name-only", "#{base}...HEAD").lines.map(&:chomp)
-      rescue Error
-        next
-      end
-      [ "unknown" ]
+      JSON.generate(hookSpecificOutput: { hookEventName: "PostToolUse", additionalContext: message })
     end
   end
 end

data/lib/browser_review_gate/installer.rb

@@ -110,25 +110,23 @@ module BrowserReviewGate
 
     def install_claude
       managed(".claude/skills/browser-pr-verification/SKILL.md", render("claude_skill.md"))
-      add_hooks(".claude/settings.json", "claude") do |hooks, command, before|
-        (hooks[before ? "PreToolUse" : "PostToolUse"] ||= []) <<
-          { "matcher" => "Bash", "hooks" => [ { "type" => "command", "command" => command, "timeout" => 60 } ] }
+      add_hook(".claude/settings.json", "claude", %w[PreToolUse]) do |hooks, command|
+        (hooks["PostToolUse"] ||= []) << { "matcher" => "Bash", "hooks" => [ { "type" => "command", "command" => command, "timeout" => 60 } ] }
       end
     end
 
     def install_cursor
       managed(".cursor/commands/browser-pr-verification.md", render("cursor_command.md"))
-      add_hooks(".cursor/hooks.json", "cursor", "version" => 1) do |hooks, command, before|
-        (hooks[before ? "beforeShellExecution" : "afterShellExecution"] ||= []) << { "command" => command }
+      add_hook(".cursor/hooks.json", "cursor", %w[beforeShellExecution], "version" => 1) do |hooks, command|
+        (hooks["afterShellExecution"] ||= []) << { "command" => command }
       end
     end
 
     # Codex runs a project hook only after the person trusts it in /hooks, so the AGENTS.md section also
     # tells the agent to publish after it opens the PR.
     def install_codex
-      add_hooks(".codex/hooks.json", "codex") do |hooks, command, before|
-        (hooks[before ? "PreToolUse" : "PostToolUse"] ||= []) <<
-          { "matcher" => "^Bash$", "hooks" => [ { "type" => "command", "command" => command, "timeout" => 60 } ] }
+      add_hook(".codex/hooks.json", "codex", %w[PreToolUse]) do |hooks, command|
+        (hooks["PostToolUse"] ||= []) << { "matcher" => "^Bash$", "hooks" => [ { "type" => "command", "command" => command, "timeout" => 60 } ] }
       end
       path = "AGENTS.md"
       current = exist?(path) ? File.read(File.join(@root, path)) : nil
@@ -139,8 +137,8 @@ module BrowserReviewGate
 
     # A machine without the gem (a cloud agent, a fresh container) must not see a failing hook after
     # every command.
-    def hook_command(agent, before)
-      "command -v browser-review-gate >/dev/null 2>&1 && #{HOOK} #{agent}#{" --before" if before} || true"
+    def hook_command(agent)
+      "command -v browser-review-gate >/dev/null 2>&1 && #{HOOK} #{agent} || true"
     end
 
     def render(name)
@@ -161,14 +159,22 @@ module BrowserReviewGate
       write(path, content, current)
     end
 
-    # Adds the after-command and before-command hook entries the file does not have yet.
-    def add_hooks(path, agent, defaults = {})
+    # Adds the after-command hook entry when the file lacks it, and removes the before-command entries
+    # version 0.1.1 put under `stale_events`: nothing holds a command back any more.
+    def add_hook(path, agent, stale_events, defaults = {})
       current = exist?(path) ? File.read(File.join(@root, path)) : nil
-      missing = [ false, true ].reject { |before| current.to_s.match?(before ? /hook #{agent} --before/ : /hook #{agent}(?! --before)/) }
-      return @log.puts("  unchanged #{path}") if missing.empty?
+      present = current.to_s.match?(/hook #{agent}(?! --before)/)
+      return @log.puts("  unchanged #{path}") if present && !current.include?("hook #{agent} --before")
 
       settings = defaults.merge(current ? JSON.parse(current) : {})
-      missing.each { |before| yield(settings["hooks"] ||= {}, hook_command(agent, before), before) }
+      hooks = (settings["hooks"] ||= {})
+      stale_events.each do |event|
+        next unless hooks[event]
+
+        hooks[event].reject! { |entry| JSON.generate(entry).include?("hook #{agent} --before") }
+        hooks.delete(event) if hooks[event].empty?
+      end
+      yield(hooks, hook_command(agent)) unless present
       write(path, "#{JSON.pretty_generate(settings)}\n", current)
     end
 
@@ -186,7 +192,7 @@ module BrowserReviewGate
     def next_steps
       <<~TEXT
         Next:
-          1. Add a CLAUDE_CODE_OAUTH_TOKEN or ANTHROPIC_API_KEY secret to the repository.
+          1. Add one secret to the repository: CLAUDE_CODE_OAUTH_TOKEN, ANTHROPIC_API_KEY, OPENAI_API_KEY or GEMINI_API_KEY.
           2. Fill start_command, url and sign_in in #{Config::PATH}.
           3. Describe what this application always checks in #{Prompts::RULES}.
           4. Commit and merge to the default branch: the workflow runs from there.

data/lib/browser_review_gate/model_api.rb

@@ -0,0 +1,108 @@
+require "json"
+require "net/http"
+require "uri"
+
+module BrowserReviewGate
+  # One request to a model provider's HTTP API, with no tools and the answer bound to a JSON schema.
+  # For projects that assess with an API key instead of the Claude Code CLI.
+  class ModelApi
+    PROVIDERS = {
+      "anthropic" => { key: "ANTHROPIC_API_KEY", model: "claude-opus-5-5" },
+      "openai" => { key: "OPENAI_API_KEY" },
+      "gemini" => { key: "GEMINI_API_KEY" }
+    }.freeze
+    MAX_TOKENS = 16_000
+
+    # `transport` takes the URL, the headers and the request body, and returns [HTTP status, body].
+    def initialize(provider:, model: nil, transport: nil)
+      settings = PROVIDERS.fetch(provider) { raise Error, "Unknown model provider #{provider.inspect} (known: #{PROVIDERS.keys.join(", ")})" }
+      @provider = provider
+      @key_name = settings.fetch(:key)
+      @default_model = settings[:model]
+      @model = model.to_s.empty? ? @default_model : model
+      @transport = transport || method(:post)
+      raise Error, "Set `model` in #{Config::PATH} to assess with #{provider}" unless @model
+    end
+
+    # Returns the model's answer as JSON text. Raises Error when the call fails or gives no answer.
+    def complete(system:, user:, schema:)
+      key = ENV[@key_name].to_s
+      raise Error, "The model gave no answer: #{@key_name} is not set" if key.empty?
+
+      url, headers, body = send("#{@provider}_request", key, system, user, schema)
+      status, text = @transport.call(url, headers, body)
+      response = parse(text)
+      raise Error, "The model gave no answer: HTTP #{status} #{failure(response, text)}" unless status.to_i == 200
+
+      answer = parse(send("#{@provider}_answer", response))
+      return JSON.generate(answer) unless answer.empty?
+
+      raise Error, "The model gave no answer: #{failure(response, text)}"
+    end
+
+    private
+
+    # A request the provider's safeguards decline is retried on the provider's recommended fallback.
+    # That is asked for only with the built-in model: a model the project chose may not accept it.
+    def anthropic_request(key, system, user, schema)
+      headers = { "x-api-key" => key, "anthropic-version" => "2023-06-01" }
+      body = { model: @model, max_tokens: MAX_TOKENS, system: system, messages: [ { role: "user", content: user } ],
+               output_config: { format: { type: "json_schema", schema: schema } } }
+      if @model == @default_model
+        headers["anthropic-beta"] = "server-side-fallback-2026-07-01"
+        body[:fallbacks] = "default"
+      end
+      [ "https://api.anthropic.com/v1/messages", headers, body ]
+    end
+
+    def anthropic_answer(response)
+      return if response["stop_reason"] == "refusal"
+
+      Array(response["content"]).find { |block| block.is_a?(Hash) && block["type"] == "text" }&.fetch("text", nil)
+    end
+
+    def openai_request(key, system, user, schema)
+      body = { model: @model, messages: [ { role: "system", content: system }, { role: "user", content: user } ],
+               response_format: { type: "json_schema", json_schema: { name: "browser_assessment", schema: schema, strict: true } } }
+      [ "https://api.openai.com/v1/chat/completions", { "Authorization" => "Bearer #{key}" }, body ]
+    end
+
+    def openai_answer(response)
+      response.dig("choices", 0, "message", "content")
+    end
+
+    def gemini_request(key, system, user, schema)
+      body = { systemInstruction: { parts: [ { text: system } ] }, contents: [ { role: "user", parts: [ { text: user } ] } ],
+               generationConfig: { responseMimeType: "application/json", responseJsonSchema: schema } }
+      [ "https://generativelanguage.googleapis.com/v1beta/models/#{@model}:generateContent", { "x-goog-api-key" => key }, body ]
+    end
+
+    def gemini_answer(response)
+      Array(response.dig("candidates", 0, "content", "parts")).filter_map { |part| part["text"] if part.is_a?(Hash) }.join
+    end
+
+    def parse(text)
+      data = JSON.parse(text.to_s)
+      data.is_a?(Hash) ? data : {}
+    rescue JSON::ParserError
+      {}
+    end
+
+    # Providers explain an error in `error.message`; a declined request has no text to return.
+    def failure(response, text)
+      message = response.dig("error", "message") if response["error"].is_a?(Hash)
+      message ||= "the request was declined" if response["stop_reason"] == "refusal" || response.dig("choices", 0, "message", "refusal")
+      (message || text.to_s.strip)[0, 300]
+    end
+
+    def post(url, headers, body)
+      uri = URI(url)
+      request = Net::HTTP::Post.new(uri, headers.merge("Content-Type" => "application/json"))
+      request.body = JSON.generate(body)
+      response = Net::HTTP.start(uri.host, uri.port, use_ssl: true, open_timeout: 10, read_timeout: 300) { |http| http.request(request) }
+      [ response.code, response.body ]
+    rescue SystemCallError, SocketError, Timeout::Error, OpenSSL::SSL::SSLError => error
+      raise Error, "The model gave no answer: #{error.message}"
+    end
+  end
+end

data/lib/browser_review_gate/model_client.rb

@@ -9,6 +9,22 @@ module BrowserReviewGate
     NPX = %w[npx --yes @anthropic-ai/claude-code@2.1.288].freeze
     CREDENTIALS = %w[CLAUDE_CODE_OAUTH_TOKEN ANTHROPIC_API_KEY].freeze
 
+    # The client the project's settings and the credentials at hand call for. The Claude Code CLI stays
+    # the default; an OpenAI or Gemini key alone selects that provider's API.
+    def self.for(config, environment: ENV)
+      provider = config.provider || detect(config, environment)
+      return new(command: config.claude_command, model: config.model) if provider == "claude-cli"
+
+      ModelApi.new(provider: provider, model: config.model)
+    end
+
+    def self.detect(config, environment)
+      set = ->(name) { !environment[name].to_s.empty? }
+      return "claude-cli" if config.claude_command || CREDENTIALS.any?(&set)
+
+      ModelApi::PROVIDERS.keys.find { |provider| set.call(ModelApi::PROVIDERS.fetch(provider).fetch(:key)) } || "claude-cli"
+    end
+
     # `runner` takes the environment, argv and stdin, and returns [stdout, stderr, success?].
     def initialize(command: nil, model: nil, runner: nil)
       @command = command.to_s.empty? ? default_command : command.split

data/lib/browser_review_gate/prompts.rb

@@ -49,21 +49,29 @@ module BrowserReviewGate
 
     PROJECT_RULES_NOTE = <<~TEXT.strip.freeze
       The maintainers of this repository wrote these rules for their application. They refine the
-      decision above and win where they disagree with it. Use the scenarios they name when you judge
-      whether an earlier report covers the change.
+      decision above and win where they disagree with it. Include the scenarios they name for the areas
+      the change touches.
     TEXT
 
     FRAME_TAIL = <<~TEXT.strip.freeze
-      ## Coverage
+      ## Scenarios
 
-      `covered_by_report` is true only when `report_cases` is present and those cases already exercise
-      every browser-visible behavior this pull request changes, including what the project rules require
-      for the areas it touches. Otherwise it is false. It is false when the decision is `not-required`.
+      When the decision is `required`, `scenarios` lists what a person must do in the browser to exercise
+      every behavior this pull request changes, including what the project rules require for the areas
+      it touches. One entry per scenario, at most 12, the fewest that cover the change. `name` says what
+      to do and what must be seen, in one sentence. `id` is a short label such as `SIGN-IN-1`: letters,
+      digits and hyphens.
+
+      When `report_cases` is present, those are the cases of an earlier browser run. If one of them
+      already exercises a scenario, give that scenario the `id` of that case, exactly. Give every other
+      scenario an `id` no case uses. Never reuse the `id` of a case that does not exercise the scenario.
+
+      When the decision is `not-required`, `scenarios` is an empty list.
 
       ## Reason
 
       `reason` is one plain sentence of at most 30 words. No links, mentions, code, or HTML. No company
-      or customer names.
+      or customer names. The same holds for scenario names.
     TEXT
 
     private

data/lib/browser_review_gate/publisher.rb

@@ -30,7 +30,7 @@ module BrowserReviewGate
       actor = @github.login
       data["verified_by"] = "AI browser agent (run by #{actor})"
 
-      comments = @github.comments(number)
+      comments = @github.trusted_comments(number)
       ensure_browser_run_wanted!(number, comments, local_sha)
       report = merged_report(data, comments)
 
@@ -54,7 +54,7 @@ module BrowserReviewGate
       data = read_report_file
       ensure_report_sha!(data, local_sha)
       report = Report.new(data.merge("verified_by" => "pending"))
-      raise Error, report.errors.join("; ") unless report.valid?
+      ensure_acceptable!(report)
 
       saved_report = SavedReport.new(sha: local_sha, shell: @shell)
       saved_report.write(data)
@@ -107,7 +107,7 @@ module BrowserReviewGate
 
     def merged_report(data, comments)
       submitted = Report.new(data)
-      raise Error, submitted.errors.join("; ") unless submitted.valid?
+      ensure_acceptable!(submitted)
 
       prior = Report.latest_comment(comments)
       report = Report.merge(prior ? Report.data_from(prior["body"]) : {}, submitted.to_h)
@@ -116,6 +116,12 @@ module BrowserReviewGate
       report
     end
 
+    # A run is taken only with the evidence of each case: the page and what was observed there.
+    def ensure_acceptable!(report)
+      problems = report.valid? ? report.evidence_errors : report.errors
+      raise Error, problems.join("; ") if problems.any?
+    end
+
     def body(report, complete)
       return report.markdown if complete
 

data/lib/browser_review_gate/report.rb

@@ -1,3 +1,4 @@
+require "digest"
 require "json"
 
 module BrowserReviewGate
@@ -7,6 +8,7 @@ module BrowserReviewGate
     SHA_PATTERN = /\A\h{7,40}\z/
     CASE_ID_PATTERN = /\A[A-Za-z0-9][A-Za-z0-9_.-]{0,63}\z/
     RESULTS = %w[pass fail].freeze
+    URL_PATTERN = %r{\A(?:https?://|/)\S*\z}
 
     attr_reader :errors
 
@@ -21,24 +23,32 @@ module BrowserReviewGate
       raise Error, "Existing browser verification report has a malformed data marker"
     end
 
-    # The newest passing report on the PR, or nil.
-    def self.latest_passing(comments)
+    # The newest report on the PR when it is well formed, or nil.
+    def self.latest(comments)
       comment = latest_comment(comments)
       report = comment && new(data_from(comment["body"]))
-      report if report&.passed?
+      report if report&.valid?
     rescue Error
       nil
     end
 
-    # Later results replace earlier ones case by case; cases that were not run again are kept.
+    # The newest report on the PR when it passes, or nil.
+    def self.latest_passing(comments)
+      report = latest(comments)
+      report if report&.passed?
+    end
+
+    # Later results replace earlier ones case by case; cases that were not run again are kept. The
+    # merged report counts the runs published on the PR and how many of them had a failing case.
     def self.merge(previous_data, latest_data)
       previous = new(previous_data)
       latest = new(latest_data)
-      return latest unless previous.valid? && latest.valid?
+      return latest unless latest.valid?
 
-      combined = previous.cases.to_h { |test_case| [ test_case["id"], test_case ] }
+      combined = previous.valid? ? previous.cases.to_h { |test_case| [ test_case["id"], test_case ] } : {}
       latest.cases.each { |test_case| combined[test_case["id"]] = test_case }
-      new(latest.to_h.merge("cases" => combined.values))
+      failed = latest.cases.all? { |test_case| test_case["result"] == "pass" } ? 0 : 1
+      new(latest.to_h.merge("cases" => combined.values, "runs" => previous.runs + 1, "failed_runs" => previous.failed_runs + failed))
     end
 
     def initialize(data)
@@ -53,7 +63,28 @@ module BrowserReviewGate
     def verified_by = @data["verified_by"]
     def cases = @data["cases"].is_a?(Array) ? @data["cases"] : []
     def excluded = @data["excluded"].is_a?(Array) ? @data["excluded"] : []
+    def runs = @data["runs"].is_a?(Integer) ? @data["runs"] : 0
+    def failed_runs = @data["failed_runs"].is_a?(Integer) ? @data["failed_runs"] : 0
     def to_h = JSON.parse(JSON.generate(@data))
+    def passed_ids = cases.select { |test_case| test_case["result"] == "pass" }.map { |test_case| test_case["id"] }
+
+    # Changes whenever a case is added, re-run or changes its outcome.
+    def fingerprint
+      Digest::SHA256.hexdigest(JSON.generate(cases.map { |test_case| test_case.values_at("id", "result", "tested_sha") }))[0, 16]
+    end
+
+    # What a newly submitted run must say about each case: where it was exercised and what was seen.
+    # Cases carried over from reports published before this was required are not checked.
+    def evidence_errors
+      cases.each_with_index.flat_map do |test_case, index|
+        next [] unless test_case.is_a?(Hash)
+
+        found = []
+        found << "case #{index + 1} url must be the page that was exercised: an http(s) URL or a path starting with /" unless url?(test_case["url"])
+        found << "case #{index + 1} details must say what was observed" unless text?(test_case["details"], 500)
+        found
+      end
+    end
 
     def passed?
       valid? && @data["coverage_complete"] && cases.all? { |test_case| test_case["result"] == "pass" }
@@ -122,6 +153,7 @@ module BrowserReviewGate
       errors << "case #{position} result must be pass or fail" unless RESULTS.include?(test_case["result"])
       errors << "case #{position} tested_sha must be a 7–40 character hexadecimal commit SHA" unless sha?(test_case["tested_sha"])
       errors << "case #{position} verified_by must be a non-empty string of at most 100 characters" unless text?(test_case["verified_by"], 100)
+      errors << "case #{position} url must be a string of at most 300 characters" unless test_case["url"].nil? || url?(test_case["url"])
       details = test_case["details"]
       errors << "case #{position} details must be a string of at most 500 characters" unless details.nil? || (details.is_a?(String) && details.length <= 500)
     end
@@ -140,10 +172,12 @@ module BrowserReviewGate
 
     def sha?(value) = value.is_a?(String) && SHA_PATTERN.match?(value)
     def text?(value, limit) = value.is_a?(String) && !value.strip.empty? && value.length <= limit
+    def url?(value) = value.is_a?(String) && value.length <= 300 && URL_PATTERN.match?(value)
 
     def markdown_case(test_case)
       detail = test_case["details"].to_s.empty? ? "" : ": #{Markdown.escape(test_case["details"])}"
-      "- #{test_case["result"].upcase} — #{Markdown.escape(test_case["name"])} (`#{test_case["id"]}`, tested " \
+      place = test_case["url"].to_s.empty? ? "" : " at #{Markdown.escape(test_case["url"])}"
+      "- #{test_case["result"].upcase} — #{Markdown.escape(test_case["name"])}#{place} (`#{test_case["id"]}`, tested " \
         "`#{test_case["tested_sha"]}` by #{Markdown.escape(test_case["verified_by"])})#{detail}"
     end
 

data/lib/browser_review_gate/stats.rb

@@ -0,0 +1,34 @@
+module BrowserReviewGate
+  # What the gate did on recent pull requests, read back from its own comments and labels.
+  class Stats
+    def initialize(github:, config: Config.new)
+      @github = github
+      @config = config
+    end
+
+    def to_h(since:)
+      counts = Hash.new(0)
+      @github.pull_requests_updated_since(since).each { |pull_request| count(pull_request, counts) }
+      %w[pull_requests assessed required verified waived owed found_a_failure].to_h { |key| [ key, counts[key] ] }
+    end
+
+    private
+
+    def count(pull_request, counts)
+      counts["pull_requests"] += 1
+      comments = @github.trusted_comments(pull_request.fetch("number"))
+      assessment = Assessment.latest(comments)
+      return unless assessment
+
+      counts["assessed"] += 1
+      return unless assessment.required?
+
+      labels = Array(pull_request["labels"]).map { |label| label["name"] }
+      counts["required"] += 1
+      counts["verified"] += 1 if labels.include?(@config.label)
+      counts["waived"] += 1 if labels.include?(@config.waiver_label)
+      counts["owed"] += 1 if pull_request["state"] == "open" && !labels.intersect?([ @config.label, @config.waiver_label ])
+      counts["found_a_failure"] += 1 if Report.latest(comments)&.failed_runs.to_i.positive?
+    end
+  end
+end

data/lib/browser_review_gate/status.rb

@@ -8,7 +8,8 @@ module BrowserReviewGate
     RUN_STEPS = %w[run_and_publish run_and_save].freeze
 
     # `probe` takes the app URL and says whether something answers there.
-    def initialize(github:, config: Config.new, shell: Shell.new, probe: nil)
+    def initialize(github:, config: Config.new, shell: Shell.new, probe: nil, prompts: Prompts.new)
+      @prompts = prompts
       @github = github
       @config = config
       @shell = shell
@@ -17,7 +18,7 @@ module BrowserReviewGate
 
     def to_h(number = nil)
       result = facts(number)
-      result["project_rules"] = Prompts::RULES if Prompts.new.rules
+      result["project_rules"] = Prompts::RULES if @prompts.rules
       result["app"] = app
       # The agent never starts the app itself: a person decides what runs on their machine.
       result["next"] = "ask_to_start_app" if RUN_STEPS.include?(result["next"]) && result["app"]["running"] == false
@@ -33,16 +34,19 @@ module BrowserReviewGate
       return { "pull_request" => nil, "local_head" => local_sha, "saved_run" => saved, "next" => saved ? "open_pull_request" : "run_and_save" } unless number
 
       pull_request = @github.pull_request(number)
-      comments = @github.comments(number)
+      comments = @github.trusted_comments(number)
       head_sha = pull_request.fetch("head").fetch("sha")
       assessment = Assessment.latest(comments)
       assessment = nil unless assessment&.for?(head_sha)
       labels = pull_request.fetch("labels").map { |label| label["name"] }
-      verified = labels.include?(@config.label) && !Report.latest_passing(comments).nil?
+      report = Report.latest(comments)
+      verified = labels.include?(@config.label) && (assessment ? assessment.covered_by?(report) : report&.passed? == true)
       waived = labels.include?(@config.waiver_label)
 
       { "pull_request" => number, "pr_head" => head_sha, "local_head" => local_sha,
-        "decision" => assessment&.decision, "reason" => assessment&.reason, "verified" => verified, "waived" => waived,
+        "decision" => assessment&.decision, "reason" => assessment&.reason,
+        # What the run must exercise; each id is the `id` of the case that reports it.
+        "scenarios" => assessment ? assessment.outstanding(report) : [], "verified" => verified, "waived" => waived,
         "saved_run" => saved, "next" => next_step(assessment, verified || waived, saved, head_sha == local_sha) }
     end
 

data/lib/browser_review_gate/templates/config.yml.erb

@@ -6,6 +6,12 @@ url:<%= url ? " #{url.inspect}" : "" %>
 # How to sign in locally, e.g. "use the seed account from db/seeds.rb". No real credentials here.
 sign_in:
 
+# Who assesses a PR in CI. Unset: the Claude Code CLI with a CLAUDE_CODE_OAUTH_TOKEN or
+# ANTHROPIC_API_KEY secret; with only an OPENAI_API_KEY or GEMINI_API_KEY secret, that provider's API.
+# Set `provider` to claude-cli, anthropic, openai or gemini to choose; openai and gemini need `model`.
+provider:
+model:
+
 # Workflow file name under .github/workflows, used to request an assessment by hand.
 workflow: <%= workflow %>
 

data/lib/browser_review_gate/templates/playbook.md.erb

@@ -15,10 +15,13 @@ This file is the playbook for the AI coding agent that does the run. The person
    - `open_pull_request`: a run is saved and waits for the PR. Tell the person to open it.
    - `ask_to_start_app`: the app is not running. Do not start it yourself: ask the person to start it (`app.start_command` says how) and stop until they have.
    - `run_and_publish` or `run_and_save`: continue with step 2.
-2. Read the diff against the base branch and the earlier report in the PR, if any. Then read `<%= rules_path %>`: it says what this project always checks and which scenarios each area needs, and its rules are mandatory. List the scenarios for the behavior that changed, plus the ones the project rules require for the areas the change touches. Do not re-run behavior an earlier passing case still covers.
+2. Decide what to run.
+   - `scenarios` in the status output is the list CI wrote for this PR. Run every one of them, and give each case the `id` of its scenario. You may add cases of your own; you may not drop or rename one from the list.
+   - With no PR yet, or an empty list, choose the scenarios yourself: read the diff against the base branch, then `<%= rules_path %>`, which says what this project always checks and which scenarios each area needs. Its rules are mandatory. CI compares your cases with its own list once the PR is open and asks for what is missing.
+   - Do not re-run behavior an earlier passing case still covers.
 3. Open the running app in a browser tool. `app` in the status output gives the URL and a sign-in hint. Never start, restart or stop the app yourself: what runs on the person's machine is their decision. Use local test data and do not destroy existing data. If the app, the data, or a browser tool is missing, report the blocker. Reading code and running unit tests does not replace a browser run.
-4. Run every scenario end to end. After key actions check the page state, console errors, and failed requests. Record what you observed. A skipped or blocked scenario is not a pass.
-5. Write the report as JSON outside the repository (see the shape below). Set `coverage_complete` to true only when every scenario from step 2 ran.
+4. Run every scenario end to end. After key actions check the page state, console errors, and failed requests. For each case record the page it was exercised on (`url`) and what you observed there, including console errors and failed requests (`details`). A case without both is refused. A skipped or blocked scenario is not a pass.
+5. Write the report as JSON outside the repository (see the shape below). Set `coverage_complete` to true only when every scenario from step 2 ran. CI checks the report against its list by case `id`: a missing scenario takes the label away again.
 6. Publish or save, from the checkout that was tested, with everything committed and pushed:
    - PR exists: `browser-review-gate publish --report PATH` (add `--pr NUMBER` when the branch has several). It posts the cases with their outcomes. Only a complete all-pass report adds the `<%= label %>` label.
    - No PR yet: `browser-review-gate save --report PATH`. The run is published when the PR for this commit is opened. A new commit makes it stale.
@@ -36,7 +39,7 @@ Some scenarios cannot be exercised locally, for example a sign-in through an ext
   "tested_sha": "<full commit SHA that was tested>",
   "coverage_complete": true,
   "cases": [
-    { "id": "BROWSER-1", "name": "The changed interaction completes", "result": "pass", "details": "What was observed." }
+    { "id": "BROWSER-1", "name": "The changed interaction completes", "result": "pass", "url": "/the/page/exercised", "details": "What was observed, with console errors and failed requests if any." }
   ],
   "excluded": [
     { "name": "Sign-in through an external provider", "reason": "Needs a real provider account; covered by integration tests." }
@@ -44,7 +47,7 @@ Some scenarios cannot be exercised locally, for example a sign-in through an ext
 }
 ```
 
-`result` is `pass` or `fail`. `excluded` is optional: list a scenario there only when the person decided it stays outside the local run. Never exclude a scenario on your own to make the run complete. Keep customer and real company names out of the report.
+`result` is `pass` or `fail`. `url` is an http(s) URL or a path starting with `/`. `excluded` is optional: list a scenario there only when the person decided it stays outside the local run. Never exclude a scenario on your own to make the run complete. Keep customer and real company names out of the report.
 
 ## Rules