browser_review_gate 0.2.2 → 0.4.0

data/lib/browser_review_gate/stats.rb

@@ -1,6 +1,12 @@
+require "time"
+
 module BrowserReviewGate
-  # What the gate did on recent pull requests, read back from its own comments and labels.
+  # What the gate did on recent pull requests, read back from its own comments and labels. Only what
+  # the comments record is counted: each PR keeps its latest assessment and its merged report, so
+  # earlier assessments of the same PR and their token usage are not visible any more.
   class Stats
+    COUNTS = %w[pull_requests assessed required verified waived owed found_a_failure runs failed_runs].freeze
+
     def initialize(github:, config: Config.new)
       @github = github
       @config = config
@@ -8,27 +14,105 @@ module BrowserReviewGate
 
     def to_h(since:)
       counts = Hash.new(0)
-      @github.pull_requests_updated_since(since).each { |pull_request| count(pull_request, counts) }
-      %w[pull_requests assessed required verified waived owed found_a_failure].to_h { |key| [ key, counts[key] ] }
+      failing = Hash.new(0)
+      hours = []
+      usage = { "assessments_with_usage" => 0, "input_tokens" => 0, "output_tokens" => 0, "assessments_with_cost" => 0, "cost_usd" => 0.0 }
+      @github.pull_requests_updated_since(since).each { |pull_request| count(pull_request, counts, failing, hours, usage) }
+
+      result = COUNTS.to_h { |key| [ key, counts[key] ] }
+      result["failing_scenarios"] = failing.sort_by { |id, times| [ -times, id ] }.first(10).to_h
+      result["hours_to_verification"] = hours.empty? ? nil : { "median" => median(hours).round(1), "measured" => hours.size }
+      usage["cost_usd"] = usage["cost_usd"].round(4)
+      result["assessment_usage"] = usage
+      result
+    end
+
+    def self.text(result, days)
+      lines = [ "Pull requests touched in the last #{days} days: #{result["pull_requests"]}",
+        "  assessed by the gate:           #{result["assessed"]}",
+        "  needed a browser run:           #{result["required"]}",
+        "    verified by a published run:  #{result["verified"]}",
+        "    waived:                       #{result["waived"]}",
+        "    open and still owing a run:   #{result["owed"]}",
+        "    a run found a failing case:   #{result["found_a_failure"]}",
+        "  browser runs published:         #{result["runs"]} (#{result["failed_runs"]} with a failing case)" ]
+      if (time = result["hours_to_verification"])
+        lines << "  PR opened to verified run:      median #{time["median"]} h over #{time["measured"]} PRs"
+      end
+      result["failing_scenarios"].each_with_index do |(id, times), index|
+        lines << "  #{index.zero? ? "failing scenarios:" : ""}".ljust(34) + "#{id} (#{times})"
+      end
+      usage = result["assessment_usage"]
+      if usage["assessments_with_usage"].positive?
+        lines << "  latest assessments, tokens:     #{usage["input_tokens"]} in, #{usage["output_tokens"]} out " \
+                 "(#{usage["assessments_with_usage"]} reported)"
+      end
+      if usage["assessments_with_cost"].positive?
+        lines << "  latest assessments, cost:       $#{usage["cost_usd"]} (#{usage["assessments_with_cost"]} reported by the provider)"
+      end
+      lines.join("\n")
     end
 
     private
 
-    def count(pull_request, counts)
+    def count(pull_request, counts, failing, hours, usage)
       counts["pull_requests"] += 1
-      comments = @github.trusted_comments(pull_request.fetch("number"))
+      number = pull_request.fetch("number")
+      comments = @github.trusted_comments(number)
       assessment = Assessment.latest(comments)
       return unless assessment
 
       counts["assessed"] += 1
+      add_usage(assessment.usage, usage)
       return unless assessment.required?
 
       labels = Array(pull_request["labels"]).map { |label| label["name"] }
+      report = Report.latest(comments, config: @config)
       counts["required"] += 1
       counts["verified"] += 1 if labels.include?(@config.label)
       counts["waived"] += 1 if labels.include?(@config.waiver_label)
       counts["owed"] += 1 if pull_request["state"] == "open" && !labels.intersect?([ @config.label, @config.waiver_label ])
-      counts["found_a_failure"] += 1 if Report.latest(comments)&.failed_runs.to_i.positive?
+      return unless report
+
+      counts["runs"] += report.runs
+      counts["failed_runs"] += report.failed_runs
+      counts["found_a_failure"] += 1 if report.failed_runs.positive?
+      report.cases.each { |test_case| failing[test_case["id"]] += 1 if report.status_of(test_case["id"]) == "failed" }
+      measure(pull_request, comments, labels, hours)
+    end
+
+    # From the PR being opened to the report comment's last update, for PRs that are verified now.
+    def measure(pull_request, comments, labels, hours)
+      return unless labels.include?(@config.label)
+
+      comment = Report.latest_comment(comments)
+      opened = pull_request["created_at"]
+      published = comment && (comment["updated_at"] || comment["created_at"])
+      return unless opened && published
+
+      hours << (Time.iso8601(published) - Time.iso8601(opened)) / 3600.0
+    rescue ArgumentError
+      nil
+    end
+
+    def add_usage(found, usage)
+      return unless found
+
+      if found["input_tokens"] || found["output_tokens"]
+        usage["assessments_with_usage"] += 1
+        usage["input_tokens"] += found["input_tokens"].to_i
+        usage["output_tokens"] += found["output_tokens"].to_i
+      end
+      return unless found["cost_usd"]
+
+      usage["assessments_with_cost"] += 1
+      usage["cost_usd"] += found["cost_usd"]
+    end
+
+    def median(values)
+      sorted = values.sort
+      middle = sorted.size / 2
+      sorted.size.odd? ? sorted[middle] : (sorted[middle - 1] + sorted[middle]) / 2.0
     end
   end
 end

data/lib/browser_review_gate/status.rb

@@ -3,35 +3,71 @@ require "openssl"
 require "uri"
 
 module BrowserReviewGate
-  # What an agent needs to know before a browser run: the decision for the PR head and the next step.
+  # What an agent needs to know before a browser run: the decision for the PR head, the scenarios still
+  # owed with their status, the environment to use, the policy, and the next step.
   class Status
     RUN_STEPS = %w[run_and_publish run_and_save].freeze
 
-    # `probe` takes the app URL and says whether something answers there.
+    # `probe` takes a URL and says whether something answers there.
     def initialize(github:, config: Config.new, shell: Shell.new, probe: nil, prompts: Prompts.new)
       @prompts = prompts
       @github = github
       @config = config
       @shell = shell
-      @probe = probe || method(:answers?)
+      @probe = probe || Status.method(:answers?)
     end
 
-    def to_h(number = nil)
+    def to_h(number = nil, environment: nil)
       result = facts(number)
       result["project_rules"] = Prompts::RULES if @prompts.rules
-      result["app"] = app
+      result["environment"] = environment_details(environment)
+      result["policy"] = @config.policy
+      result["report"] = report_instructions
       # The agent never starts the app itself: a person decides what runs on their machine.
-      result["next"] = "ask_to_start_app" if RUN_STEPS.include?(result["next"]) && result["app"]["running"] == false
+      result["next"] = "ask_to_start_app" if RUN_STEPS.include?(result["next"]) && result["environment"]["reachable"] == false
       result
     end
 
+    # Any HTTP answer counts, redirects are not followed, and only http(s) is tried. A local
+    # certificate is not checked. The URL comes from the project settings.
+    def self.answers?(url)
+      uri = URI(url)
+      return false unless %w[http https].include?(uri.scheme) && uri.host
+
+      Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https", verify_mode: OpenSSL::SSL::VERIFY_NONE,
+        open_timeout: 2, read_timeout: 3) { |http| http.head(uri.request_uri) }
+      true
+    rescue StandardError
+      false
+    end
+
+    # The value of `header` in the answer at `url`, or nil. Same limits as answers?.
+    def self.header_at(url, header)
+      uri = URI(url)
+      return unless %w[http https].include?(uri.scheme) && uri.host
+
+      Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https", verify_mode: OpenSSL::SSL::VERIFY_NONE,
+        open_timeout: 2, read_timeout: 3) { |http| http.head(uri.request_uri)[header] }
+    rescue StandardError
+      nil
+    end
+
+    # The URL a health check asks: an absolute `healthcheck`, or its path on the environment URL.
+    def self.healthcheck_url(environment)
+      check = environment["healthcheck"]
+      return environment["url"] if check.nil?
+      return check if check.start_with?("http://", "https://")
+
+      URI.join(environment["url"], check).to_s
+    end
+
     private
 
     def facts(number)
       local_sha = @shell.call("git", "rev-parse", "HEAD").strip
       saved = SavedReport.new(sha: local_sha, shell: @shell).exist?
       number ||= @github.pull_request_number_for_branch
-      return { "pull_request" => nil, "local_head" => local_sha, "saved_run" => saved, "next" => saved ? "open_pull_request" : "run_and_save" } unless number
+      return { "pull_request" => nil, "local_head" => local_sha, "saved_run" => saved, "scenarios" => [], "next" => saved ? "open_pull_request" : "run_and_save" } unless number
 
       pull_request = @github.pull_request(number)
       comments = @github.trusted_comments(number)
@@ -39,17 +75,25 @@ module BrowserReviewGate
       assessment = Assessment.latest(comments)
       assessment = nil unless assessment&.for?(head_sha)
       labels = pull_request.fetch("labels").map { |label| label["name"] }
-      report = Report.latest(comments)
-      verified = labels.include?(@config.label) && (assessment ? assessment.covered_by?(report) : report&.passed? == true)
+      report = Report.latest(comments, config: @config)
+      commits = @github.pull_request_commits(number)
+      reuse = @config.scenarios["reuse_passing"]
+      verified = labels.include?(@config.label) && assessment&.covered_by?(report, commits: commits, reuse_passing: reuse) == true
       waived = labels.include?(@config.waiver_label)
 
       { "pull_request" => number, "pr_head" => head_sha, "local_head" => local_sha,
         "decision" => assessment&.decision, "reason" => assessment&.reason,
         # What the run must exercise; each id is the `id` of the case that reports it.
-        "scenarios" => assessment ? assessment.outstanding(report) : [], "verified" => verified, "waived" => waived,
+        "scenarios" => assessment ? scenarios(assessment, report, commits, reuse) : [], "verified" => verified, "waived" => waived,
         "saved_run" => saved, "next" => next_step(assessment, verified || waived, saved, head_sha == local_sha) }
     end
 
+    def scenarios(assessment, report, commits, reuse)
+      assessment.outstanding(report, commits: commits, reuse_passing: reuse).map do |scenario|
+        scenario.merge("status" => report ? report.status_of(scenario["id"], commits: commits) : "missing")
+      end
+    end
+
     def next_step(assessment, cleared, saved, on_pr_head)
       return "nothing" if cleared || (assessment && !assessment.required?)
       return "check_out_pr_head" unless on_pr_head
@@ -59,20 +103,18 @@ module BrowserReviewGate
       "run_and_publish"
     end
 
-    def app
-      details = @config.app
-      details["running"] = @probe.call(@config.url) if @config.url
+    def environment_details(name)
+      environment = @config.environment(name)
+      return { "name" => nil, "reachable" => nil, "note" => "No environment is configured in #{Config::PATH}." } unless environment
+
+      details = environment.slice("name", "url", "healthcheck", "start_command", "sign_in")
+      details["reachable"] = environment["url"] ? @probe.call(Status.healthcheck_url(environment)) : nil
       details
     end
 
-    # Any HTTP answer counts; a local certificate is not checked.
-    def answers?(url)
-      uri = URI(url)
-      Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https", verify_mode: OpenSSL::SSL::VERIFY_NONE,
-        open_timeout: 2, read_timeout: 3) { |http| http.head(uri.path.empty? ? "/" : uri.path) }
-      true
-    rescue StandardError
-      false
+    def report_instructions
+      { "schema_version" => Report::SCHEMA_VERSION, "results" => Report::RESULTS, "capabilities" => Report::CAPABILITIES,
+        "evidence_paths" => "relative to the directory of the report file, outside the repository" }
     end
   end
 end

data/lib/browser_review_gate/templates/agents_block.md.erb

@@ -1,5 +1,5 @@
 <!-- browser_review_gate:start -->
 ## Browser verification before human review
 
-A PR that changes browser-visible behavior needs a browser run before a human reviews it; CI takes back reviewer requests until one is published. When asked to run `<%= command %>` (with or without a PR number), or to verify a branch or PR in the browser, read `<%= playbook_path %>` and follow it exactly. Right after opening a PR with `gh pr create`, run `browser-review-gate publish` if `browser-review-gate status` says `publish_saved`.
+A PR that changes browser-visible behavior needs a browser run before a human reviews it; CI takes back reviewer requests until one is published. When asked to run `<%= command %>` (with or without a PR number), or to verify a branch or PR in the browser, read `<%= playbook_path %>` and follow it exactly. Right after opening a PR with `gh pr create`, run `browser-review-gate verify`: it publishes a run saved for this commit.
 <!-- browser_review_gate:end -->

data/lib/browser_review_gate/templates/config.yml.erb

@@ -1,10 +1,79 @@
 # Settings for browser_review_gate. Every key is optional.
+#
+# [enforced] settings are checked by the gem: it refuses a report or keeps review on hold.
+# [agent] settings are instructions the AI coding agent receives; the gem cannot make a browser
+# tool take a screenshot, it can only refuse a run without one when `gate.require_evidence` is on.
 
-# Where the running app answers, and how the person starts it. The agent never starts it itself.
-start_command:<%= start_command ? " #{start_command.inspect}" : "" %>
-url:<%= url ? " #{url.inspect}" : "" %>
-# How to sign in locally, e.g. "use the seed account from db/seeds.rb". No real credentials here.
-sign_in:
+# Where the app runs for a browser run. The agent opens `url`; `healthcheck` (a path on `url`, or a
+# full URL) is what `verify`, `status` and `doctor` ask to see whether the app answers. Nobody starts
+# the app automatically: `start_command` is what the person runs. `sign_in` is a hint such as "use the
+# seed account from db/seeds.rb"; never put real credentials here, this file is in the repository.
+# A cloud agent cannot reach localhost: give it an environment with a URL it can reach (staging, a
+# preview deployment) and select it with `--environment`.
+environments:
+  local:
+    url:<%= url ? " #{url.inspect}" : "" %>
+    healthcheck:<%= url ? ' "/up"' : "" %>
+    start_command:<%= start_command ? " #{start_command.inspect}" : "" %>
+    sign_in:
+#  staging:
+#    url: "https://staging.example.com"
+#    healthcheck: "/up"
+
+verification:
+  # The environment used when none is named with --environment.
+  environment: local
+  # [enforced] A case that passes but reports console errors / failed network requests counts as failed.
+  fail_on_console_errors: false
+  fail_on_network_errors: false
+
+evidence:
+  # [agent] never, on_failure or always. [enforced] with gate.require_evidence, for tools that can.
+  screenshots: on_failure
+  trace: never
+  # [agent] Record console errors and failed requests per case. [enforced] with gate.require_evidence.
+  console_logs: true
+  network_logs: true
+  # [enforced] Saved runs and kept evidence files in the git directory are removed after this many days.
+  retention_days: 14
+  # [enforced] Hosts that evidence links may point to, e.g. your artifact bucket. Empty: no links.
+  link_hosts: []
+
+scenarios:
+  # [enforced] The most scenarios the assessing model may ask for (1-30). Mandatory ones come on top.
+  max_count: 12
+  # [enforced] A passing case from an earlier commit of the PR keeps counting after unrelated commits.
+  # false: every scenario must pass on the newest commit.
+  reuse_passing: true
+  # [enforced] Scenarios added to every assessment that needs a run, whatever the model proposes. With
+  # `paths`, only when the change touches them, and then a run is needed whatever the model decides.
+  # `matrix: true` repeats the scenario for every combination below (at most 24 cases in total).
+  mandatory: []
+#    - id: SIGN-IN
+#      name: Sign in and land on the dashboard
+#      why: Every change to authentication must keep sign-in working.
+#      paths: ["app/controllers/sessions_controller.rb", "app/views/sessions/**"]
+#      matrix: true
+  matrix: {}
+#    role: [admin, member]
+#    viewport: [desktop, mobile]
+
+gate:
+  # [enforced] Refuse runs whose passing cases lack the evidence asked for above.
+  require_evidence: false
+  # [enforced] Any failing case keeps review on hold, not only failing required scenarios.
+  block_on_failure: true
+
+provenance:
+  # [enforced] Check each case's Playwright trace against the tested commit. off: no check. warn: the
+  # result is shown in the PR. require: a passing case counts only when its trace shows a response
+  # from the tested commit (via revision_header), a request to the case's page, and request times
+  # inside the run. The app must send the header: add BrowserReviewGate::RevisionHeader to the
+  # development middleware. With warn or require the agent is asked for a trace of every case.
+  trace: "off"
+  revision_header: "X-App-Revision"
+  check_pages: true
+  clock_skew_seconds: 300
 
 # Who assesses a PR in CI. Unset: the Claude Code CLI with a CLAUDE_CODE_OAUTH_TOKEN or
 # ANTHROPIC_API_KEY secret; with only an OPENAI_API_KEY or GEMINI_API_KEY secret, that provider's API.

data/lib/browser_review_gate/templates/playbook.md.erb

@@ -1,56 +1,96 @@
 <!-- browser_review_gate:managed <%= BrowserReviewGate::VERSION %> — re-run `browser-review-gate install` to update. -->
 # Browser verification before human review
 
-A pull request that changes what a person sees or does in a browser must be exercised in a browser before a human reviews it. CI decides whether a PR needs that and writes the decision in one "Browser verification" comment. Until a passing run is published, reviewer requests on such a PR are taken back.
+A pull request that changes what a person sees or does in a browser must be exercised in a browser before a human reviews it. CI decides whether a PR needs that and writes the decision in one "Browser verification" comment, with the scenarios a run must pass. Until a run covering them is published, reviewer requests on such a PR are taken back.
 
 This file is the playbook for the AI coding agent that does the run. The person starts it with `<%= command %> NUMBER` for an open PR, or `<%= command %>` on a branch that has no PR yet.
 
+**Who does what.** You, the agent, drive the browser with your browser tool (Playwright, Chrome DevTools MCP or similar) and write down what you saw. `browser-review-gate` does the bookkeeping: which scenarios are owed, which environment to use, whether the app answers, and checking and publishing your report. It never drives a browser, and it cannot see what you did: your report is your account, so it must be accurate.
+
 ## Steps
 
-1. Run `browser-review-gate status` (add `--pr NUMBER` when a number was given). It prints JSON; follow `next`:
-   - `nothing`: no run is needed, or the PR is already verified. Say that review can be requested.
+1. Run `browser-review-gate verify --json` (add `--pr NUMBER` when a number was given, `--environment NAME` when the person named one). Follow `outcome`:
+   - `nothing`: no run is needed, or the PR is already verified or waived. Say that review can be requested.
    - `check_out_pr_head`: check out the PR head with a clean worktree, then start again.
-   - `request_assessment`: run `browser-review-gate request-assessment NUMBER`, wait about a minute, then start again.
-   - `publish_saved`: a run saved for this commit exists. Run `browser-review-gate publish` and stop; nothing runs again.
+   - `assessment_requested`: CI was asked to assess the PR. Wait about a minute, then start again.
+   - `published` / `not_verified`: a run saved for this commit was published. Report the result and stop.
    - `open_pull_request`: a run is saved and waits for the PR. Tell the person to open it.
-   - `ask_to_start_app`: the app is not running. Do not start it yourself: ask the person to start it (`app.start_command` says how) and stop until they have.
+   - `ask_to_start_app`: the environment does not answer. Do not start it yourself: ask the person to start it (`environment.start_command` says how) and stop until they have.
    - `run_and_publish` or `run_and_save`: continue with step 2.
 2. Decide what to run.
-   - `scenarios` in the status output is the list CI wrote for this PR. Run every one of them, and give each case the `id` of its scenario. You may add cases of your own; you may not drop or rename one from the list.
-   - With no PR yet, or an empty list, choose the scenarios yourself: read the diff against the base branch, then `<%= rules_path %>`, which says what this project always checks and which scenarios each area needs. Its rules are mandatory. CI compares your cases with its own list once the PR is open and asks for what is missing.
-   - Do not re-run behavior an earlier passing case still covers.
-3. Open the running app in a browser tool. `app` in the status output gives the URL and a sign-in hint. Never start, restart or stop the app yourself: what runs on the person's machine is their decision. Use local test data and do not destroy existing data. If the app, the data, or a browser tool is missing, report the blocker. Reading code and running unit tests does not replace a browser run.
-4. Run every scenario end to end. After key actions check the page state, console errors, and failed requests. For each case record the page it was exercised on (`url`) and what you observed there, including console errors and failed requests (`details`). A case without both is refused. A skipped or blocked scenario is not a pass.
-5. Write the report as JSON outside the repository (see the shape below). Set `coverage_complete` to true only when every scenario from step 2 ran. CI checks the report against its list by case `id`: a missing scenario takes the label away again.
-6. Publish or save, from the checkout that was tested, with everything committed and pushed:
-   - PR exists: `browser-review-gate publish --report PATH` (add `--pr NUMBER` when the branch has several). It posts the cases with their outcomes. Only a complete all-pass report adds the `<%= label %>` label.
-   - No PR yet: `browser-review-gate save --report PATH`. The run is published when the PR for this commit is opened. A new commit makes it stale.
-7. Check the command result and the PR. A non-zero exit can still mean a report was posted: a failing or incomplete report is visible in the PR and review stays closed.
-8. Request a reviewer only when the person named one and only after the run passed. Reviewers whose requests CI took back earlier are asked again automatically. Otherwise say that review can be requested.
+   - `scenarios` is the list CI wrote for this PR, with each one's `status` (`missing`, `failed`, `blocked`, `skipped`, `stale`). Run every one of them and give each case the `id` of its scenario. Scenarios marked `critical` or in the `mandatory` category come from the project settings; they cannot be left out. You may add cases of your own; you may not drop or rename one from the list.
+   - With no PR yet, or an empty list, choose the scenarios yourself: read the diff against the base branch, then `<%= rules_path %>`, which says what this project always checks. Its rules are mandatory. CI compares your cases with its own list once the PR is open and asks for what is missing.
+   - Do not re-run a scenario whose earlier case still counts; `verify` lists only what is owed.
+3. Open `environment.url` in your browser tool. `environment.sign_in` is a sign-in hint. Never start, restart or stop the app yourself. Use test data and do not destroy existing data. If the app, the data or a browser tool is missing, report the blocker. Reading code and running unit tests does not replace a browser run.
+4. Run every scenario end to end. For each case record:
+   - `url`: the page it was exercised on; `details`: what you observed;
+   - evidence, as `policy.agent` in the `verify` output asks: screenshots (`never`, `on_failure`, `always`), a trace, and the console errors and failed requests you saw (`[]` when there were none). Save files in a directory outside the repository, next to the report.
+   - When `policy.enforced.trace_check` is `warn` or `require`, record a Playwright trace (`.zip`) or a HAR network log (`.har`, which Playwright and Chrome DevTools can save) for every case, one per case, and list it as `evidence.trace`. If your browser tool can save neither, stop and tell the person: with `require`, every passing case would be refused. The gem opens it and checks that the app answered with the tested commit, that the case's page was requested, and that the requests fall between `started_at` and `finished_at`. With `require`, a passing case without such a trace does not count. Record the trace while running the case; a trace from another run will not match.
+   - A scenario you could not run is `blocked` (something prevented it) or `skipped` (deliberately not run), with a `reason`. Neither is a pass.
+5. Write the report as JSON next to the evidence (see the shape below). Set `coverage_complete` to true only when every scenario from step 2 ran.
+6. Publish, from the checkout that was tested, with everything committed and pushed: `browser-review-gate verify --report PATH` (add `--pr NUMBER` when the branch has several PRs). With a PR it is published; without one it is saved and published when the PR for this commit is opened.
+7. Check the result. Exit status 0 means review can proceed; 7 means the report was published but review stays on hold (a scenario is missing, failed, or broke a project policy); 1 means the report was refused, with the reason. A refused report is not published: fix it and publish again.
+8. Request a reviewer only when the person named one and only after the run passed. Reviewers whose requests CI took back earlier are asked again automatically.
+
+## What the gem checks, and what it cannot
+
+- It refuses a report with an unknown key, a missing field, an evidence path outside the report directory, a file that is not the image or archive its name says, a link to a host the project did not list, or a commit other than the checked-out one.
+- With `gate.require_evidence`, a passing case without the screenshots, trace or logs the project asks for does not count, unless your tool lacks that capability (say so in `tool.capabilities`).
+- With `verification.fail_on_console_errors` / `fail_on_network_errors`, a passing case that lists such errors counts as failed.
+- Only cases tested on a commit of this PR count.
+- With `provenance.trace`, it reads each case's Playwright trace and checks it against the tested commit and the case's page.
+- It cannot tell whether a browser really did what you wrote. File digests show that files existed when the run was published, not who made them.
 
 ## When a scenario cannot be run
 
-Some scenarios cannot be exercised locally, for example a sign-in through an external provider. Do not mark the run complete and do not drop the scenario silently. Tell the person. They can list it under `excluded` in the report, or someone other than the author can put the `<%= waiver_label %>` label on the PR to let review proceed without a run.
+Some scenarios cannot be exercised locally, for example a sign-in through an external provider. Report it as `blocked` with the reason, or, when the person decided so, list it under `excluded`. Neither counts as a pass, so review stays on hold for that scenario: tell the person. Someone other than the author can put the `<%= waiver_label %>` label on the PR to let review proceed without a run.
 
 ## Report shape
 
 ```json
 {
-  "tested_sha": "<full commit SHA that was tested>",
+  "schema_version": 2,
+  "tested_sha": "<full 40-character SHA of the commit that was tested>",
+  "environment": "local",
+  "tool": { "name": "Playwright MCP", "version": "0.0.40", "capabilities": ["screenshots", "trace", "console", "network"] },
+  "started_at": "2026-11-11T10:00:00Z",
+  "finished_at": "2026-11-11T10:06:30Z",
   "coverage_complete": true,
   "cases": [
-    { "id": "BROWSER-1", "name": "The changed interaction completes", "result": "pass", "url": "/the/page/exercised", "details": "What was observed, with console errors and failed requests if any." }
+    {
+      "id": "BROWSER-1",
+      "name": "Submit the sign-up form and land on the dashboard",
+      "result": "pass",
+      "url": "/signup",
+      "details": "The form submitted; the dashboard showed the new account's name.",
+      "evidence": {
+        "screenshots": ["artifacts/signup-done.png"],
+        "trace": "artifacts/signup-trace.zip",
+        "console_errors": [],
+        "network_errors": []
+      }
+    },
+    {
+      "id": "SIGN-IN.member.mobile",
+      "name": "Sign in as a member at phone width",
+      "result": "blocked",
+      "reason": "The member seed account is missing from the local database.",
+      "details": "The sign-in form rejected the seed credentials."
+    }
   ],
   "excluded": [
-    { "name": "Sign-in through an external provider", "reason": "Needs a real provider account; covered by integration tests." }
+    { "id": "OAUTH-1", "name": "Sign-in through an external provider", "reason": "Needs a real provider account; covered by integration tests." }
   ]
 }
 ```
 
-`result` is `pass` or `fail`. `url` is an http(s) URL or a path starting with `/`. `excluded` is optional: list a scenario there only when the person decided it stays outside the local run. Never exclude a scenario on your own to make the run complete. Keep customer and real company names out of the report.
+- `result`: `pass`, `fail`, `blocked` or `skipped`. `url` is an http(s) URL or a path starting with `/`; required for `pass` and `fail`. `reason` is required for `blocked` and `skipped`.
+- `tool.capabilities`: what your browser tool could record, from `screenshots`, `trace`, `console`, `network`. Claim only what it really can.
+- `evidence` paths are relative to the report file's directory. Accepted: `.png`, `.jpg`, `.jpeg`, `.webp` screenshots and a `.zip` Playwright trace or `.har` network log, at most 10 files of 20 MB each. `network_errors` entries may be strings or `{ "method", "url", "status" }`; query strings are dropped. `links` may point to evidence kept elsewhere, on hosts listed in `evidence.link_hosts`.
+- Everything you write is scrubbed of values that look like credentials, but do not rely on it: never put cookies, tokens, passwords or customer data into a report, a screenshot or a trace. Keep customer and real company names out of the report.
 
 ## Rules
 
-- Run the awkward path in the browser, or mark coverage incomplete and leave review closed. Do not declare a run done from tests and code reading.
-- Keep the report file and browser artifacts outside the repository. The publisher refuses a checkout with modified or untracked files and lists them.
-- After an unrelated commit, do not run everything again: use the PR comment and the earlier report to pick only what is new.
+- Run the awkward path in the browser, or report it as blocked and leave review on hold. Do not declare a run done from tests and code reading.
+- Keep the report file and evidence outside the repository. The publisher refuses a checkout with modified or untracked files and lists them.
+- After an unrelated commit, do not run everything again: `verify` lists only what is new or no longer counts.

data/lib/browser_review_gate/templates/rules.md.erb

@@ -16,7 +16,10 @@ one is ignored. The built-in rules stay in force; these are added on top and win
 ## Always check
 
 <!-- Example: Run every scenario once as an administrator and once as a regular member.
-     Example: Repeat the main scenario at a phone-sized window. -->
+     Example: Repeat the main scenario at a phone-sized window.
+     These are read by the model and the agent. To make a scenario or a role/viewport combination
+     impossible to leave out, declare it under `scenarios.mandatory` and `scenarios.matrix` in
+     .github/browser-review-gate.yml: those are added by the gem and checked case by case. -->
 
 ## Scenarios by area
 

data/lib/browser_review_gate/trace_check.rb

@@ -0,0 +1,119 @@
+require "json"
+require "time"
+require "uri"
+
+module BrowserReviewGate
+  # Reads a Playwright trace (trace.zip) or a HAR network log (.har, which Playwright, Chrome DevTools
+  # and other tools write) and checks that it shows the case really ran against the
+  # tested code: a response carried the app's revision header with the tested commit, the case's page
+  # was requested, and the requests fall inside the run's time window.
+  #
+  # This catches an agent that reports a run it did not do. It does not stop someone who builds a
+  # trace by hand; a trace is a file like any other.
+  class TraceCheck
+    NETWORK_ENTRY = /(?:\A|\/)(?:\d+-)?trace\.network\z/
+
+    def initialize(config)
+      @settings = config.provenance
+    end
+
+    # Returns { "status" => "verified" | "failed", "revision" => ..., "requests" => n, "findings" => [...] }.
+    def call(path, tested_sha:, url:, started_at:, finished_at:)
+      requests = read(path)
+      findings = []
+      revisions = requests.filter_map { |request| request[:revision] }.uniq
+      matched = revisions.find { |revision| tested_sha.start_with?(revision) }
+      if revisions.empty?
+        findings << "no response carried the #{@settings["revision_header"]} header"
+      elsif matched.nil?
+        findings << "the app answered with revision #{revisions.first[0, 12]}, not the tested commit"
+      end
+      findings << "the case's page #{url} was not requested" if @settings["check_pages"] && url && !page_requested?(requests, url)
+      findings.concat(time_findings(requests, started_at, finished_at))
+      findings << "the trace has no network requests" if requests.empty?
+
+      { "status" => findings.empty? ? "verified" : "failed", "revision" => matched && tested_sha[0, 12],
+        "requests" => requests.size, "findings" => findings.first(5) }.compact
+    end
+
+    private
+
+    def read(path)
+      return read_har(path) if File.extname(path).casecmp?(".har")
+
+      requests = []
+      ZipReader.new(path).each(->(name) { NETWORK_ENTRY.match?(name) }) do |_name, content|
+        content.each_line do |line|
+          request = parse(line)
+          requests << request if request
+        end
+      end
+      requests
+    end
+
+    def read_har(path)
+      text = File.binread(path, ZipReader::MAX_ENTRY_BYTES + 1).to_s.force_encoding(Encoding::UTF_8).delete_prefix("\uFEFF")
+      log = JSON.parse(text)
+      entries = log.is_a?(Hash) ? log.dig("log", "entries") : nil
+      raise Error, "the HAR file has no log.entries" unless entries.is_a?(Array)
+
+      entries.filter_map { |entry| request(entry) }
+    rescue JSON::ParserError
+      raise Error, "the HAR file is not valid JSON"
+    end
+
+    # One HAR-like resource snapshot per line; anything else is skipped.
+    def parse(line)
+      event = JSON.parse(line)
+      event.is_a?(Hash) && event["type"] == "resource-snapshot" ? request(event["snapshot"]) : nil
+    rescue JSON::ParserError
+      nil
+    end
+
+    # A HAR entry, which is also what a Playwright resource snapshot is.
+    def request(snapshot)
+      return unless snapshot.is_a?(Hash) && snapshot["request"].is_a?(Hash)
+
+      headers = snapshot.dig("response", "headers")
+      header = Array(headers).find { |entry| entry.is_a?(Hash) && entry["name"].to_s.casecmp?(@settings["revision_header"]) }
+      { url: snapshot["request"]["url"].to_s, time: time(snapshot["startedDateTime"]),
+        revision: header && header["value"].to_s.strip.downcase[/\A\h{7,40}\z/] }
+    end
+
+    def time(value)
+      Time.iso8601(value.to_s)
+    rescue ArgumentError
+      nil
+    end
+
+    # The case's page counts as requested when a request has the same path, and the same host when
+    # the case names a full URL.
+    def page_requested?(requests, url)
+      wanted = URI.parse(url)
+      requests.any? do |request|
+        seen = URI.parse(request[:url])
+        seen.path.to_s.chomp("/") == wanted.path.to_s.chomp("/") && (wanted.host.nil? || wanted.host == seen.host)
+      rescue URI::InvalidURIError
+        false
+      end
+    rescue URI::InvalidURIError
+      false
+    end
+
+    def time_findings(requests, started_at, finished_at)
+      return [] unless started_at && finished_at
+
+      skew = @settings["clock_skew_seconds"]
+      from = Time.iso8601(started_at) - skew
+      to = Time.iso8601(finished_at) + skew
+      times = requests.filter_map { |request| request[:time] }
+      return [ "the trace has no request times" ] if times.empty? && requests.any?
+      return [] if times.empty?
+
+      outside = times.count { |seen| seen < from || seen > to }
+      outside.positive? ? [ "#{outside} of #{times.size} requests fall outside the run's started_at..finished_at" ] : []
+    rescue ArgumentError
+      []
+    end
+  end
+end