browser_review_gate 0.2.2 → 0.3.0

data/lib/browser_review_gate/evidence.rb

@@ -0,0 +1,170 @@
+require "digest"
+require "fileutils"
+require "uri"
+
+module BrowserReviewGate
+  # Turns the evidence an agent lists in a submitted report into what is published.
+  #
+  # Files (screenshots, traces) are read from the directory of the report file, checked, hashed and
+  # copied into the git directory; the PR gets their names, sizes and SHA-256 digests, never the
+  # files. A digest shows that a file of that type existed when the run was published. It does not
+  # show that a browser produced it. Console and network entries are the agent's own account; they are
+  # trimmed and scrubbed of anything that looks like a credential.
+  class Evidence
+    MAX_FILE_BYTES = 20 * 1024 * 1024
+    MAX_FILES = 10
+    MAX_ENTRIES = 50
+    MAX_ENTRY = 500
+    KINDS = { "screenshots" => "screenshot", "trace" => "trace" }.freeze
+    # Accepted files by extension, each with the bytes its content must start with.
+    TYPES = {
+      ".png" => [ "image/png", [ "\x89PNG\r\n\x1A\n".b ] ],
+      ".jpg" => [ "image/jpeg", [ "\xFF\xD8\xFF".b ] ],
+      ".jpeg" => [ "image/jpeg", [ "\xFF\xD8\xFF".b ] ],
+      ".webp" => [ "image/webp", [ "RIFF".b ] ],
+      ".zip" => [ "application/zip", [ "PK\x03\x04".b ] ]
+    }.freeze
+    TRACE_TYPES = %w[.zip].freeze
+    SECRET = /
+      (?<key>authorization|proxy-authorization|cookie|set-cookie|x-api-key|api[_-]?key|access[_-]?token|refresh[_-]?token|
+        id[_-]?token|token|secret|password|passwd|session(?:[_-]?id)?|csrf[_-]?token|auth)
+      (?<sep>["']?\s*[:=]\s*["']?)(?:bearer\s+|basic\s+)?[^\s"',;&]+
+    /xi
+    BEARER = /\b(bearer|basic)\s+[A-Za-z0-9._~+\/=-]{8,}/i
+    JWT = /\beyJ[A-Za-z0-9_-]{5,}\.[A-Za-z0-9_-]{5,}\.[A-Za-z0-9_-]{5,}/
+    LONG_TOKEN = /\b[A-Za-z0-9_-]{40,}\b/
+
+    # Masks values that look like credentials. Applied to everything an agent writes into a report.
+    def self.scrub(text)
+      text.to_s.gsub(SECRET) { "#{$~[:key]}#{$~[:sep]}[REDACTED]" }.gsub(BEARER) { "#{$1} [REDACTED]" }
+        .gsub(JWT, "[REDACTED]").gsub(LONG_TOKEN, "[REDACTED]")
+    end
+
+    # Origin and path only: query strings and fragments carry tokens too often to publish.
+    def self.safe_url(value)
+      uri = URI.parse(value.to_s)
+      return scrub(value.to_s.split(/[?#]/).first.to_s)[0, 300] unless uri.scheme
+
+      port = uri.port == uri.default_port ? "" : ":#{uri.port}"
+      scrub("#{uri.scheme}://#{uri.host}#{port}#{uri.path}")[0, 300]
+    rescue URI::InvalidURIError
+      "[unparseable URL]"
+    end
+
+    attr_reader :files
+
+    # `base_dir` is where the report file lives; every evidence path is relative to it.
+    def initialize(base_dir:, config: Config.new)
+      @base_dir = base_dir && File.realpath(base_dir)
+      @config = config
+      @files = []
+    rescue SystemCallError
+      raise Error, "The report directory #{base_dir} cannot be read"
+    end
+
+    # Returns the report data with each case's evidence in its published form. Raises Error naming the
+    # case and the problem when a path, file or entry is not acceptable.
+    def process(data)
+      return data unless data.is_a?(Hash) && data["cases"].is_a?(Array)
+
+      data.merge("cases" => data["cases"].each_with_index.map { |test_case, index| process_case(test_case, index + 1) })
+    end
+
+    # Copies the accepted files under `dir`, by digest, so the run can be looked at later.
+    def keep(dir)
+      files.each do |file|
+        target = File.join(dir, "#{file["sha256"]}#{File.extname(file["name"])}")
+        FileUtils.mkdir_p(dir)
+        FileUtils.cp(file.fetch("source"), target) unless File.exist?(target)
+      end
+    end
+
+    private
+
+    def process_case(test_case, position)
+      return test_case unless test_case.is_a?(Hash) && test_case.key?("evidence")
+
+      evidence = test_case["evidence"]
+      raise Error, "case #{position} evidence must be an object" unless evidence.is_a?(Hash)
+
+      unknown = evidence.keys - %w[screenshots trace console_errors network_errors links]
+      raise Error, "case #{position} evidence has unknown keys: #{unknown.join(", ")}" if unknown.any?
+
+      published = {}
+      listed = Array(evidence["screenshots"]).map { |path| [ "screenshots", path ] }
+      listed << [ "trace", evidence["trace"] ] if evidence["trace"]
+      raise Error, "case #{position} evidence lists more than #{MAX_FILES} files" if listed.size > MAX_FILES
+
+      published["files"] = listed.map { |kind, path| file(kind, path, position) } if listed.any?
+      published["console_errors"] = entries(evidence["console_errors"], position, "console_errors") { |entry| Evidence.scrub(entry) } if evidence.key?("console_errors")
+      published["network_errors"] = entries(evidence["network_errors"], position, "network_errors") { |entry| network(entry) } if evidence.key?("network_errors")
+      published["links"] = links(evidence["links"], position) if evidence.key?("links")
+      test_case.merge("evidence" => published)
+    end
+
+    def file(kind, path, position)
+      raise Error, "case #{position} evidence files need a report file to be relative to" unless @base_dir
+      raise Error, "case #{position} #{kind} path must be a relative path inside the report directory" unless relative?(path)
+
+      full = File.join(@base_dir, path)
+      raise Error, "case #{position} #{kind} file #{path} does not exist" unless File.exist?(full)
+      raise Error, "case #{position} #{kind} file #{path} must be a regular file, not a link" if File.symlink?(full) || !File.file?(full)
+
+      real = File.realpath(full)
+      raise Error, "case #{position} #{kind} file #{path} is outside the report directory" unless real.start_with?("#{@base_dir}/")
+
+      extension = File.extname(real).downcase
+      allowed = kind == "trace" ? TRACE_TYPES : TYPES.keys - TRACE_TYPES
+      raise Error, "case #{position} #{kind} file #{path} must be one of #{allowed.join(", ")}" unless allowed.include?(extension)
+
+      bytes = File.size(real)
+      raise Error, "case #{position} #{kind} file #{path} is larger than #{MAX_FILE_BYTES / 1024 / 1024} MB" if bytes > MAX_FILE_BYTES
+
+      media_type, signatures = TYPES.fetch(extension)
+      head = File.binread(real, 16).to_s
+      raise Error, "case #{position} #{kind} file #{path} is not a #{media_type} file" unless signatures.any? { |signature| head.start_with?(signature) }
+
+      entry = { "kind" => KINDS.fetch(kind), "name" => File.basename(real)[0, 100], "media_type" => media_type,
+                "bytes" => bytes, "sha256" => Digest::SHA256.file(real).hexdigest }
+      @files << entry.merge("source" => real)
+      entry
+    end
+
+    def relative?(path)
+      path.is_a?(String) && !path.empty? && path.length <= 300 && !path.start_with?("/", "~") &&
+        !path.include?("\0") && !path.split(%r{[/\\]}).include?("..")
+    end
+
+    def entries(value, position, key)
+      raise Error, "case #{position} evidence #{key} must be a list" unless value.is_a?(Array)
+      raise Error, "case #{position} evidence #{key} has more than #{MAX_ENTRIES} entries" if value.size > MAX_ENTRIES
+
+      value.map { |entry| yield(entry).to_s[0, MAX_ENTRY] }
+    end
+
+    # A failed request as "METHOD URL -> STATUS", with the URL reduced to origin and path.
+    def network(entry)
+      return Evidence.safe_url(entry) unless entry.is_a?(Hash)
+
+      method = entry["method"].to_s.upcase[/\A[A-Z]{1,10}\z/]
+      status = entry["status"].to_s[/\A\d{1,3}|\A[a-z_:]{1,40}\z/i]
+      [ method, Evidence.safe_url(entry["url"]), status && "-> #{status}" ].compact.join(" ")
+    end
+
+    # Links to evidence kept elsewhere, accepted only on hosts the project listed.
+    def links(value, position)
+      raise Error, "case #{position} evidence links must be a list" unless value.is_a?(Array)
+      raise Error, "case #{position} evidence has more than #{MAX_FILES} links" if value.size > MAX_FILES
+
+      value.map do |link|
+        uri = URI.parse(link.to_s)
+        allowed = uri.scheme == "https" && uri.userinfo.nil? && @config.evidence["link_hosts"].include?(uri.host)
+        raise Error, "case #{position} evidence link #{link.to_s[0, 100]} must be https on a host listed in evidence.link_hosts" unless allowed
+
+        link.to_s[0, 300]
+      rescue URI::InvalidURIError
+        raise Error, "case #{position} evidence link is not a valid URL"
+      end
+    end
+  end
+end

data/lib/browser_review_gate/gate.rb

@@ -39,7 +39,7 @@ module BrowserReviewGate
       return [ :open, "Browser verification waived by #{waived_by}." ] if waived_by
       return [ :closed, "The browser-testing assessment for this commit is not available." ] unless assessment
       return [ :open, "Browser verification is not needed." ] unless assessment.required?
-      return [ :open, "Required browser cases are verified." ] if verified?(pull_request, comments, assessment)
+      return [ :open, "Required browser cases are verified." ] if verified?(number, pull_request, comments, assessment)
 
       [ :closed, "Browser run needed: #{@config.command} #{number}" ]
     end
@@ -48,8 +48,11 @@ module BrowserReviewGate
       pull_request.fetch("labels").any? { |entry| entry["name"] == label }
     end
 
-    def verified?(pull_request, comments, assessment)
-      labelled?(pull_request, @config.label) && assessment.covered_by?(Report.latest(comments))
+    # Only cases tested on a commit of this PR count: a report about other code proves nothing here.
+    def verified?(number, pull_request, comments, assessment)
+      labelled?(pull_request, @config.label) &&
+        assessment.covered_by?(Report.latest(comments, config: @config), commits: @github.pull_request_commits(number),
+          reuse_passing: @config.scenarios["reuse_passing"])
     end
 
     # A waiver counts only from someone other than the author, unless the project allows otherwise.
@@ -146,8 +149,8 @@ module BrowserReviewGate
     def notice(number, assessment, comments)
       return "Review request paused. The browser-testing assessment for this commit is not available. Request review again to retry it." unless assessment
 
-      report = Report.latest_passing(comments)
-      missing = report && !assessment.covered_by?(report) ? "the published browser run does not cover the latest commits" :
+      report = Report.latest(comments, config: @config)
+      missing = report&.passed? && !assessment.covered_by?(report, commits: @github.pull_request_commits(number)) ? "the published browser run does not cover the latest commits" :
                            "no successful local browser run has been published yet"
       "Review request paused. Browser testing is required for this change, and #{missing}. " \
         "Run `#{@config.command} #{number}` in your AI coding agent from the PR branch."

data/lib/browser_review_gate/github.rb

@@ -51,6 +51,13 @@ module BrowserReviewGate
       raise Error, "gh returned output that is not a pull request list"
     end
 
+    # Full SHAs of the PR's commits, oldest first. GitHub lists at most 250.
+    def pull_request_commits(number)
+      pages("repos/#{repository}/pulls/#{number}/commits").map { |commit| commit.fetch("sha") }
+    rescue KeyError
+      raise Error, "gh returned commits without a sha for PR #{number}"
+    end
+
     def files(number)
       pages("repos/#{repository}/pulls/#{number}/files")
     end

data/lib/browser_review_gate/model_api.rb

@@ -24,6 +24,9 @@ module BrowserReviewGate
       raise Error, "Set `model` in #{Config::PATH} to assess with #{provider}" unless @model
     end
 
+    # Token counts of the last call as the provider reported them, or nil. Providers report no cost.
+    attr_reader :last_usage
+
     # Returns the model's answer as JSON text. Raises Error when the call fails or gives no answer.
     def complete(system:, user:, schema:)
       key = ENV[@key_name].to_s
@@ -32,6 +35,7 @@ module BrowserReviewGate
       url, headers, body = send("#{@provider}_request", key, system, user, schema)
       status, text = @transport.call(url, headers, body)
       response = parse(text)
+      @last_usage = usage(response)
       raise Error, "The model gave no answer: HTTP #{status} #{failure(response, text)}" unless status.to_i == 200
 
       answer = parse(send("#{@provider}_answer", response))
@@ -81,6 +85,18 @@ module BrowserReviewGate
       Array(response.dig("candidates", 0, "content", "parts")).filter_map { |part| part["text"] if part.is_a?(Hash) }.join
     end
 
+    def usage(response)
+      input, output = case @provider
+                      when "anthropic" then [ response.dig("usage", "input_tokens"), response.dig("usage", "output_tokens") ]
+                      when "openai" then [ response.dig("usage", "prompt_tokens"), response.dig("usage", "completion_tokens") ]
+                      else [ response.dig("usageMetadata", "promptTokenCount"), response.dig("usageMetadata", "candidatesTokenCount") ]
+                      end
+      return unless input.is_a?(Integer) || output.is_a?(Integer)
+
+      { "provider" => @provider, "model" => (response["model"] || response["modelVersion"] || @model).to_s,
+        "input_tokens" => input, "output_tokens" => output }.compact
+    end
+
     def parse(text)
       data = JSON.parse(text.to_s)
       data.is_a?(Hash) ? data : {}

data/lib/browser_review_gate/model_client.rb

@@ -32,6 +32,9 @@ module BrowserReviewGate
       @runner = runner || method(:run)
     end
 
+    # Token counts and cost of the last call as the CLI reported them, or nil.
+    attr_reader :last_usage
+
     # Returns the model's answer as JSON text. Raises Error when the call fails or gives no answer.
     def complete(system:, user:, schema:)
       argv = @command + [ "--print", "--tools", "", "--setting-sources", "", "--system-prompt", system,
@@ -39,6 +42,7 @@ module BrowserReviewGate
       argv += [ "--model", @model ] if @model
       stdout, stderr, success = @runner.call(environment, argv, user)
       result = parse(stdout)
+      @last_usage = usage(result)
       answer = result["structured_output"] if success && result["is_error"] == false
       return JSON.generate(answer) if answer.is_a?(Hash)
 
@@ -47,6 +51,12 @@ module BrowserReviewGate
 
     private
 
+    def usage(result)
+      tokens = result["usage"].is_a?(Hash) ? result["usage"] : {}
+      { "provider" => "claude-cli", "model" => @model, "input_tokens" => tokens["input_tokens"],
+        "output_tokens" => tokens["output_tokens"], "cost_usd" => result["total_cost_usd"] }.compact.then { |found| found.size > 1 ? found : nil }
+    end
+
     def parse(stdout)
       result = JSON.parse(stdout.to_s)
       result.is_a?(Hash) ? result : {}

data/lib/browser_review_gate/prompts.rb

@@ -30,10 +30,10 @@ module BrowserReviewGate
       text(RULES)
     end
 
-    def assessment
+    def assessment(max_count: 12)
       parts = [ FRAME_HEAD, "## Decision\n\n#{policy}" ]
       parts << "## Project rules\n\n#{PROJECT_RULES_NOTE}\n\n#{rules}" if rules
-      parts << FRAME_TAIL
+      parts << format(FRAME_TAIL, max_count: max_count)
       parts.join("\n\n")
     end
 
@@ -58,9 +58,14 @@ module BrowserReviewGate
 
       When the decision is `required`, `scenarios` lists what a person must do in the browser to exercise
       every behavior this pull request changes, including what the project rules require for the areas
-      it touches. One entry per scenario, at most 12, the fewest that cover the change. `name` says what
-      to do and what must be seen, in one sentence. `id` is a short label such as `SIGN-IN-1`: letters,
-      digits and hyphens.
+      it touches. One entry per scenario, at most %<max_count>d, the fewest that cover the change. `name` says
+      what to do and what must be seen, in one sentence. `id` is a short label such as `SIGN-IN-1`:
+      letters, digits and hyphens. Derive the id from the behavior, not from its position, so the same
+      behavior gets the same id when the pull request changes. `priority` is `critical` when a failure
+      would block people from a core task or expose data, otherwise `normal`. `category` is one short
+      lowercase word for the area, such as `forms`, `navigation` or `permissions`. `why` is one sentence
+      saying which change makes this scenario necessary. Scenarios the project settings make mandatory
+      are added after you answer; do not list them.
 
       When `report_cases` is present, those are the cases of an earlier browser run. If one of them
       already exercises a scenario, give that scenario the `id` of that case, exactly. Give every other

data/lib/browser_review_gate/publisher.rb

@@ -27,6 +27,7 @@ module BrowserReviewGate
       saved_report = SavedReport.new(sha: local_sha, shell: @shell)
       data = load_report(saved_report)
       ensure_report_sha!(data, local_sha)
+      ensure_environment!(data)
       actor = @github.login
       data["verified_by"] = "AI browser agent (run by #{actor})"
 
@@ -37,9 +38,10 @@ module BrowserReviewGate
       complete = report.passed?
       labelled = pull_request.fetch("labels").any? { |label| label["name"] == @config.label }
       @github.remove_label(number, @config.label) if labelled && !complete
-      write_comment(number, comments, actor, body(report, complete))
+      write_comment(number, comments, actor, body(report, complete, @github.pull_request_commits(number)))
       @github.add_label(number, @config.label) if complete
       saved_report.delete unless @report_path
+      SavedReport.prune(shell: @shell, days: @config.evidence["retention_days"])
       complete
     rescue KeyError => error
       raise Error, "Browser verification data is incomplete: #{error.message}"
@@ -53,11 +55,13 @@ module BrowserReviewGate
       ensure_clean_worktree!
       data = read_report_file
       ensure_report_sha!(data, local_sha)
-      report = Report.new(data.merge("verified_by" => "pending"))
+      ensure_environment!(data)
+      report = Report.new(data.merge("verified_by" => "pending"), config: @config)
       ensure_acceptable!(report)
 
       saved_report = SavedReport.new(sha: local_sha, shell: @shell)
       saved_report.write(data)
+      SavedReport.prune(shell: @shell, days: @config.evidence["retention_days"])
       saved_report.path
     rescue JSON::ParserError => error
       raise Error, "Browser verification data is not valid JSON: #{error.message}"
@@ -89,14 +93,28 @@ module BrowserReviewGate
       raise Error, "No saved browser verification report for the current commit"
     end
 
+    # Evidence files are read here, next to the report, and kept in the git directory by digest.
     def read_report_file
       raise Error, "A report file is required" unless @report_path
 
-      JSON.parse(File.read(@report_path))
+      data = JSON.parse(File.read(@report_path))
+      evidence = Evidence.new(base_dir: File.dirname(File.expand_path(@report_path)), config: @config)
+      processed = evidence.process(data)
+      evidence.keep(SavedReport.evidence_dir(shell: @shell))
+      processed
     rescue SystemCallError => error
       raise Error, "Cannot read report file #{@report_path}: #{error.message}"
     end
 
+    # A run names the environment it used; with environments configured it must be one of them.
+    def ensure_environment!(data)
+      name = data["environment"]
+      return if @config.environments.empty? ? Config::NAME_PATTERN.match?(name.to_s) : @config.environments.key?(name)
+
+      known = @config.environments.keys
+      raise Error, "Report environment must be #{known.empty? ? "a short lowercase name" : "one of #{known.join(", ")}"}, not #{name.inspect}"
+    end
+
     # A report made before the PR existed may arrive before the gate has assessed the commit.
     def ensure_browser_run_wanted!(number, comments, tested_sha)
       assessment = Assessment.latest(comments)
@@ -106,11 +124,16 @@ module BrowserReviewGate
     end
 
     def merged_report(data, comments)
-      submitted = Report.new(data)
+      submitted = Report.new(data, config: @config)
       ensure_acceptable!(submitted)
 
       prior = Report.latest_comment(comments)
-      report = Report.merge(prior ? Report.data_from(prior["body"]) : {}, submitted.to_h)
+      prior_data = begin
+        prior ? Report.data_from(prior["body"]) : {}
+      rescue Error
+        {}
+      end
+      report = Report.merge(prior_data, submitted.to_h, config: @config)
       raise Error, report.errors.join("; ") unless report.valid?
 
       report
@@ -118,15 +141,15 @@ module BrowserReviewGate
 
     # A run is taken only with the evidence of each case: the page and what was observed there.
     def ensure_acceptable!(report)
-      problems = report.valid? ? report.evidence_errors : report.errors
+      problems = report.valid? ? report.submission_errors : report.errors
       raise Error, problems.join("; ") if problems.any?
     end
 
-    def body(report, complete)
-      return report.markdown if complete
+    def body(report, complete, commits)
+      return report.markdown(commits: commits) if complete
 
       [ "**Not verified.** Complete every required browser scenario and pass all reported cases before human review.",
-        "", report.markdown ].join("\n")
+        "", report.markdown(commits: commits) ].join("\n")
     end
 
     # Only the author's own earlier report comment can be edited in place.