browser_review_gate 0.2.2 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +32 -0
- data/README.md +134 -62
- data/lib/browser_review_gate/assessment.rb +74 -33
- data/lib/browser_review_gate/assessor.rb +38 -10
- data/lib/browser_review_gate/cli.rb +44 -19
- data/lib/browser_review_gate/config.rb +223 -10
- data/lib/browser_review_gate/doctor.rb +216 -0
- data/lib/browser_review_gate/evidence.rb +170 -0
- data/lib/browser_review_gate/gate.rb +8 -5
- data/lib/browser_review_gate/github.rb +7 -0
- data/lib/browser_review_gate/model_api.rb +16 -0
- data/lib/browser_review_gate/model_client.rb +10 -0
- data/lib/browser_review_gate/prompts.rb +10 -5
- data/lib/browser_review_gate/publisher.rb +32 -9
- data/lib/browser_review_gate/report.rb +222 -47
- data/lib/browser_review_gate/saved_report.rb +15 -1
- data/lib/browser_review_gate/stats.rb +90 -6
- data/lib/browser_review_gate/status.rb +52 -21
- data/lib/browser_review_gate/templates/agents_block.md.erb +1 -1
- data/lib/browser_review_gate/templates/config.yml.erb +63 -5
- data/lib/browser_review_gate/templates/playbook.md.erb +63 -25
- data/lib/browser_review_gate/templates/rules.md.erb +4 -1
- data/lib/browser_review_gate/verify.rb +86 -0
- data/lib/browser_review_gate/version.rb +1 -1
- data/lib/browser_review_gate.rb +3 -0
- metadata +4 -1
|
@@ -11,7 +11,9 @@ module BrowserReviewGate
|
|
|
11
11
|
eject policy|playbook copy a built-in text into the project to rewrite it there
|
|
12
12
|
|
|
13
13
|
For an AI coding agent, from the checkout that was tested
|
|
14
|
-
|
|
14
|
+
verify [--pr NUMBER] [--environment NAME] [--report PATH] [--no-request] [--json]
|
|
15
|
+
what to run, and with --report publish (or save) the finished run
|
|
16
|
+
status [--pr NUMBER] [--environment NAME] what the PR needs and what to do next, as JSON
|
|
15
17
|
publish [--pr NUMBER] [--report PATH] publish observed results (default: the saved run)
|
|
16
18
|
save --report PATH keep results until the PR for this commit is opened
|
|
17
19
|
request-assessment NUMBER ask CI to assess a PR that has no decision yet
|
|
@@ -21,11 +23,13 @@ module BrowserReviewGate
|
|
|
21
23
|
ci NUMBER assess the PR and take back premature reviewer requests
|
|
22
24
|
|
|
23
25
|
For the team
|
|
24
|
-
|
|
26
|
+
doctor [--json] check the setup and say what to fix
|
|
27
|
+
stats [--days N] [--json] what the gate did on PRs of the last N days (default 30)
|
|
25
28
|
TEXT
|
|
26
29
|
|
|
27
|
-
# `github`, `config` and `
|
|
28
|
-
def initialize(argv, stdin: $stdin, stdout: $stdout, stderr: $stderr, github: nil, config: nil, model: nil)
|
|
30
|
+
# `github`, `config`, `model` and `shell` are replaced in tests.
|
|
31
|
+
def initialize(argv, stdin: $stdin, stdout: $stdout, stderr: $stderr, github: nil, config: nil, model: nil, shell: nil)
|
|
32
|
+
@shell = shell
|
|
29
33
|
@argv = argv.dup
|
|
30
34
|
@github = github
|
|
31
35
|
@config = config
|
|
@@ -42,6 +46,8 @@ module BrowserReviewGate
|
|
|
42
46
|
when "install" then install
|
|
43
47
|
when "eject" then eject
|
|
44
48
|
when "status" then status
|
|
49
|
+
when "verify" then verify
|
|
50
|
+
when "doctor" then doctor
|
|
45
51
|
when "publish" then publish
|
|
46
52
|
when "save" then save
|
|
47
53
|
when "request-assessment" then request_assessment
|
|
@@ -58,7 +64,10 @@ module BrowserReviewGate
|
|
|
58
64
|
|
|
59
65
|
private
|
|
60
66
|
|
|
61
|
-
|
|
67
|
+
# Local commands read the settings from the default branch when it has them; see Config.load_trusted.
|
|
68
|
+
def config = @config ||= trusted_config.first
|
|
69
|
+
def settings_source = @config ? trusted_config.last : nil
|
|
70
|
+
def trusted_config = @trusted_config ||= (@config ? [ @config, "given" ] : Config.load_trusted(root))
|
|
62
71
|
|
|
63
72
|
# Settings and rules live at the top of the repository, wherever the command is run from.
|
|
64
73
|
def root
|
|
@@ -74,6 +83,9 @@ module BrowserReviewGate
|
|
|
74
83
|
parser.on("--pr NUMBER", Integer) { |value| parsed[:pr] = value } if flags.include?(:pr)
|
|
75
84
|
parser.on("--report PATH") { |value| parsed[:report] = value } if flags.include?(:report)
|
|
76
85
|
parser.on("--days N", Integer) { |value| parsed[:days] = value } if flags.include?(:days)
|
|
86
|
+
parser.on("--environment NAME") { |value| parsed[:environment] = value } if flags.include?(:environment)
|
|
87
|
+
parser.on("--json") { parsed[:json] = true } if flags.include?(:json)
|
|
88
|
+
parser.on("--no-request") { parsed[:no_request] = true } if flags.include?(:no_request)
|
|
77
89
|
parser.on("--agents LIST", Array) { |value| parsed[:agents] = value } if flags.include?(:install)
|
|
78
90
|
parser.on("--gem-source SOURCE") { |value| parsed[:gem_source] = value } if flags.include?(:install)
|
|
79
91
|
parser.on("--workflow NAME") { |value| parsed[:workflow] = value } if flags.include?(:install)
|
|
@@ -101,10 +113,32 @@ module BrowserReviewGate
|
|
|
101
113
|
end
|
|
102
114
|
|
|
103
115
|
def status
|
|
104
|
-
|
|
116
|
+
parsed = options(:pr, :environment)
|
|
117
|
+
result = Status.new(github: github, config: config, prompts: Prompts.new(root)).to_h(parsed[:pr], environment: parsed[:environment])
|
|
118
|
+
@stdout.puts JSON.pretty_generate(result.merge("settings_source" => trusted_config.last))
|
|
105
119
|
0
|
|
106
120
|
end
|
|
107
121
|
|
|
122
|
+
def verify
|
|
123
|
+
parsed = options(:pr, :environment, :report, :json, :no_request)
|
|
124
|
+
outcome, result = Verify.new(github: github, config: config, prompts: Prompts.new(root), shell: @shell || Shell.new)
|
|
125
|
+
.run(number: parsed[:pr], environment: parsed[:environment], report: parsed[:report], request: !parsed[:no_request])
|
|
126
|
+
message = Verify.describe(outcome, result)
|
|
127
|
+
if parsed[:json]
|
|
128
|
+
@stdout.puts JSON.pretty_generate(result.merge("outcome" => outcome, "message" => message, "settings_source" => trusted_config.last))
|
|
129
|
+
else
|
|
130
|
+
@stdout.puts message
|
|
131
|
+
end
|
|
132
|
+
Verify.exit_status(outcome)
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
def doctor
|
|
136
|
+
json = options(:json)[:json]
|
|
137
|
+
checks = Doctor.new(root: root, github: @github).checks
|
|
138
|
+
@stdout.puts(json ? Doctor.json(checks) : Doctor.text(checks))
|
|
139
|
+
checks.any? { |check| check.status == "FAIL" } ? 1 : 0
|
|
140
|
+
end
|
|
141
|
+
|
|
108
142
|
def publish
|
|
109
143
|
parsed = options(:pr, :report)
|
|
110
144
|
if Publisher.new(github: github, number: parsed[:pr], report_path: parsed[:report], config: config).publish
|
|
@@ -130,25 +164,16 @@ module BrowserReviewGate
|
|
|
130
164
|
end
|
|
131
165
|
|
|
132
166
|
def stats
|
|
133
|
-
|
|
167
|
+
parsed = options(:days, :json)
|
|
168
|
+
days = parsed.fetch(:days, 30)
|
|
134
169
|
raise Error, "--days must be a positive number" unless days.positive?
|
|
135
170
|
|
|
136
|
-
|
|
137
|
-
@stdout.puts
|
|
138
|
-
" assessed by the gate: #{counts["assessed"]}",
|
|
139
|
-
" needed a browser run: #{counts["required"]}",
|
|
140
|
-
" verified by a published run: #{counts["verified"]}",
|
|
141
|
-
" waived: #{counts["waived"]}",
|
|
142
|
-
" open and still owing a run: #{counts["owed"]}",
|
|
143
|
-
" a run found a failing case: #{counts["found_a_failure"]}"
|
|
171
|
+
result = Stats.new(github: github, config: config).to_h(since: (Time.now.utc - days * 86_400).strftime("%Y-%m-%d"))
|
|
172
|
+
@stdout.puts(parsed[:json] ? JSON.pretty_generate(result) : Stats.text(result, days))
|
|
144
173
|
0
|
|
145
174
|
end
|
|
146
175
|
|
|
147
|
-
# Version 0.1.1 installed a `--before` hook that held commands back. It is gone; a project that
|
|
148
|
-
# still has the entry gets a silent no-op until the installer removes it.
|
|
149
176
|
def hook
|
|
150
|
-
return 0 if @argv.delete("--before")
|
|
151
|
-
|
|
152
177
|
agent = @argv.shift
|
|
153
178
|
message = Hook.new(publisher_factory: -> { Publisher.new(github: github, config: config) }).after(@stdin.read)
|
|
154
179
|
output = Hook.render(message, agent)
|
|
@@ -1,9 +1,18 @@
|
|
|
1
|
+
require "uri"
|
|
1
2
|
require "yaml"
|
|
2
3
|
|
|
3
4
|
module BrowserReviewGate
|
|
4
5
|
# Project settings from .github/browser-review-gate.yml. Every key is optional.
|
|
6
|
+
#
|
|
7
|
+
# Some settings are enforced by this gem (it refuses a report or keeps review closed); others are
|
|
8
|
+
# instructions the agent receives through `status` and `verify`. `ENFORCED` and `AGENT` say which.
|
|
5
9
|
class Config
|
|
6
10
|
PATH = ".github/browser-review-gate.yml"
|
|
11
|
+
NAME_PATTERN = /\A[a-z][a-z0-9_-]{0,31}\z/
|
|
12
|
+
EVIDENCE_MODES = %w[never on_failure always].freeze
|
|
13
|
+
MAX_SCENARIOS = 30
|
|
14
|
+
MAX_MATRIX_CASES = 24
|
|
15
|
+
|
|
7
16
|
DEFAULTS = {
|
|
8
17
|
"label" => "browser-verified",
|
|
9
18
|
# A person other than the author adds this label to let review proceed without a browser run.
|
|
@@ -16,19 +25,45 @@ module BrowserReviewGate
|
|
|
16
25
|
"provider" => nil,
|
|
17
26
|
"model" => nil,
|
|
18
27
|
"claude_command" => nil,
|
|
19
|
-
|
|
20
|
-
"
|
|
21
|
-
"
|
|
22
|
-
"
|
|
28
|
+
"environments" => nil,
|
|
29
|
+
"verification" => nil,
|
|
30
|
+
"evidence" => nil,
|
|
31
|
+
"scenarios" => nil,
|
|
32
|
+
"gate" => nil,
|
|
23
33
|
"skip_authors" => [ "dependabot[bot]", "github-actions[bot]" ],
|
|
24
34
|
"skip_branch_prefixes" => [],
|
|
25
35
|
# A PR that only touches these paths cannot change browser behavior; no model call is made.
|
|
26
36
|
"ignore_paths" => [ "docs/**", "test/**", "spec/**", "**/*.md", ".github/**" ]
|
|
27
37
|
}.freeze
|
|
28
38
|
|
|
39
|
+
SECTIONS = {
|
|
40
|
+
"verification" => { "environment" => nil, "fail_on_console_errors" => false, "fail_on_network_errors" => false },
|
|
41
|
+
"evidence" => { "screenshots" => "never", "trace" => "never", "console_logs" => false, "network_logs" => false,
|
|
42
|
+
"retention_days" => 14, "link_hosts" => [] },
|
|
43
|
+
"scenarios" => { "max_count" => 12, "reuse_passing" => true, "mandatory" => [], "matrix" => {} },
|
|
44
|
+
"gate" => { "require_evidence" => false, "block_on_failure" => true }
|
|
45
|
+
}.freeze
|
|
46
|
+
|
|
47
|
+
ENVIRONMENT_KEYS = %w[url healthcheck start_command sign_in].freeze
|
|
48
|
+
MANDATORY_KEYS = %w[id name why paths matrix].freeze
|
|
49
|
+
|
|
29
50
|
def self.load(root = Dir.pwd)
|
|
30
51
|
path = File.join(root, PATH)
|
|
31
|
-
|
|
52
|
+
parse(File.exist?(path) ? File.read(path) : "")
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
# Settings as the default branch has them, for local commands run on a PR branch: a pull request
|
|
56
|
+
# must not point the agent at another URL or loosen the policy for its own run. Falls back to the
|
|
57
|
+
# checkout when the default branch has no settings yet. Returns [config, source].
|
|
58
|
+
def self.load_trusted(root = Dir.pwd, shell: Shell.new)
|
|
59
|
+
text = shell.call("git", "-C", root, "show", "origin/HEAD:#{PATH}")
|
|
60
|
+
[ parse(text), "origin/HEAD" ]
|
|
61
|
+
rescue Error
|
|
62
|
+
[ load(root), "working tree" ]
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
def self.parse(text)
|
|
66
|
+
new(text.to_s.strip.empty? ? {} : YAML.safe_load(text))
|
|
32
67
|
rescue Psych::Exception => error
|
|
33
68
|
raise Error, "#{PATH} is not valid YAML: #{error.message}"
|
|
34
69
|
end
|
|
@@ -36,15 +71,34 @@ module BrowserReviewGate
|
|
|
36
71
|
def initialize(data = {})
|
|
37
72
|
data = {} unless data.is_a?(Hash)
|
|
38
73
|
unknown = data.keys - DEFAULTS.keys
|
|
74
|
+
moved = unknown & %w[url start_command sign_in]
|
|
75
|
+
raise Error, "#{PATH}: #{moved.join(", ")} moved under environments.local in 0.3.0" if moved.any?
|
|
39
76
|
raise Error, "#{PATH} has unknown keys: #{unknown.join(", ")}" if unknown.any?
|
|
40
77
|
|
|
41
78
|
@data = DEFAULTS.merge(data.compact)
|
|
79
|
+
@sections = SECTIONS.to_h { |name, defaults| [ name, section(name, defaults) ] }
|
|
80
|
+
@environments = read_environments
|
|
81
|
+
validate
|
|
42
82
|
end
|
|
43
83
|
|
|
44
|
-
DEFAULTS.
|
|
84
|
+
(DEFAULTS.keys - %w[environments verification evidence scenarios gate]).each do |key|
|
|
85
|
+
define_method(key) { @data[key] }
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
attr_reader :environments
|
|
89
|
+
|
|
90
|
+
def verification = @sections["verification"]
|
|
91
|
+
def evidence = @sections["evidence"]
|
|
92
|
+
def scenarios = @sections["scenarios"]
|
|
93
|
+
def gate = @sections["gate"]
|
|
94
|
+
|
|
95
|
+
# The environment a run uses: the one named, the configured default, or the only one there is.
|
|
96
|
+
def environment(name = nil)
|
|
97
|
+
name ||= verification["environment"] || (environments.size == 1 ? environments.keys.first : nil)
|
|
98
|
+
return if name.nil?
|
|
99
|
+
raise Error, "Unknown environment #{name.inspect}; #{PATH} defines: #{environments.keys.join(", ")}" unless environments.key?(name)
|
|
45
100
|
|
|
46
|
-
|
|
47
|
-
{ "start_command" => start_command, "url" => url, "sign_in" => sign_in }.compact
|
|
101
|
+
environments.fetch(name).merge("name" => name)
|
|
48
102
|
end
|
|
49
103
|
|
|
50
104
|
def skip?(author:, branch:)
|
|
@@ -52,12 +106,171 @@ module BrowserReviewGate
|
|
|
52
106
|
end
|
|
53
107
|
|
|
54
108
|
# "dir/**" means everything under dir; other patterns are globs where "**/" spans directories.
|
|
55
|
-
def ignored?(path)
|
|
56
|
-
|
|
109
|
+
def ignored?(path) = Config.match?(ignore_paths, path)
|
|
110
|
+
|
|
111
|
+
def self.match?(patterns, path)
|
|
112
|
+
patterns.any? do |pattern|
|
|
57
113
|
next path.start_with?(pattern.delete_suffix("**")) if pattern.end_with?("/**")
|
|
58
114
|
|
|
59
115
|
File.fnmatch?(pattern, path, File::FNM_PATHNAME | File::FNM_DOTMATCH)
|
|
60
116
|
end
|
|
61
117
|
end
|
|
118
|
+
|
|
119
|
+
# Mandatory scenarios that apply to a change touching `paths`, expanded over the matrix where asked.
|
|
120
|
+
# They are added by this code, not proposed by the model, so a PR cannot talk its way out of them.
|
|
121
|
+
def mandatory_scenarios(paths)
|
|
122
|
+
scenarios["mandatory"].select { |entry| entry["paths"].nil? || paths.any? { |path| Config.match?(entry["paths"], path) } }
|
|
123
|
+
.flat_map { |entry| expand(entry) }
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
# What the settings ask of a run, split by who makes sure it happens.
|
|
127
|
+
def policy
|
|
128
|
+
{
|
|
129
|
+
"enforced" => {
|
|
130
|
+
"fail_on_console_errors" => verification["fail_on_console_errors"],
|
|
131
|
+
"fail_on_network_errors" => verification["fail_on_network_errors"],
|
|
132
|
+
"require_evidence" => gate["require_evidence"],
|
|
133
|
+
"block_on_failure" => gate["block_on_failure"],
|
|
134
|
+
"reuse_passing" => scenarios["reuse_passing"],
|
|
135
|
+
"max_scenarios" => scenarios["max_count"]
|
|
136
|
+
},
|
|
137
|
+
"agent" => {
|
|
138
|
+
"screenshots" => evidence["screenshots"], "trace" => evidence["trace"],
|
|
139
|
+
"console_logs" => evidence["console_logs"], "network_logs" => evidence["network_logs"]
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
end
|
|
143
|
+
|
|
144
|
+
private
|
|
145
|
+
|
|
146
|
+
def section(name, defaults)
|
|
147
|
+
value = @data[name] || {}
|
|
148
|
+
raise Error, "#{PATH}: #{name} must be a mapping" unless value.is_a?(Hash)
|
|
149
|
+
|
|
150
|
+
unknown = value.keys - defaults.keys
|
|
151
|
+
raise Error, "#{PATH}: #{name} has unknown keys: #{unknown.join(", ")}" if unknown.any?
|
|
152
|
+
|
|
153
|
+
defaults.merge(value.compact)
|
|
154
|
+
end
|
|
155
|
+
|
|
156
|
+
def read_environments
|
|
157
|
+
listed = @data["environments"]
|
|
158
|
+
return {} unless listed
|
|
159
|
+
raise Error, "#{PATH}: environments must be a mapping of names to settings" unless listed.is_a?(Hash)
|
|
160
|
+
|
|
161
|
+
listed.to_h do |name, settings|
|
|
162
|
+
raise Error, "#{PATH}: environment name #{name.inspect} must be lowercase letters, digits, - or _" unless name.is_a?(String) && NAME_PATTERN.match?(name)
|
|
163
|
+
raise Error, "#{PATH}: environments.#{name} must be a mapping" unless settings.is_a?(Hash)
|
|
164
|
+
|
|
165
|
+
unknown = settings.keys - ENVIRONMENT_KEYS
|
|
166
|
+
raise Error, "#{PATH}: environments.#{name} has unknown keys: #{unknown.join(", ")}" if unknown.any?
|
|
167
|
+
|
|
168
|
+
[ name, settings.compact ]
|
|
169
|
+
end
|
|
170
|
+
end
|
|
171
|
+
|
|
172
|
+
def validate
|
|
173
|
+
environments.each do |name, settings|
|
|
174
|
+
url = settings["url"]
|
|
175
|
+
raise Error, "#{PATH}: environments.#{name}.url must be an http(s) URL" unless url.nil? || Config.http_url?(url)
|
|
176
|
+
|
|
177
|
+
check = settings["healthcheck"]
|
|
178
|
+
unless check.nil? || (check.is_a?(String) && (check.start_with?("/") || Config.http_url?(check)))
|
|
179
|
+
raise Error, "#{PATH}: environments.#{name}.healthcheck must be a path starting with / or an http(s) URL"
|
|
180
|
+
end
|
|
181
|
+
end
|
|
182
|
+
environment # the configured default must exist
|
|
183
|
+
validate_flags
|
|
184
|
+
validate_evidence
|
|
185
|
+
validate_scenarios
|
|
186
|
+
end
|
|
187
|
+
|
|
188
|
+
def validate_flags
|
|
189
|
+
{ "verification" => %w[fail_on_console_errors fail_on_network_errors], "evidence" => %w[console_logs network_logs],
|
|
190
|
+
"scenarios" => %w[reuse_passing], "gate" => %w[require_evidence block_on_failure] }.each do |name, keys|
|
|
191
|
+
keys.each do |key|
|
|
192
|
+
raise Error, "#{PATH}: #{name}.#{key} must be true or false" unless [ true, false ].include?(@sections[name][key])
|
|
193
|
+
end
|
|
194
|
+
end
|
|
195
|
+
end
|
|
196
|
+
|
|
197
|
+
def validate_evidence
|
|
198
|
+
%w[screenshots trace].each do |key|
|
|
199
|
+
next if EVIDENCE_MODES.include?(evidence[key])
|
|
200
|
+
|
|
201
|
+
raise Error, "#{PATH}: evidence.#{key} must be one of #{EVIDENCE_MODES.join(", ")}"
|
|
202
|
+
end
|
|
203
|
+
days = evidence["retention_days"]
|
|
204
|
+
raise Error, "#{PATH}: evidence.retention_days must be a whole number from 1 to 365" unless days.is_a?(Integer) && days.between?(1, 365)
|
|
205
|
+
|
|
206
|
+
hosts = evidence["link_hosts"]
|
|
207
|
+
return if hosts.is_a?(Array) && hosts.all? { |host| host.is_a?(String) && host.match?(/\A[a-z0-9.-]+\z/) }
|
|
208
|
+
|
|
209
|
+
raise Error, "#{PATH}: evidence.link_hosts must be a list of host names"
|
|
210
|
+
end
|
|
211
|
+
|
|
212
|
+
def validate_scenarios
|
|
213
|
+
count = scenarios["max_count"]
|
|
214
|
+
raise Error, "#{PATH}: scenarios.max_count must be a whole number from 1 to #{MAX_SCENARIOS}" unless count.is_a?(Integer) && count.between?(1, MAX_SCENARIOS)
|
|
215
|
+
|
|
216
|
+
validate_matrix
|
|
217
|
+
mandatory = scenarios["mandatory"]
|
|
218
|
+
raise Error, "#{PATH}: scenarios.mandatory must be a list" unless mandatory.is_a?(Array)
|
|
219
|
+
|
|
220
|
+
mandatory.each_with_index { |entry, index| validate_mandatory(entry, index + 1) }
|
|
221
|
+
ids = mandatory_scenarios_all.map { |scenario| scenario["id"] }
|
|
222
|
+
raise Error, "#{PATH}: scenarios.mandatory ids must be unique after the matrix is applied" unless ids.uniq.size == ids.size
|
|
223
|
+
|
|
224
|
+
expanded = mandatory.count { |entry| entry["matrix"] } * matrix_combinations.size
|
|
225
|
+
raise Error, "#{PATH}: the matrix makes #{expanded} mandatory cases; keep it at #{MAX_MATRIX_CASES} or fewer" if expanded > MAX_MATRIX_CASES
|
|
226
|
+
end
|
|
227
|
+
|
|
228
|
+
def validate_matrix
|
|
229
|
+
matrix = scenarios["matrix"]
|
|
230
|
+
valid = matrix.is_a?(Hash) && matrix.all? do |name, values|
|
|
231
|
+
NAME_PATTERN.match?(name.to_s) && values.is_a?(Array) && values.any? && values.all? { |value| NAME_PATTERN.match?(value.to_s) }
|
|
232
|
+
end
|
|
233
|
+
raise Error, "#{PATH}: scenarios.matrix maps a dimension (e.g. role) to a list of short lowercase values" unless valid
|
|
234
|
+
end
|
|
235
|
+
|
|
236
|
+
def validate_mandatory(entry, position)
|
|
237
|
+
raise Error, "#{PATH}: scenarios.mandatory #{position} must be a mapping" unless entry.is_a?(Hash)
|
|
238
|
+
|
|
239
|
+
unknown = entry.keys - MANDATORY_KEYS
|
|
240
|
+
raise Error, "#{PATH}: scenarios.mandatory #{position} has unknown keys: #{unknown.join(", ")}" if unknown.any?
|
|
241
|
+
raise Error, "#{PATH}: scenarios.mandatory #{position} id must use letters, digits, dots, underscores or hyphens" unless entry["id"].is_a?(String) && Report::CASE_ID_PATTERN.match?(entry["id"])
|
|
242
|
+
raise Error, "#{PATH}: scenarios.mandatory #{position} needs a name" unless entry["name"].is_a?(String) && !entry["name"].strip.empty?
|
|
243
|
+
raise Error, "#{PATH}: scenarios.mandatory #{position} paths must be a list of globs" unless entry["paths"].nil? || (entry["paths"].is_a?(Array) && entry["paths"].all?(String))
|
|
244
|
+
raise Error, "#{PATH}: scenarios.mandatory #{position} matrix must be true or false" unless [ nil, true, false ].include?(entry["matrix"])
|
|
245
|
+
end
|
|
246
|
+
|
|
247
|
+
def mandatory_scenarios_all = scenarios["mandatory"].flat_map { |entry| expand(entry) }
|
|
248
|
+
|
|
249
|
+
# Every combination of the matrix dimensions, e.g. [{"role"=>"admin","viewport"=>"mobile"}, ...].
|
|
250
|
+
def matrix_combinations
|
|
251
|
+
matrix = scenarios["matrix"]
|
|
252
|
+
return [ {} ] if matrix.empty?
|
|
253
|
+
|
|
254
|
+
matrix.values.first.product(*matrix.values.drop(1)).map { |values| matrix.keys.zip(values).to_h }
|
|
255
|
+
end
|
|
256
|
+
|
|
257
|
+
def expand(entry)
|
|
258
|
+
base = { "id" => entry["id"], "name" => entry["name"].split.join(" "), "priority" => "critical", "category" => "mandatory",
|
|
259
|
+
"why" => entry["why"] || "Required by the project settings for the areas this change touches." }
|
|
260
|
+
return [ base ] unless entry["matrix"]
|
|
261
|
+
|
|
262
|
+
matrix_combinations.map do |combination|
|
|
263
|
+
suffix = combination.values.join(".")
|
|
264
|
+
details = combination.map { |dimension, value| "#{dimension} #{value}" }.join(", ")
|
|
265
|
+
base.merge("id" => "#{entry["id"]}.#{suffix}", "name" => "#{base["name"]} (#{details})", "matrix" => combination)
|
|
266
|
+
end
|
|
267
|
+
end
|
|
268
|
+
|
|
269
|
+
def self.http_url?(value)
|
|
270
|
+
uri = URI.parse(value.to_s)
|
|
271
|
+
%w[http https].include?(uri.scheme) && !uri.host.to_s.empty? && uri.userinfo.nil?
|
|
272
|
+
rescue URI::InvalidURIError
|
|
273
|
+
false
|
|
274
|
+
end
|
|
62
275
|
end
|
|
63
276
|
end
|
|
@@ -0,0 +1,216 @@
|
|
|
1
|
+
require "json"
|
|
2
|
+
require "stringio"
|
|
3
|
+
|
|
4
|
+
module BrowserReviewGate
|
|
5
|
+
# Checks a project's setup and says what to fix. A check that cannot be verified from here is SKIP,
|
|
6
|
+
# never PASS. Nothing secret is printed: secrets are checked by name only.
|
|
7
|
+
class Doctor
|
|
8
|
+
Check = Struct.new(:name, :status, :message, :fix) do
|
|
9
|
+
def to_h = { "name" => name, "status" => status, "message" => message, "fix" => fix }.compact
|
|
10
|
+
end
|
|
11
|
+
|
|
12
|
+
SECRETS = %w[CLAUDE_CODE_OAUTH_TOKEN ANTHROPIC_API_KEY OPENAI_API_KEY GEMINI_API_KEY].freeze
|
|
13
|
+
BROWSER_TOOLS = { "playwright" => "Playwright", "chrome-devtools" => "Chrome DevTools MCP", "browsermcp" => "Browser MCP",
|
|
14
|
+
"puppeteer" => "Puppeteer" }.freeze
|
|
15
|
+
AGENT_FILES = {
|
|
16
|
+
"claude" => [ ".claude/skills/browser-pr-verification/SKILL.md", ".claude/settings.json" ],
|
|
17
|
+
"cursor" => [ ".cursor/commands/browser-pr-verification.md", ".cursor/hooks.json" ],
|
|
18
|
+
"codex" => [ "AGENTS.md", ".codex/hooks.json" ]
|
|
19
|
+
}.freeze
|
|
20
|
+
|
|
21
|
+
# `probe` takes a URL and says whether something answers there.
|
|
22
|
+
def initialize(root:, shell: Shell.new, github: nil, probe: nil, config_loader: nil)
|
|
23
|
+
@root = root
|
|
24
|
+
@shell = shell
|
|
25
|
+
@github = github
|
|
26
|
+
@probe = probe || Status.method(:answers?)
|
|
27
|
+
@config_loader = config_loader || -> { Config.load(root) }
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def checks
|
|
31
|
+
@checks = []
|
|
32
|
+
ruby
|
|
33
|
+
gem_executable
|
|
34
|
+
repository
|
|
35
|
+
config = configuration
|
|
36
|
+
gh_ready = github_cli
|
|
37
|
+
repository_access if gh_ready
|
|
38
|
+
environments(config) if config
|
|
39
|
+
agents(config)
|
|
40
|
+
browser_tools
|
|
41
|
+
workflow(config) if config
|
|
42
|
+
required_check(config) if config && gh_ready
|
|
43
|
+
provider_secrets(config) if config && gh_ready
|
|
44
|
+
@checks
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def self.text(checks)
|
|
48
|
+
width = checks.map { |check| check.name.length }.max.to_i
|
|
49
|
+
lines = checks.map do |check|
|
|
50
|
+
line = "#{check.status.ljust(4)} #{check.name.ljust(width)} #{check.message}"
|
|
51
|
+
check.fix ? "#{line}\n#{" " * (width + 8)}fix: #{check.fix}" : line
|
|
52
|
+
end
|
|
53
|
+
counts = %w[PASS WARN FAIL SKIP].map { |status| "#{checks.count { |check| check.status == status }} #{status}" }
|
|
54
|
+
(lines + [ "", counts.join(", ") ]).join("\n")
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
def self.json(checks) = JSON.pretty_generate("checks" => checks.map(&:to_h), "ok" => checks.none? { |check| check.status == "FAIL" })
|
|
58
|
+
|
|
59
|
+
private
|
|
60
|
+
|
|
61
|
+
def add(name, status, message, fix = nil) = @checks << Check.new(name, status, message, fix)
|
|
62
|
+
def exist?(path) = File.exist?(File.join(@root, path))
|
|
63
|
+
def read(path) = exist?(path) ? File.read(File.join(@root, path)) : ""
|
|
64
|
+
|
|
65
|
+
def run(*argv)
|
|
66
|
+
@shell.call(*argv)
|
|
67
|
+
rescue Error
|
|
68
|
+
nil
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
def ruby
|
|
72
|
+
return add("ruby", "PASS", "Ruby #{RUBY_VERSION}") if Gem::Version.new(RUBY_VERSION) >= Gem::Version.new("3.2")
|
|
73
|
+
|
|
74
|
+
add("ruby", "FAIL", "Ruby #{RUBY_VERSION} is older than 3.2", "Use Ruby 3.2 or newer")
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
def gem_executable
|
|
78
|
+
found = ENV["PATH"].to_s.split(File::PATH_SEPARATOR).any? { |dir| File.executable?(File.join(dir, "browser-review-gate")) }
|
|
79
|
+
return add("gem", "PASS", "browser_review_gate #{VERSION} runs; a browser-review-gate executable is on PATH") if found
|
|
80
|
+
|
|
81
|
+
add("gem", "WARN", "browser_review_gate #{VERSION} runs, but browser-review-gate is not on PATH",
|
|
82
|
+
"bundle add browser_review_gate --group development, or run it with bundle exec")
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
def repository
|
|
86
|
+
top = run("git", "rev-parse", "--show-toplevel")
|
|
87
|
+
return add("repository", "PASS", "git repository at #{top.strip}") if top
|
|
88
|
+
|
|
89
|
+
add("repository", "FAIL", "not inside a git repository", "Run from the project checkout")
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
def configuration
|
|
93
|
+
config = @config_loader.call
|
|
94
|
+
add("settings", exist?(Config::PATH) ? "PASS" : "WARN", exist?(Config::PATH) ? "#{Config::PATH} is valid" : "#{Config::PATH} is missing; defaults apply",
|
|
95
|
+
exist?(Config::PATH) ? nil : "browser-review-gate install")
|
|
96
|
+
config
|
|
97
|
+
rescue Error => error
|
|
98
|
+
add("settings", "FAIL", error.message, "Fix #{Config::PATH}")
|
|
99
|
+
nil
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
def github_cli
|
|
103
|
+
unless run("gh", "--version")
|
|
104
|
+
add("gh", "FAIL", "the GitHub CLI is not installed", "Install gh: https://cli.github.com")
|
|
105
|
+
return false
|
|
106
|
+
end
|
|
107
|
+
unless run("gh", "auth", "status")
|
|
108
|
+
add("gh", "FAIL", "the GitHub CLI is not signed in", "gh auth login")
|
|
109
|
+
return false
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
add("gh", "PASS", "installed and signed in")
|
|
113
|
+
true
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
def repository_access
|
|
117
|
+
github = @github || GitHub.new(shell: @shell)
|
|
118
|
+
name = github.repository
|
|
119
|
+
push = run("gh", "api", "repos/#{name}", "--jq", ".permissions.push").to_s.strip
|
|
120
|
+
return add("repository access", "PASS", "#{name}: you can push, so your reports count") if push == "true"
|
|
121
|
+
return add("repository access", "WARN", "#{name}: you cannot push; reports you publish are ignored", "Ask for write access") if push == "false"
|
|
122
|
+
|
|
123
|
+
add("repository access", "SKIP", "#{name}: permissions could not be read")
|
|
124
|
+
rescue Error => error
|
|
125
|
+
add("repository access", "FAIL", error.message.lines.first.to_s.strip, "Check that the repository has a GitHub remote")
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
def environments(config)
|
|
129
|
+
return add("environments", "WARN", "no environment is configured", "Add environments.local.url in #{Config::PATH}") if config.environments.empty?
|
|
130
|
+
|
|
131
|
+
config.environments.each do |name, settings|
|
|
132
|
+
next add("environment #{name}", "WARN", "no url", "Set environments.#{name}.url") unless settings["url"]
|
|
133
|
+
|
|
134
|
+
url = Status.healthcheck_url(settings.merge("name" => name))
|
|
135
|
+
if @probe.call(url)
|
|
136
|
+
add("environment #{name}", "PASS", "answers at #{url}")
|
|
137
|
+
else
|
|
138
|
+
add("environment #{name}", "WARN", "does not answer at #{url}",
|
|
139
|
+
settings["start_command"] ? "Start it with #{settings["start_command"]} before a run" : "Make it reachable from where the agent runs")
|
|
140
|
+
end
|
|
141
|
+
end
|
|
142
|
+
end
|
|
143
|
+
|
|
144
|
+
def agents(config)
|
|
145
|
+
installer = Installer.new(root: @root, log: StringIO.new)
|
|
146
|
+
installer.detect_agents.each do |agent|
|
|
147
|
+
entry, hooks = AGENT_FILES.fetch(agent)
|
|
148
|
+
wired = agent == "codex" ? read(entry).include?("browser_review_gate:start") : exist?(entry)
|
|
149
|
+
hooked = read(hooks).include?("browser-review-gate hook #{agent}")
|
|
150
|
+
if wired && hooked
|
|
151
|
+
add("agent #{agent}", "PASS", "instructions and post-command hook installed")
|
|
152
|
+
else
|
|
153
|
+
add("agent #{agent}", "WARN", [ wired ? nil : "#{entry} missing", hooked ? nil : "hook missing in #{hooks}" ].compact.join("; "),
|
|
154
|
+
"browser-review-gate install --agents #{agent}")
|
|
155
|
+
end
|
|
156
|
+
end
|
|
157
|
+
add("playbook", exist?(Installer::PLAYBOOK) ? "PASS" : "WARN", exist?(Installer::PLAYBOOK) ? Installer::PLAYBOOK : "#{Installer::PLAYBOOK} missing",
|
|
158
|
+
exist?(Installer::PLAYBOOK) ? nil : "browser-review-gate install")
|
|
159
|
+
rules = Prompts.new(@root).rules
|
|
160
|
+
add("project rules", rules ? "PASS" : "WARN", rules ? "#{Prompts::RULES} has rules" : "#{Prompts::RULES} has no rules yet",
|
|
161
|
+
rules ? nil : "Describe what this application always checks")
|
|
162
|
+
config
|
|
163
|
+
end
|
|
164
|
+
|
|
165
|
+
# Only what the project's own files show. A tool configured elsewhere cannot be seen from here.
|
|
166
|
+
def browser_tools
|
|
167
|
+
files = [ ".mcp.json", ".cursor/mcp.json", ".codex/config.toml", ".claude/settings.json", "package.json" ]
|
|
168
|
+
text = files.map { |path| read(path) }.join("\n").downcase
|
|
169
|
+
found = BROWSER_TOOLS.select { |key, _label| text.include?(key) }.values
|
|
170
|
+
return add("browser tool", "PASS", "configured in the project: #{found.join(", ")}") if found.any?
|
|
171
|
+
|
|
172
|
+
add("browser tool", "SKIP", "none found in the project's files; one may be configured per user",
|
|
173
|
+
"Give the agent a browser tool, e.g. Playwright MCP or Chrome DevTools MCP")
|
|
174
|
+
end
|
|
175
|
+
|
|
176
|
+
def workflow(config)
|
|
177
|
+
path = ".github/workflows/#{config.workflow}"
|
|
178
|
+
return add("workflow", "FAIL", "#{path} is missing", "browser-review-gate install") unless exist?(path)
|
|
179
|
+
|
|
180
|
+
version = read(path)[/browser_review_gate:managed (\S+)/, 1]
|
|
181
|
+
return add("workflow", "PASS", "#{path} (#{version})") if version == VERSION
|
|
182
|
+
return add("workflow", "SKIP", "#{path} was not written by the installer; cannot tell its version") unless version
|
|
183
|
+
|
|
184
|
+
add("workflow", "WARN", "#{path} was written by #{version}; this is #{VERSION}", "browser-review-gate install")
|
|
185
|
+
end
|
|
186
|
+
|
|
187
|
+
def required_check(config)
|
|
188
|
+
github = @github || GitHub.new(shell: @shell)
|
|
189
|
+
branch = run("gh", "api", "repos/#{github.repository}", "--jq", ".default_branch").to_s.strip
|
|
190
|
+
return add("required check", "SKIP", "the default branch could not be read") if branch.empty?
|
|
191
|
+
|
|
192
|
+
contexts = run("gh", "api", "repos/#{github.repository}/branches/#{branch}/protection/required_status_checks", "--jq", ".contexts[]")
|
|
193
|
+
return add("required check", "SKIP", "branch protection of #{branch} could not be read (needs admin access, or none is set)",
|
|
194
|
+
"To block merging, make #{config.status_context} a required status check") unless contexts
|
|
195
|
+
return add("required check", "PASS", "#{config.status_context} is required on #{branch}") if contexts.lines.map(&:strip).include?(config.status_context)
|
|
196
|
+
|
|
197
|
+
add("required check", "WARN", "#{config.status_context} is not required on #{branch}; the gate does not block merging",
|
|
198
|
+
"Make #{config.status_context} a required status check")
|
|
199
|
+
rescue Error
|
|
200
|
+
add("required check", "SKIP", "could not be checked")
|
|
201
|
+
end
|
|
202
|
+
|
|
203
|
+
def provider_secrets(config)
|
|
204
|
+
names = run("gh", "secret", "list", "--json", "name", "--jq", ".[].name")
|
|
205
|
+
return add("model credentials", "SKIP", "repository secrets could not be listed (needs admin access)") unless names
|
|
206
|
+
|
|
207
|
+
present = names.lines.map(&:strip) & SECRETS
|
|
208
|
+
return add("model credentials", "FAIL", "no model secret in the repository", "Add one of #{SECRETS.join(", ")}") if present.empty?
|
|
209
|
+
if config.provider.to_s.match?(/\A(openai|gemini)\z/) && config.model.nil?
|
|
210
|
+
return add("model credentials", "FAIL", "#{config.provider} needs `model`", "Set model in #{Config::PATH}")
|
|
211
|
+
end
|
|
212
|
+
|
|
213
|
+
add("model credentials", "PASS", "secret present: #{present.join(", ")}")
|
|
214
|
+
end
|
|
215
|
+
end
|
|
216
|
+
end
|