rail_verdict 1.4.0 → 1.8.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +2 -1
- data/lib/rail_verdict/analyzers/brakeman.rb +309 -0
- data/lib/rail_verdict/analyzers/bundler_audit.rb +1 -1
- data/lib/rail_verdict/analyzers/minitest.rb +45 -9
- data/lib/rail_verdict/analyzers/rspec.rb +35 -6
- data/lib/rail_verdict/change_intelligence.rb +274 -0
- data/lib/rail_verdict/change_surfaces.rb +232 -0
- data/lib/rail_verdict/check.rb +83 -14
- data/lib/rail_verdict/cli.rb +294 -6
- data/lib/rail_verdict/comparison.rb +18 -2
- data/lib/rail_verdict/configuration.rb +12 -2
- data/lib/rail_verdict/contracts/analyzer_result.rb +1 -1
- data/lib/rail_verdict/contracts/finding.rb +6 -0
- data/lib/rail_verdict/doctor.rb +17 -5
- data/lib/rail_verdict/engineering_policy.rb +394 -0
- data/lib/rail_verdict/findings_command.rb +7 -1
- data/lib/rail_verdict/handoff.rb +4 -1
- data/lib/rail_verdict/mcp/server.rb +8 -0
- data/lib/rail_verdict/mcp/tools/create_workflow_receipt.rb +135 -0
- data/lib/rail_verdict/mcp/tools/get_engineering_policy.rb +117 -0
- data/lib/rail_verdict/mcp/tools/get_pr_intelligence.rb +1 -1
- data/lib/rail_verdict/mcp/tools/get_review_packet.rb +116 -0
- data/lib/rail_verdict/mcp/tools/verify_review_observation.rb +120 -0
- data/lib/rail_verdict/mcp.rb +12 -0
- data/lib/rail_verdict/pr_intelligence.rb +39 -1
- data/lib/rail_verdict/rails_context/resolvers/test_candidates.rb +38 -4
- data/lib/rail_verdict/repair/verification_plan.rb +33 -1
- data/lib/rail_verdict/reporters/console.rb +40 -0
- data/lib/rail_verdict/reporters/engineering_policy.rb +50 -0
- data/lib/rail_verdict/reporters/github_annotations.rb +6 -1
- data/lib/rail_verdict/reporters/pr_intelligence.rb +54 -5
- data/lib/rail_verdict/reporters/review.rb +73 -0
- data/lib/rail_verdict/reporters/sarif.rb +6 -1
- data/lib/rail_verdict/reuse.rb +106 -0
- data/lib/rail_verdict/review_observation.rb +95 -0
- data/lib/rail_verdict/review_packet.rb +159 -0
- data/lib/rail_verdict/schema_validator.rb +34 -2
- data/lib/rail_verdict/test_selection.rb +186 -0
- data/lib/rail_verdict/verification_environment.rb +16 -7
- data/lib/rail_verdict/verification_plan.rb +97 -0
- data/lib/rail_verdict/version.rb +1 -1
- data/lib/rail_verdict/workflow_receipt.rb +77 -0
- data/lib/rail_verdict.rb +11 -0
- data/schemas/configuration-v1.5.schema.json +2 -1
- data/schemas/configuration-v1.6.schema.json +177 -0
- data/schemas/configuration-v1.7.schema.json +499 -0
- data/schemas/engineering-policy-v1.schema.json +107 -0
- data/schemas/pr-intelligence-v1.1.schema.json +407 -0
- data/schemas/repair-packet-v1.schema.json +42 -0
- data/schemas/review-observation-v1.schema.json +68 -0
- data/schemas/review-packet-v1.schema.json +175 -0
- data/schemas/workflow-receipt-v1.schema.json +47 -0
- metadata +23 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 564018ea0a8253a34b54578dc59864000267937ac5175aa437903a5afb67ece8
|
|
4
|
+
data.tar.gz: ed89e1c5d4b7610b12eca7d66eca82e268dc17297ac040107ff26f79e620ac86
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: b41423f2cedfac216e9997745f93d8480fc78d8b0b4b0d7274c2c4f9a32d430287b7250d0229e5c23374f91097dfe178df2164bc3f5bc9e0d1dcd0282382e378
|
|
7
|
+
data.tar.gz: efef3d68cf12a5f40788a946b086a74428b06afca0f1f68b9cad79d8dbbc8fdfe81ca0def0d8de3090d050afbee9e2e224f25be7b1f1961c5788a599c40efa34
|
data/README.md
CHANGED
|
@@ -35,7 +35,8 @@ Running test suites alone is not enough to answer a fundamental question:
|
|
|
35
35
|
Existing tools produce fragmented formats and disparate semantics:
|
|
36
36
|
|
|
37
37
|
- **RuboCop** reports style and lint offenses;
|
|
38
|
-
- **
|
|
38
|
+
- **Brakeman** reports Rails security vulnerabilities;
|
|
39
|
+
- **RSpec** and **Minitest** report test outcomes and failures (with intelligent Rails-aware targeted test selection);
|
|
39
40
|
- **SimpleCov** reports line coverage metrics;
|
|
40
41
|
- **bundler-audit** reports gem dependency advisories;
|
|
41
42
|
- **Git** tracks what actually changed.
|
|
@@ -0,0 +1,309 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
require "tmpdir"
|
|
5
|
+
require "securerandom"
|
|
6
|
+
|
|
7
|
+
require_relative "_shared"
|
|
8
|
+
|
|
9
|
+
module RailVerdict
|
|
10
|
+
module Analyzers
|
|
11
|
+
class Brakeman
|
|
12
|
+
ANALYZER_ID = "brakeman"
|
|
13
|
+
SUPPORTED_VERSIONS = Gem::Requirement.new(">= 7.0", "< 9")
|
|
14
|
+
|
|
15
|
+
SEVERITY_MAP = {
|
|
16
|
+
"High" => "high",
|
|
17
|
+
"Medium" => "medium",
|
|
18
|
+
"Weak" => "low"
|
|
19
|
+
}.freeze
|
|
20
|
+
|
|
21
|
+
CONFIDENCE_MAP = {
|
|
22
|
+
"High" => "high",
|
|
23
|
+
"Medium" => "medium",
|
|
24
|
+
"Weak" => "low"
|
|
25
|
+
}.freeze
|
|
26
|
+
|
|
27
|
+
Probe = Struct.new(:status, :version, :message, keyword_init: true)
|
|
28
|
+
|
|
29
|
+
class MalformedOutput < StandardError; end
|
|
30
|
+
|
|
31
|
+
def initialize(command_resolver: nil)
|
|
32
|
+
@command_resolver = command_resolver || method(:default_command)
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def probe(repository_root, runner: ProcessRunner, timeout_seconds: 15.0)
|
|
36
|
+
effective_timeout = [timeout_seconds.to_f, 5.0].min
|
|
37
|
+
command = @command_resolver.call(repository_root)
|
|
38
|
+
clean_prefix = clean_args_prefix(command.fetch(:args_prefix))
|
|
39
|
+
clean_command = command.merge(args_prefix: clean_prefix)
|
|
40
|
+
invocation = Shared.invocation_for(clean_command, ["--version"])
|
|
41
|
+
result = runner.run(
|
|
42
|
+
clean_command.fetch(:executable),
|
|
43
|
+
invocation.fetch("argv"),
|
|
44
|
+
chdir: repository_root,
|
|
45
|
+
timeout_seconds: effective_timeout
|
|
46
|
+
)
|
|
47
|
+
|
|
48
|
+
return Probe.new(status: "unavailable", message: Shared.detail_for(result)) if result.status == :spawn_failed
|
|
49
|
+
return Probe.new(status: "timed_out", message: Shared.detail_for(result)) if result.status == :timed_out
|
|
50
|
+
return Probe.new(status: "signaled", message: Shared.detail_for(result)) if result.status == :signaled
|
|
51
|
+
return Probe.new(status: "truncated", message: Shared.detail_for(result)) if Shared.truncated?(result)
|
|
52
|
+
return Probe.new(status: "unavailable", message: Shared.detail_for(result)) unless result.exit_code == 0
|
|
53
|
+
|
|
54
|
+
version = Shared.parse_semver(result.stdout)
|
|
55
|
+
return Probe.new(status: "unsupported", message: "Brakeman version could not be parsed") unless version
|
|
56
|
+
return Probe.new(status: "unsupported", version: version, message: "unsupported Brakeman version #{version}") unless SUPPORTED_VERSIONS.satisfied_by?(Gem::Version.new(version))
|
|
57
|
+
|
|
58
|
+
Probe.new(status: "succeeded", version: version)
|
|
59
|
+
rescue ArgumentError
|
|
60
|
+
Probe.new(status: "unsupported", message: "Brakeman reported an invalid version")
|
|
61
|
+
rescue KeyError, ArgumentError => error
|
|
62
|
+
Probe.new(status: "malformed", message: Shared.bounded_message(error.message))
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
def run(repository_root, runner: ProcessRunner, timeout_seconds: 30.0, probe_result: nil, configuration: nil)
|
|
66
|
+
command = @command_resolver.call(repository_root)
|
|
67
|
+
clean_prefix = clean_args_prefix(command.fetch(:args_prefix))
|
|
68
|
+
clean_command = command.merge(args_prefix: clean_prefix)
|
|
69
|
+
probe_result ||= probe(repository_root, runner: runner, timeout_seconds: timeout_seconds)
|
|
70
|
+
version_invocation = Shared.invocation_for(clean_command, ["--version"])
|
|
71
|
+
|
|
72
|
+
unless probe_result.status == "succeeded"
|
|
73
|
+
return [Shared.failure_result(analyzer_id: ANALYZER_ID, invocation: version_invocation, status: probe_result.status, message: probe_result.message, tool_version: probe_result.version), []]
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
output_path = File.join(Dir.tmpdir, "railverdict-brakeman-#{SecureRandom.hex(8)}.json")
|
|
77
|
+
public_invocation = Shared.invocation_for(clean_command, ["-q", "--no-pager", "-f", "json"])
|
|
78
|
+
run_argv = clean_prefix.dup.concat(["-q", "--no-pager", "-f", "json", "-o", output_path])
|
|
79
|
+
|
|
80
|
+
max_stdout = resolve_stdout_limit(configuration, repository_root, 16 * 1024 * 1024)
|
|
81
|
+
tool_version = probe_result.version
|
|
82
|
+
|
|
83
|
+
begin
|
|
84
|
+
result = runner.run(
|
|
85
|
+
clean_command.fetch(:executable),
|
|
86
|
+
run_argv,
|
|
87
|
+
chdir: repository_root,
|
|
88
|
+
timeout_seconds: timeout_seconds,
|
|
89
|
+
max_stdout_bytes: max_stdout
|
|
90
|
+
)
|
|
91
|
+
|
|
92
|
+
return [Shared.failure_result(analyzer_id: ANALYZER_ID, invocation: public_invocation, status: "unavailable", message: Shared.detail_for(result), tool_version: tool_version), []] if result.status == :spawn_failed
|
|
93
|
+
return [Shared.failure_result(analyzer_id: ANALYZER_ID, invocation: public_invocation, status: "truncated", message: Shared.detail_for(result), tool_version: tool_version), []] if Shared.truncated?(result)
|
|
94
|
+
return [Shared.failure_result(analyzer_id: ANALYZER_ID, invocation: public_invocation, status: "timed_out", message: Shared.detail_for(result), tool_version: tool_version), []] if result.status == :timed_out
|
|
95
|
+
return [Shared.failure_result(analyzer_id: ANALYZER_ID, invocation: public_invocation, status: "signaled", message: Shared.detail_for(result), tool_version: tool_version), []] if result.status == :signaled
|
|
96
|
+
|
|
97
|
+
unless File.file?(output_path)
|
|
98
|
+
detail = Shared.detail_for(result)
|
|
99
|
+
msg = detail.strip.empty? ? "Brakeman did not produce structured output" : detail
|
|
100
|
+
status = result.exit_code && result.exit_code != 0 ? "failed" : "malformed"
|
|
101
|
+
return [Shared.failure_result(analyzer_id: ANALYZER_ID, invocation: public_invocation, status: status, message: msg, tool_version: tool_version), []]
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
begin
|
|
105
|
+
bytes = File.binread(output_path)
|
|
106
|
+
rescue SystemCallError => error
|
|
107
|
+
return [Shared.failure_result(analyzer_id: ANALYZER_ID, invocation: public_invocation, status: "malformed", message: Shared.bounded_message(error.message), tool_version: tool_version), []]
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
if bytes.bytesize > max_stdout
|
|
111
|
+
return [Shared.failure_result(analyzer_id: ANALYZER_ID, invocation: public_invocation, status: "truncated", message: "Brakeman output exceeds #{max_stdout} bytes", tool_version: tool_version), []]
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
text = bytes.dup.force_encoding(Encoding::UTF_8)
|
|
115
|
+
unless text.valid_encoding?
|
|
116
|
+
return [Shared.failure_result(analyzer_id: ANALYZER_ID, invocation: public_invocation, status: "parse_failed", message: "Brakeman output is not valid UTF-8", tool_version: tool_version), []]
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
begin
|
|
120
|
+
document = JSON.parse(text)
|
|
121
|
+
rescue JSON::ParserError => error
|
|
122
|
+
return [Shared.failure_result(analyzer_id: ANALYZER_ID, invocation: public_invocation, status: "parse_failed", message: Shared.bounded_message(error.message), tool_version: tool_version), []]
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
begin
|
|
126
|
+
summary, findings = normalize_document(document)
|
|
127
|
+
rescue MalformedOutput => error
|
|
128
|
+
return [Shared.failure_result(analyzer_id: ANALYZER_ID, invocation: public_invocation, status: "malformed", message: Shared.bounded_message(error.message), tool_version: tool_version), []]
|
|
129
|
+
end
|
|
130
|
+
|
|
131
|
+
# Process exit reconciliation:
|
|
132
|
+
# 0: Clean scan (0 warnings) or explicit no-exit-on-warn
|
|
133
|
+
# 3: Warnings reported by Brakeman (findings > 0)
|
|
134
|
+
# Any other status: execution failure
|
|
135
|
+
warnings_count = summary["security_warnings"] || 0
|
|
136
|
+
if result.exit_code == 0
|
|
137
|
+
# Clean scan or non-failing mode
|
|
138
|
+
elsif result.exit_code == 3
|
|
139
|
+
if warnings_count == 0 && findings.empty?
|
|
140
|
+
detail = Shared.detail_for(result)
|
|
141
|
+
msg = detail.strip.empty? ? "Brakeman exited with status 3 but reported 0 warnings" : detail
|
|
142
|
+
return [Shared.failure_result(analyzer_id: ANALYZER_ID, invocation: public_invocation, status: "failed", message: msg, tool_version: tool_version), []]
|
|
143
|
+
end
|
|
144
|
+
else
|
|
145
|
+
detail = Shared.detail_for(result)
|
|
146
|
+
msg = detail.strip.empty? ? "Brakeman exited with unexpected status #{result.exit_code}" : detail
|
|
147
|
+
return [Shared.failure_result(analyzer_id: ANALYZER_ID, invocation: public_invocation, status: "failed", message: msg, tool_version: tool_version), []]
|
|
148
|
+
end
|
|
149
|
+
|
|
150
|
+
analyzer_result = AnalyzerResult.new(
|
|
151
|
+
analyzer: ANALYZER_ID,
|
|
152
|
+
tool_version: tool_version,
|
|
153
|
+
invocation: public_invocation,
|
|
154
|
+
execution_status: "succeeded",
|
|
155
|
+
finding_ids: findings.map(&:id),
|
|
156
|
+
evidence_summary: summary
|
|
157
|
+
)
|
|
158
|
+
[analyzer_result, findings]
|
|
159
|
+
ensure
|
|
160
|
+
begin
|
|
161
|
+
File.unlink(output_path) if output_path && File.exist?(output_path)
|
|
162
|
+
rescue StandardError
|
|
163
|
+
nil
|
|
164
|
+
end
|
|
165
|
+
end
|
|
166
|
+
end
|
|
167
|
+
|
|
168
|
+
private
|
|
169
|
+
|
|
170
|
+
def default_command(repository_root)
|
|
171
|
+
if File.file?(File.join(repository_root, "Gemfile"))
|
|
172
|
+
{ executable: "bundle", args_prefix: ["exec", "brakeman"] }
|
|
173
|
+
else
|
|
174
|
+
{ executable: "brakeman", args_prefix: [] }
|
|
175
|
+
end
|
|
176
|
+
end
|
|
177
|
+
|
|
178
|
+
def clean_args_prefix(prefix)
|
|
179
|
+
cleaned = []
|
|
180
|
+
skip_next = false
|
|
181
|
+
Array(prefix).each_with_index do |arg, i|
|
|
182
|
+
if skip_next
|
|
183
|
+
skip_next = false
|
|
184
|
+
next
|
|
185
|
+
end
|
|
186
|
+
if (arg == "-f" || arg == "--format" || arg == "-o" || arg == "--output") && Array(prefix)[i + 1]
|
|
187
|
+
skip_next = true
|
|
188
|
+
next
|
|
189
|
+
elsif arg.start_with?("-f=") || arg.start_with?("--format=") || arg.start_with?("-o=") || arg.start_with?("--output=")
|
|
190
|
+
next
|
|
191
|
+
elsif arg == "-q" || arg == "--quiet" || arg == "--no-pager"
|
|
192
|
+
next
|
|
193
|
+
end
|
|
194
|
+
cleaned << arg
|
|
195
|
+
end
|
|
196
|
+
cleaned
|
|
197
|
+
end
|
|
198
|
+
|
|
199
|
+
def normalize_document(document)
|
|
200
|
+
raise MalformedOutput, "Brakeman JSON root must be an object" unless document.is_a?(Hash)
|
|
201
|
+
|
|
202
|
+
scan_info = document["scan_info"]
|
|
203
|
+
warnings = document["warnings"]
|
|
204
|
+
errors = document["errors"]
|
|
205
|
+
|
|
206
|
+
raise MalformedOutput, "Brakeman JSON is missing scan_info" unless scan_info.is_a?(Hash)
|
|
207
|
+
raise MalformedOutput, "Brakeman JSON is missing warnings array" unless warnings.is_a?(Array)
|
|
208
|
+
|
|
209
|
+
findings = []
|
|
210
|
+
warnings.each_with_index do |warning, index|
|
|
211
|
+
finding = normalize_warning(warning, index)
|
|
212
|
+
findings << finding if finding
|
|
213
|
+
end
|
|
214
|
+
|
|
215
|
+
findings = findings.uniq(&:fingerprint).sort_by(&:sort_key)
|
|
216
|
+
summary_h = build_summary(scan_info, findings.length)
|
|
217
|
+
|
|
218
|
+
[summary_h, findings]
|
|
219
|
+
end
|
|
220
|
+
|
|
221
|
+
def normalize_warning(warning, index)
|
|
222
|
+
raise MalformedOutput, "Brakeman warning #{index} must be an object" unless warning.is_a?(Hash)
|
|
223
|
+
|
|
224
|
+
check_name = warning["check_name"] || warning["warning_type"] || "warning_#{warning['warning_code'] || index}"
|
|
225
|
+
rule_id = check_name.to_s.gsub(/[^a-zA-Z0-9_-]/, "_")[0, 64]
|
|
226
|
+
rule_id = "warning_#{index}" if rule_id.empty?
|
|
227
|
+
|
|
228
|
+
raw_msg = warning["message"] || warning["warning_type"] || "security warning"
|
|
229
|
+
message = Shared.normalize_finding_message(ANALYZER_ID, raw_msg)
|
|
230
|
+
|
|
231
|
+
confidence_str = warning["confidence"].to_s
|
|
232
|
+
confidence = CONFIDENCE_MAP.fetch(confidence_str) { "medium" }
|
|
233
|
+
severity = SEVERITY_MAP.fetch(confidence_str) { "medium" }
|
|
234
|
+
|
|
235
|
+
category = "security"
|
|
236
|
+
path = normalize_path(warning["file"] || "unknown")
|
|
237
|
+
start_line = extract_line(warning["line"])
|
|
238
|
+
|
|
239
|
+
location = { "path" => path }
|
|
240
|
+
location["start_line"] = start_line if start_line && start_line >= 1
|
|
241
|
+
location["end_line"] = location["start_line"] if location.key?("start_line")
|
|
242
|
+
|
|
243
|
+
fingerprint = Finding.fingerprint_for(
|
|
244
|
+
analyzer: ANALYZER_ID,
|
|
245
|
+
rule_id: rule_id,
|
|
246
|
+
path: path,
|
|
247
|
+
message: message
|
|
248
|
+
)
|
|
249
|
+
|
|
250
|
+
Finding.new(
|
|
251
|
+
fingerprint: fingerprint,
|
|
252
|
+
origin: "deterministic",
|
|
253
|
+
analyzer: ANALYZER_ID,
|
|
254
|
+
rule_id: rule_id,
|
|
255
|
+
category: category,
|
|
256
|
+
severity: severity,
|
|
257
|
+
confidence: confidence,
|
|
258
|
+
state: "observed",
|
|
259
|
+
evidence_ref: "native:brakeman:#{fingerprint.delete_prefix("sha256:")[0, 12]}",
|
|
260
|
+
location: location,
|
|
261
|
+
message: message
|
|
262
|
+
)
|
|
263
|
+
rescue ArgumentError => error
|
|
264
|
+
raise MalformedOutput, "Brakeman warning #{index} is malformed: #{error.message}"
|
|
265
|
+
end
|
|
266
|
+
|
|
267
|
+
def normalize_path(file)
|
|
268
|
+
return "unknown" unless file.is_a?(String) && !file.empty?
|
|
269
|
+
|
|
270
|
+
cleaned = file.delete_prefix("./").delete_prefix("/")
|
|
271
|
+
return cleaned if cleaned.match?(Finding::LOCATION_PATH_PATTERN)
|
|
272
|
+
|
|
273
|
+
"unknown"
|
|
274
|
+
end
|
|
275
|
+
|
|
276
|
+
def extract_line(line)
|
|
277
|
+
return nil if line.nil?
|
|
278
|
+
val = Integer(line) rescue nil
|
|
279
|
+
val && val >= 1 ? val : nil
|
|
280
|
+
end
|
|
281
|
+
|
|
282
|
+
def build_summary(scan_info, security_warnings_count)
|
|
283
|
+
{
|
|
284
|
+
"security_warnings" => security_warnings_count,
|
|
285
|
+
"checks_performed" => scan_info["checks_performed"].is_a?(Array) ? scan_info["checks_performed"].length : 0,
|
|
286
|
+
"duration_seconds" => Float(scan_info["duration"] || scan_info["duration_seconds"] || 0),
|
|
287
|
+
"rails_version" => scan_info["rails_version"].to_s,
|
|
288
|
+
"ruby_version" => scan_info["ruby_version"].to_s,
|
|
289
|
+
"brakeman_version" => scan_info["brakeman_version"].to_s
|
|
290
|
+
}
|
|
291
|
+
rescue ArgumentError, TypeError
|
|
292
|
+
raise MalformedOutput, "Brakeman scan_info fields have invalid types"
|
|
293
|
+
end
|
|
294
|
+
|
|
295
|
+
def resolve_stdout_limit(configuration, repository_root, default_bytes)
|
|
296
|
+
raw_limit = nil
|
|
297
|
+
if configuration
|
|
298
|
+
sel = configuration.analyzers[ANALYZER_ID] rescue nil
|
|
299
|
+
raw_limit = sel && sel["output_limit_bytes"]
|
|
300
|
+
end
|
|
301
|
+
raw_limit ||= default_bytes
|
|
302
|
+
limit = Integer(raw_limit) rescue default_bytes
|
|
303
|
+
limit = default_bytes if limit <= 0
|
|
304
|
+
max = RailVerdict::ProcessRunner::MAX_SAFE_STDOUT_BYTES
|
|
305
|
+
[[limit, max].min, 1024].max
|
|
306
|
+
end
|
|
307
|
+
end
|
|
308
|
+
end
|
|
309
|
+
end
|
|
@@ -147,7 +147,7 @@ module RailVerdict
|
|
|
147
147
|
raw_msg = advisory["title"] || advisory["description"] || entry["title"] || entry["description"] || "vulnerability in #{gem_name}"
|
|
148
148
|
message = Shared.normalize_finding_message(ANALYZER_ID, raw_msg)
|
|
149
149
|
|
|
150
|
-
rule_id = "
|
|
150
|
+
rule_id = "advisory:#{id}"
|
|
151
151
|
category = "dependency"
|
|
152
152
|
|
|
153
153
|
path = "Gemfile.lock"
|
|
@@ -48,7 +48,7 @@ module RailVerdict
|
|
|
48
48
|
Probe.new(status: "malformed", message: Shared.bounded_message(error.message))
|
|
49
49
|
end
|
|
50
50
|
|
|
51
|
-
def run(repository_root, runner: ProcessRunner, timeout_seconds: 30.0, probe_result: nil, configuration: nil)
|
|
51
|
+
def run(repository_root, runner: ProcessRunner, timeout_seconds: 30.0, probe_result: nil, configuration: nil, target_files: nil, test_scope: "full", fallback_reason: nil)
|
|
52
52
|
probe_result ||= probe(repository_root, runner: runner, timeout_seconds: timeout_seconds)
|
|
53
53
|
|
|
54
54
|
unless probe_result.status == "succeeded"
|
|
@@ -57,13 +57,40 @@ module RailVerdict
|
|
|
57
57
|
return [Shared.failure_result(analyzer_id: ANALYZER_ID, invocation: version_invocation, status: probe_result.status, message: probe_result.message, tool_version: probe_result.version), []]
|
|
58
58
|
end
|
|
59
59
|
|
|
60
|
+
target_list = Array(target_files).compact.reject(&:empty?)
|
|
61
|
+
if target_files.is_a?(Array) && target_files.empty? && test_scope == "targeted"
|
|
62
|
+
summary = {
|
|
63
|
+
"schema_version" => "1.0",
|
|
64
|
+
"runner" => "minitest #{probe_result.version}",
|
|
65
|
+
"seed" => 0,
|
|
66
|
+
"tests_total" => 0,
|
|
67
|
+
"assertions" => 0,
|
|
68
|
+
"failures" => 0,
|
|
69
|
+
"errors" => 0,
|
|
70
|
+
"skips" => 0,
|
|
71
|
+
"duration_seconds" => 0.0,
|
|
72
|
+
"test_scope" => "targeted",
|
|
73
|
+
"target_files" => [],
|
|
74
|
+
"fallback_reason" => fallback_reason
|
|
75
|
+
}.compact
|
|
76
|
+
analyzer_result = AnalyzerResult.new(
|
|
77
|
+
analyzer: ANALYZER_ID,
|
|
78
|
+
tool_version: probe_result.version,
|
|
79
|
+
invocation: Shared.invocation_for(@command_resolver.call(repository_root), ["run"]),
|
|
80
|
+
execution_status: "succeeded",
|
|
81
|
+
finding_ids: [],
|
|
82
|
+
evidence_summary: summary
|
|
83
|
+
)
|
|
84
|
+
return [analyzer_result, []]
|
|
85
|
+
end
|
|
86
|
+
|
|
60
87
|
command = @command_resolver.call(repository_root)
|
|
61
88
|
reporter_path = resolve_reporter_path
|
|
62
89
|
unless reporter_path
|
|
63
90
|
return [Shared.failure_result(analyzer_id: ANALYZER_ID, invocation: Shared.invocation_for(command, ["run"]), status: "malformed", message: "installed Minitest reporter is missing", tool_version: probe_result.version), []]
|
|
64
91
|
end
|
|
65
92
|
|
|
66
|
-
invocation = Shared.invocation_for(command, ["run"])
|
|
93
|
+
invocation = Shared.invocation_for(command, target_list.empty? ? ["run"] : ["run"] + target_list)
|
|
67
94
|
output_path = File.join(Dir.tmpdir, "railverdict-minitest-#{SecureRandom.hex(8)}.json")
|
|
68
95
|
env_reset_required = false
|
|
69
96
|
previous_env = ENV["RAILVERDICT_MINITEST_OUTPUT"]
|
|
@@ -71,7 +98,7 @@ module RailVerdict
|
|
|
71
98
|
ENV["RAILVERDICT_MINITEST_OUTPUT"] = output_path
|
|
72
99
|
env_reset_required = true
|
|
73
100
|
augmented_argv = invocation.fetch("argv").dup
|
|
74
|
-
augmented_argv = inject_reporter_require(augmented_argv, reporter_path)
|
|
101
|
+
augmented_argv = inject_reporter_require(augmented_argv, reporter_path, target_files: target_list)
|
|
75
102
|
result = runner.run(
|
|
76
103
|
command.fetch(:executable),
|
|
77
104
|
augmented_argv,
|
|
@@ -89,7 +116,7 @@ module RailVerdict
|
|
|
89
116
|
return document if document.is_a?(Array) && document.first.is_a?(AnalyzerResult)
|
|
90
117
|
|
|
91
118
|
begin
|
|
92
|
-
summary, findings = normalize_document(document)
|
|
119
|
+
summary, findings = normalize_document(document, test_scope: test_scope, target_files: target_list, fallback_reason: fallback_reason)
|
|
93
120
|
rescue MalformedOutput => error
|
|
94
121
|
return [Shared.failure_result(analyzer_id: ANALYZER_ID, invocation: invocation, status: "malformed", message: Shared.bounded_message(error.message), tool_version: tool_version), []]
|
|
95
122
|
end
|
|
@@ -175,17 +202,23 @@ analyzer_result = AnalyzerResult.new(
|
|
|
175
202
|
base
|
|
176
203
|
end
|
|
177
204
|
|
|
178
|
-
def inject_reporter_require(argv, reporter_path)
|
|
205
|
+
def inject_reporter_require(argv, reporter_path, target_files: nil)
|
|
179
206
|
argv = argv.dup
|
|
207
|
+
test_expr = if target_files && !target_files.empty?
|
|
208
|
+
files_list = target_files.map { |f| File.expand_path(f) }.inspect
|
|
209
|
+
"#{files_list}.each{|f| require f }"
|
|
210
|
+
else
|
|
211
|
+
"Dir['test/**/*_test.rb'].sort.each{|f| require File.expand_path(f) }"
|
|
212
|
+
end
|
|
180
213
|
if argv.include?("run")
|
|
181
214
|
idx = argv.index("run")
|
|
182
215
|
argv[idx] = "-r"
|
|
183
216
|
argv.insert(idx + 1, reporter_path)
|
|
184
217
|
argv.insert(idx + 2, "-e")
|
|
185
|
-
argv.insert(idx + 3,
|
|
218
|
+
argv.insert(idx + 3, test_expr)
|
|
186
219
|
else
|
|
187
220
|
argv.concat(["-r", reporter_path])
|
|
188
|
-
argv.concat(["-e",
|
|
221
|
+
argv.concat(["-e", test_expr])
|
|
189
222
|
end
|
|
190
223
|
argv
|
|
191
224
|
end
|
|
@@ -220,7 +253,7 @@ analyzer_result = AnalyzerResult.new(
|
|
|
220
253
|
end
|
|
221
254
|
end
|
|
222
255
|
|
|
223
|
-
def normalize_document(document)
|
|
256
|
+
def normalize_document(document, test_scope: "full", target_files: nil, fallback_reason: nil)
|
|
224
257
|
raise MalformedOutput, "Minitest JSON root must be an object" unless document.is_a?(Hash)
|
|
225
258
|
|
|
226
259
|
required = %w[schema_version runner seed tests_total assertions failures errors skips duration_seconds tests]
|
|
@@ -242,6 +275,9 @@ analyzer_result = AnalyzerResult.new(
|
|
|
242
275
|
|
|
243
276
|
findings = findings.uniq { |f| f.fingerprint }.sort_by(&:sort_key)
|
|
244
277
|
summary = build_summary(document)
|
|
278
|
+
summary["test_scope"] = test_scope if test_scope
|
|
279
|
+
summary["target_files"] = target_files if target_files && !target_files.empty?
|
|
280
|
+
summary["fallback_reason"] = fallback_reason if fallback_reason
|
|
245
281
|
[summary, findings]
|
|
246
282
|
end
|
|
247
283
|
|
|
@@ -260,7 +296,7 @@ analyzer_result = AnalyzerResult.new(
|
|
|
260
296
|
|
|
261
297
|
severity = status == "errored" ? "critical" : "high"
|
|
262
298
|
category = "test"
|
|
263
|
-
rule_id = "
|
|
299
|
+
rule_id = "test:#{class_name}##{method_name}"
|
|
264
300
|
raw_msg = test["failure_message"] || test["method_name"] || "test failed: #{method_name}"
|
|
265
301
|
message = Shared.normalize_finding_message(ANALYZER_ID, raw_msg)
|
|
266
302
|
|
|
@@ -50,7 +50,7 @@ module RailVerdict
|
|
|
50
50
|
Probe.new(status: "malformed", message: Shared.bounded_message(error.message))
|
|
51
51
|
end
|
|
52
52
|
|
|
53
|
-
def run(repository_root, runner: ProcessRunner, timeout_seconds: 30.0, probe_result: nil, configuration: nil)
|
|
53
|
+
def run(repository_root, runner: ProcessRunner, timeout_seconds: 30.0, probe_result: nil, configuration: nil, target_files: nil, test_scope: "full", fallback_reason: nil)
|
|
54
54
|
command = @command_resolver.call(repository_root)
|
|
55
55
|
clean_prefix = clean_args_prefix(command.fetch(:args_prefix))
|
|
56
56
|
clean_command = command.merge(args_prefix: clean_prefix)
|
|
@@ -61,9 +61,35 @@ module RailVerdict
|
|
|
61
61
|
return [Shared.failure_result(analyzer_id: ANALYZER_ID, invocation: version_invocation, status: probe_result.status, message: probe_result.message, tool_version: probe_result.version), []]
|
|
62
62
|
end
|
|
63
63
|
|
|
64
|
+
if target_files.is_a?(Array) && target_files.empty? && test_scope == "targeted"
|
|
65
|
+
summary = {
|
|
66
|
+
"tests_total" => 0,
|
|
67
|
+
"duration_seconds" => 0.0,
|
|
68
|
+
"failures" => 0,
|
|
69
|
+
"errors" => 0,
|
|
70
|
+
"assertions" => 0,
|
|
71
|
+
"skips" => 0,
|
|
72
|
+
"runner" => "rspec #{probe_result.version}",
|
|
73
|
+
"test_scope" => "targeted",
|
|
74
|
+
"target_files" => [],
|
|
75
|
+
"fallback_reason" => fallback_reason
|
|
76
|
+
}.compact
|
|
77
|
+
analyzer_result = AnalyzerResult.new(
|
|
78
|
+
analyzer: ANALYZER_ID,
|
|
79
|
+
tool_version: probe_result.version,
|
|
80
|
+
invocation: Shared.invocation_for(clean_command, ["--format", "json"]),
|
|
81
|
+
execution_status: "succeeded",
|
|
82
|
+
finding_ids: [],
|
|
83
|
+
evidence_summary: summary
|
|
84
|
+
)
|
|
85
|
+
return [analyzer_result, []]
|
|
86
|
+
end
|
|
87
|
+
|
|
64
88
|
output_path = File.join(Dir.tmpdir, "railverdict-rspec-#{SecureRandom.hex(8)}.json")
|
|
65
|
-
|
|
66
|
-
|
|
89
|
+
target_list = Array(target_files).compact.reject(&:empty?)
|
|
90
|
+
public_argv = target_list.empty? ? ["--format", "json"] : target_list + ["--format", "json"]
|
|
91
|
+
public_invocation = Shared.invocation_for(clean_command, public_argv)
|
|
92
|
+
run_argv = clean_prefix.dup.concat(target_list).concat(["--format", "json", "--out", output_path])
|
|
67
93
|
|
|
68
94
|
max_stdout = resolve_stdout_limit(configuration, repository_root, 16 * 1024 * 1024)
|
|
69
95
|
tool_version = probe_result.version
|
|
@@ -111,7 +137,7 @@ module RailVerdict
|
|
|
111
137
|
end
|
|
112
138
|
|
|
113
139
|
begin
|
|
114
|
-
summary, findings = normalize_document(document)
|
|
140
|
+
summary, findings = normalize_document(document, test_scope: test_scope, target_files: target_list, fallback_reason: fallback_reason)
|
|
115
141
|
rescue MalformedOutput => error
|
|
116
142
|
return [Shared.failure_result(analyzer_id: ANALYZER_ID, invocation: public_invocation, status: "malformed", message: Shared.bounded_message(error.message), tool_version: tool_version), []]
|
|
117
143
|
end
|
|
@@ -184,7 +210,7 @@ module RailVerdict
|
|
|
184
210
|
cleaned
|
|
185
211
|
end
|
|
186
212
|
|
|
187
|
-
def normalize_document(document)
|
|
213
|
+
def normalize_document(document, test_scope: "full", target_files: nil, fallback_reason: nil)
|
|
188
214
|
raise MalformedOutput, "RSpec JSON root must be an object" unless document.is_a?(Hash)
|
|
189
215
|
|
|
190
216
|
summary = document["summary"]
|
|
@@ -203,6 +229,9 @@ module RailVerdict
|
|
|
203
229
|
findings = findings.uniq { |f| f.fingerprint }.sort_by(&:sort_key)
|
|
204
230
|
summary_h = build_summary(summary, version)
|
|
205
231
|
summary_h["tests_total"] = examples.length
|
|
232
|
+
summary_h["test_scope"] = test_scope if test_scope
|
|
233
|
+
summary_h["target_files"] = target_files if target_files && !target_files.empty?
|
|
234
|
+
summary_h["fallback_reason"] = fallback_reason if fallback_reason
|
|
206
235
|
|
|
207
236
|
[summary_h, findings]
|
|
208
237
|
end
|
|
@@ -222,7 +251,7 @@ module RailVerdict
|
|
|
222
251
|
raw_msg = (example["exception"] && example["exception"]["message"]) || example["full_description"] || example["description"] || nil
|
|
223
252
|
message = Shared.normalize_finding_message(ANALYZER_ID, raw_msg.nil? || raw_msg.to_s.strip.empty? ? "rspec example failed" : raw_msg)
|
|
224
253
|
|
|
225
|
-
rule_id = "
|
|
254
|
+
rule_id = "example:#{example[id] || id_for(example, index)}"
|
|
226
255
|
path = normalize_path(example["file_path"] || example["file"] || "spec/unknown_spec.rb")
|
|
227
256
|
failure_line, failure_path = failure_location(example, path)
|
|
228
257
|
start_line = failure_line || example["line_number"] || extract_line(example["id"])
|