branchproof 0.9.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +42 -0
- data/README.md +230 -28
- data/doc/Branchproof/Configuration.md +28 -0
- data/doc/Branchproof/CoveragePolicy.md +27 -0
- data/doc/Branchproof/Evidence.md +3 -0
- data/doc/Branchproof/FocusedReport.md +1 -1
- data/doc/Branchproof/Project.md +4 -1
- data/doc/Branchproof/RSpecAdapter/ClassRunnerGuard.md +11 -0
- data/doc/Branchproof/RSpecAdapter/ContextLifecycle.md +14 -0
- data/doc/Branchproof/RSpecAdapter/DefaultDiscovery.md +11 -0
- data/doc/Branchproof/RSpecAdapter/ExampleLifecycle.md +11 -0
- data/doc/Branchproof/RSpecAdapter/RunnerGuard.md +11 -0
- data/doc/Branchproof/RSpecAdapter/UnsupportedRunner.md +12 -0
- data/doc/Branchproof/RSpecAdapter.md +47 -0
- data/doc/Branchproof/RailsSupport.md +35 -2
- data/doc/Branchproof/Report.md +11 -2
- data/doc/Branchproof/ReportSelection.md +42 -0
- data/doc/Branchproof/Runtime.md +3 -0
- data/doc/Branchproof/Worker.md +8 -1
- data/doc/Branchproof.md +11 -1
- data/doc/CHANGELOG.md +42 -0
- data/doc/README.md +230 -28
- data/lib/branchproof/cli.rb +191 -38
- data/lib/branchproof/comparison.rb +58 -7
- data/lib/branchproof/configuration.rb +111 -0
- data/lib/branchproof/coverage_policy.rb +169 -0
- data/lib/branchproof/evidence.rb +41 -2
- data/lib/branchproof/focused_report.rb +149 -27
- data/lib/branchproof/project.rb +49 -2
- data/lib/branchproof/rails_support.rb +101 -0
- data/lib/branchproof/report.rb +118 -10
- data/lib/branchproof/report_selection.rb +158 -0
- data/lib/branchproof/rspec_adapter.rb +423 -0
- data/lib/branchproof/runtime.rb +4 -0
- data/lib/branchproof/saved_report.rb +38 -5
- data/lib/branchproof/version.rb +1 -1
- data/lib/branchproof/worker.rb +75 -15
- data/lib/branchproof.rb +4 -0
- data/llms.txt +11 -1
- data/sig/branchproof.rbs +63 -4
- metadata +16 -2
data/lib/branchproof/cli.rb
CHANGED
|
@@ -33,10 +33,10 @@ module Branchproof
|
|
|
33
33
|
|
|
34
34
|
inventory = build_inventory(options)
|
|
35
35
|
evidence = empty_evidence(inventory, options)
|
|
36
|
-
baseline = if options
|
|
37
|
-
{ status: "INCOMPLETE", executed_tests: 0, failed_tests: 0, skipped_tests: 0, finalized: false }
|
|
38
|
-
else
|
|
36
|
+
baseline = if run_worker?(options)
|
|
39
37
|
run_worker(options, inventory, evidence)
|
|
38
|
+
else
|
|
39
|
+
{ status: "INCOMPLETE", executed_tests: 0, failed_tests: 0, skipped_tests: 0, finalized: false }
|
|
40
40
|
end
|
|
41
41
|
if value(baseline, :evidence)
|
|
42
42
|
merge_status = evidence.merge(snapshot: value(baseline, :evidence))
|
|
@@ -77,7 +77,8 @@ module Branchproof
|
|
|
77
77
|
report = Report.new(inventory: inventory, evidence: value(baseline, :evidence) || snapshot,
|
|
78
78
|
analysis: analysis, minima: minima, baseline: baseline, diagnostics: diagnostics,
|
|
79
79
|
level: options[:level], missing_only: options[:missing_only], view: options[:view],
|
|
80
|
-
run_metadata: run_metadata(options, baseline)
|
|
80
|
+
run_metadata: run_metadata(options, baseline), minimum: options[:minimum],
|
|
81
|
+
focus: options[:focus], top: options[:top])
|
|
81
82
|
output_report(report, options)
|
|
82
83
|
report.exit_code
|
|
83
84
|
rescue ArgumentError => e
|
|
@@ -93,11 +94,14 @@ module Branchproof
|
|
|
93
94
|
def help
|
|
94
95
|
@stdout.write(<<~HELP)
|
|
95
96
|
Usage:
|
|
96
|
-
branchproof analyze [SOURCE_GLOB ...] [--test TEST_GLOB] [--project auto|ruby|rails]
|
|
97
|
-
[--view decisions|conditions|tests|decision-tables] [--level 1|2|3] [--missing-only]
|
|
98
|
-
[--
|
|
97
|
+
branchproof analyze [SOURCE_GLOB ...] [--test TEST_GLOB] [--project auto|ruby|rails] [--framework auto|minitest|rspec]
|
|
98
|
+
[--view decisions|conditions|tests|decision-tables] [--level 1|2|3] [--missing-only] [--minimum CRITERION=THRESHOLD]
|
|
99
|
+
[--focus PATH[:LINE]] [--top N]
|
|
100
|
+
[--format terminal|json] [--output PATH] [--limits PATH] [--config PATH|--no-config]
|
|
101
|
+
[--no-reachability] [-- RUNNER_ARGS]
|
|
99
102
|
branchproof report SNAPSHOT [--view decisions|conditions|tests|decision-tables]
|
|
100
|
-
[--level 1|2|3] [--missing-only] [--
|
|
103
|
+
[--level 1|2|3] [--missing-only] [--minimum CRITERION=THRESHOLD] [--focus PATH[:LINE]] [--top N]
|
|
104
|
+
[--format terminal|json] [--output PATH]
|
|
101
105
|
branchproof compare BEFORE AFTER [--format terminal|json] [--output PATH] [--fail-on-regression]
|
|
102
106
|
mcdc accepts the same commands as a compatibility alias.
|
|
103
107
|
JSON always contains full evidence; --view requires terminal output.
|
|
@@ -142,14 +146,15 @@ module Branchproof
|
|
|
142
146
|
end
|
|
143
147
|
|
|
144
148
|
report = Report.from_document(document: document, level: options[:level], view: options[:view],
|
|
145
|
-
missing_only: options[:missing_only]
|
|
149
|
+
missing_only: options[:missing_only], minimum: options[:minimum_overrides],
|
|
150
|
+
focus: options[:focus], top: options[:top])
|
|
146
151
|
output_report(report, options)
|
|
147
152
|
report.exit_code
|
|
148
153
|
end
|
|
149
154
|
end
|
|
150
155
|
|
|
151
156
|
def parse_offline(command, args)
|
|
152
|
-
options = { format: :terminal, view: :decisions, missing_only: false }
|
|
157
|
+
options = { format: :terminal, view: :decisions, missing_only: false, minimum_overrides: {} }
|
|
153
158
|
paths = []
|
|
154
159
|
until args.empty?
|
|
155
160
|
token = args.shift
|
|
@@ -166,7 +171,7 @@ module Branchproof
|
|
|
166
171
|
raise ArgumentError, "--fail-on-regression requires compare" unless command == "compare"
|
|
167
172
|
|
|
168
173
|
options[:fail_on_regression] = true
|
|
169
|
-
when "--view", "--level", "--missing-only"
|
|
174
|
+
when "--view", "--level", "--missing-only", "--minimum", "--focus", "--top"
|
|
170
175
|
raise ArgumentError, "#{token} requires report" unless command == "report"
|
|
171
176
|
|
|
172
177
|
case token
|
|
@@ -176,6 +181,14 @@ module Branchproof
|
|
|
176
181
|
when "--level"
|
|
177
182
|
options[:level] = Integer(args.shift.to_s, 10)
|
|
178
183
|
raise ArgumentError, "level must be 1, 2, or 3" unless (1..3).cover?(options[:level])
|
|
184
|
+
when "--minimum"
|
|
185
|
+
add_minimum_override!(options, args.shift)
|
|
186
|
+
when "--focus"
|
|
187
|
+
options[:focus] = args.shift
|
|
188
|
+
raise ArgumentError, "--focus requires PATH or PATH:LINE" if options[:focus].nil? || options[:focus].start_with?("-")
|
|
189
|
+
when "--top"
|
|
190
|
+
options[:top] = args.shift
|
|
191
|
+
raise ArgumentError, "--top requires a positive integer" if options[:top].nil?
|
|
179
192
|
else options[:missing_only] = true
|
|
180
193
|
end
|
|
181
194
|
else
|
|
@@ -188,6 +201,7 @@ module Branchproof
|
|
|
188
201
|
raise ArgumentError, "#{command} requires #{expected} saved report #{expected == 1 ? "path" : "paths"}" unless paths.length == expected
|
|
189
202
|
|
|
190
203
|
validate_view!(options)
|
|
204
|
+
validate_selection!(options)
|
|
191
205
|
[options, paths]
|
|
192
206
|
end
|
|
193
207
|
|
|
@@ -205,12 +219,31 @@ module Branchproof
|
|
|
205
219
|
source = value(test, :source) || {}
|
|
206
220
|
[value(test, :id), { relative_path: relative_path(value(source, :path), root), line: value(source, :line) }]
|
|
207
221
|
end
|
|
208
|
-
|
|
222
|
+
project_metadata = value(baseline, :project) || options[:project]
|
|
223
|
+
selected_test_files = value(baseline, :selected_test_files)
|
|
224
|
+
test_files = selected_test_files || options[:tests]
|
|
225
|
+
metadata = {
|
|
226
|
+
captured_at: Time.now.utc.iso8601, requested_level: options[:level], project_kind: options[:project][:kind],
|
|
209
227
|
project_root: root, source_patterns: options[:source_patterns].map { |path| relative_path(path, root) },
|
|
228
|
+
exclude_patterns: Array(options[:exclude]).map { |path| relative_pattern(path, root) },
|
|
210
229
|
test_patterns: options[:test_patterns].map { |path| relative_path(path, root) },
|
|
211
|
-
test_files:
|
|
230
|
+
test_files: Array(test_files).map { |path| relative_path(path, root) }, runner_args: options[:runner_args],
|
|
212
231
|
seed: value(baseline, :seed), limits: options[:limits], test_locations: locations,
|
|
213
|
-
reachability: options[:reachability]
|
|
232
|
+
reachability: options[:reachability]
|
|
233
|
+
}
|
|
234
|
+
if options[:configuration]
|
|
235
|
+
metadata[:excluded_files] = Array(options[:excluded_files]).map { |path| relative_path(path, root) }
|
|
236
|
+
metadata[:selected_source_files] = Array(options[:selected_source_files]).map do |path|
|
|
237
|
+
relative_path(path, root)
|
|
238
|
+
end
|
|
239
|
+
end
|
|
240
|
+
%i[selected_test_files selected_example_ids].each do |key|
|
|
241
|
+
metadata[key] = value(baseline, key) if value(baseline, key)
|
|
242
|
+
end
|
|
243
|
+
%i[framework framework_version rspec_rails_version rails_version].each do |key|
|
|
244
|
+
metadata[key] = value(project_metadata, key) if value(project_metadata, key)
|
|
245
|
+
end
|
|
246
|
+
metadata
|
|
214
247
|
end
|
|
215
248
|
|
|
216
249
|
def relative_path(path, root)
|
|
@@ -219,6 +252,10 @@ module Branchproof
|
|
|
219
252
|
Pathname.new(File.expand_path(path, root)).relative_path_from(Pathname.new(root)).to_s
|
|
220
253
|
end
|
|
221
254
|
|
|
255
|
+
def relative_pattern(path, root)
|
|
256
|
+
path.to_s.empty? ? path.to_s : relative_path(path, root)
|
|
257
|
+
end
|
|
258
|
+
|
|
222
259
|
def parse(argv)
|
|
223
260
|
return nil if argv.empty? || argv.first != "analyze"
|
|
224
261
|
|
|
@@ -228,7 +265,9 @@ module Branchproof
|
|
|
228
265
|
args = args[0...delimiter] if delimiter
|
|
229
266
|
options = { level: 3, format: :terminal, output: nil, tests: [], source_paths: [], limits: Limits.default,
|
|
230
267
|
runner_args: runner_args, project: nil, missing_only: false, view: :decisions,
|
|
231
|
-
reachability: true
|
|
268
|
+
reachability: true, project_mode: "auto", framework: "auto", explicit_tests: false,
|
|
269
|
+
explicit_project: false, explicit_framework: false, explicit_sources: false,
|
|
270
|
+
config_path: nil, config_disabled: false, minimum_overrides: {} }
|
|
232
271
|
until args.empty?
|
|
233
272
|
token = args.shift
|
|
234
273
|
case token
|
|
@@ -237,6 +276,14 @@ module Branchproof
|
|
|
237
276
|
options[:explicit_view] = true
|
|
238
277
|
when "--missing-only"
|
|
239
278
|
options[:missing_only] = true
|
|
279
|
+
when "--minimum"
|
|
280
|
+
add_minimum_override!(options, args.shift)
|
|
281
|
+
when "--focus"
|
|
282
|
+
options[:focus] = args.shift
|
|
283
|
+
raise ArgumentError, "--focus requires PATH or PATH:LINE" if options[:focus].nil? || options[:focus].start_with?("-")
|
|
284
|
+
when "--top"
|
|
285
|
+
options[:top] = args.shift
|
|
286
|
+
raise ArgumentError, "--top requires a positive integer" if options[:top].nil?
|
|
240
287
|
when "--no-reachability"
|
|
241
288
|
options[:reachability] = false
|
|
242
289
|
when "--level"
|
|
@@ -255,11 +302,25 @@ module Branchproof
|
|
|
255
302
|
when "--test"
|
|
256
303
|
options[:tests] << args.shift
|
|
257
304
|
raise ArgumentError, "--test requires a glob" if options[:tests].last.nil? || options[:tests].last.empty?
|
|
305
|
+
|
|
306
|
+
options[:explicit_tests] = true
|
|
307
|
+
when "--framework"
|
|
308
|
+
options[:framework] = args.shift
|
|
309
|
+
raise ArgumentError, "--framework requires auto, minitest, or rspec" if options[:framework].nil? || options[:framework].empty?
|
|
310
|
+
|
|
311
|
+
options[:explicit_framework] = true
|
|
258
312
|
when "--project"
|
|
259
|
-
|
|
260
|
-
raise ArgumentError, "--project requires auto, ruby, or rails" if
|
|
313
|
+
options[:project_mode] = args.shift
|
|
314
|
+
raise ArgumentError, "--project requires auto, ruby, or rails" if options[:project_mode].nil? || options[:project_mode].empty?
|
|
315
|
+
|
|
316
|
+
options[:explicit_project] = true
|
|
317
|
+
when "--config"
|
|
318
|
+
options[:config_path] = args.shift
|
|
319
|
+
raise ArgumentError, "--config requires a readable JSON path" if options[:config_path].nil? || options[:config_path].empty?
|
|
320
|
+
when "--no-config"
|
|
321
|
+
raise ArgumentError, "--config and --no-config are mutually exclusive" if options[:config_path]
|
|
261
322
|
|
|
262
|
-
options[:
|
|
323
|
+
options[:config_disabled] = true
|
|
263
324
|
when "--limits"
|
|
264
325
|
limits_path = args.shift
|
|
265
326
|
raise ArgumentError, "--limits requires a readable JSON path" unless limits_path && File.file?(limits_path)
|
|
@@ -271,23 +332,76 @@ module Branchproof
|
|
|
271
332
|
raise ArgumentError, "unknown option: #{token}" if token.start_with?("-")
|
|
272
333
|
|
|
273
334
|
options[:source_paths] << token
|
|
335
|
+
options[:explicit_sources] = true
|
|
274
336
|
end
|
|
275
337
|
end
|
|
338
|
+
root = Dir.pwd
|
|
339
|
+
if options[:config_path] && options[:config_disabled]
|
|
340
|
+
raise ArgumentError, "--config and --no-config are mutually exclusive"
|
|
341
|
+
end
|
|
342
|
+
|
|
343
|
+
configuration = Configuration.load(path: options[:config_path] || ".branchproof.json", root: root,
|
|
344
|
+
explicit: !options[:config_path].nil?, disabled: options[:config_disabled])
|
|
345
|
+
options[:configuration] = configuration
|
|
346
|
+
if configuration
|
|
347
|
+
options[:project_mode] = configuration[:project] if !options[:explicit_project] && configuration.key?(:project)
|
|
348
|
+
options[:framework] = configuration[:framework] if !options[:explicit_framework] && configuration.key?(:framework)
|
|
349
|
+
if !options[:explicit_sources] && configuration.key?(:sources)
|
|
350
|
+
options[:source_paths] = configuration[:sources].dup
|
|
351
|
+
end
|
|
352
|
+
if !options[:explicit_tests] && configuration.key?(:tests)
|
|
353
|
+
options[:tests] = configuration[:tests].dup
|
|
354
|
+
options[:explicit_tests] = true
|
|
355
|
+
end
|
|
356
|
+
options[:exclude] = Array(configuration[:exclude]).dup
|
|
357
|
+
options[:minimum] = configuration.fetch(:minimum, {}).dup.merge(options[:minimum_overrides])
|
|
358
|
+
else
|
|
359
|
+
options[:exclude] = []
|
|
360
|
+
options[:minimum] = options[:minimum_overrides].dup
|
|
361
|
+
end
|
|
362
|
+
options[:project] = Project.new(root: root, mode: options[:project_mode], framework: options[:framework]).to_h
|
|
276
363
|
validate_view!(options)
|
|
364
|
+
validate_selection!(options)
|
|
277
365
|
if options[:missing_only] && (options[:format] != :terminal || options[:level] == 1)
|
|
278
366
|
raise ArgumentError, "--missing-only requires terminal format and level 2 or 3"
|
|
279
367
|
end
|
|
280
368
|
|
|
281
|
-
options[:project] ||= Project.new(root: Dir.pwd, mode: "auto").to_h
|
|
282
369
|
options[:source_paths] = default_sources if options[:source_paths].empty?
|
|
283
|
-
options[:tests] = default_tests if options[:tests].empty?
|
|
370
|
+
options[:tests] = default_tests(options[:project]) if options[:tests].empty?
|
|
284
371
|
options[:source_patterns] = options[:source_paths].dup
|
|
285
|
-
options[:test_patterns] =
|
|
372
|
+
options[:test_patterns] = options[:explicit_tests] ? options[:tests].dup : options[:project][:test_patterns]
|
|
286
373
|
options[:tests] = expand_paths(options[:tests], root: options[:project][:root])
|
|
287
374
|
options
|
|
288
375
|
end
|
|
289
376
|
|
|
377
|
+
def add_minimum_override!(options, argument)
|
|
378
|
+
text = argument.to_s
|
|
379
|
+
match = text.match(/\A([a-z_]+)=([0-9]+(?:\.[0-9]+)?)\z/)
|
|
380
|
+
raise ArgumentError, "minimum must be CRITERION=THRESHOLD" unless match
|
|
381
|
+
|
|
382
|
+
criterion = match[1]
|
|
383
|
+
threshold_text = match[2]
|
|
384
|
+
threshold = threshold_text.include?(".") ? Float(threshold_text) : Integer(threshold_text, 10)
|
|
385
|
+
normalized = CoveragePolicy.normalize(criterion => threshold)
|
|
386
|
+
criterion = normalized.keys.first
|
|
387
|
+
raise ArgumentError, "duplicate coverage criterion: #{criterion}" if options[:minimum_overrides].key?(criterion)
|
|
388
|
+
|
|
389
|
+
options[:minimum_overrides][criterion] = normalized.fetch(criterion)
|
|
390
|
+
rescue ArgumentError
|
|
391
|
+
raise
|
|
392
|
+
rescue TypeError
|
|
393
|
+
raise ArgumentError, "minimum threshold must be a finite number from 0 to 100"
|
|
394
|
+
end
|
|
395
|
+
|
|
396
|
+
def validate_selection!(options)
|
|
397
|
+
return unless options[:focus] || options[:top]
|
|
398
|
+
raise ArgumentError, "focus and top filters are terminal-only" if options[:format] == :json
|
|
399
|
+
|
|
400
|
+
ReportSelection.new(focus: options[:focus], top: options[:top])
|
|
401
|
+
end
|
|
402
|
+
|
|
290
403
|
def build_inventory(options)
|
|
404
|
+
root = options[:project][:root]
|
|
291
405
|
test_paths = options[:tests].filter_map do |path|
|
|
292
406
|
File.realpath(path)
|
|
293
407
|
rescue StandardError
|
|
@@ -298,13 +412,21 @@ module Branchproof
|
|
|
298
412
|
rescue StandardError
|
|
299
413
|
nil
|
|
300
414
|
end
|
|
301
|
-
|
|
302
|
-
|
|
415
|
+
excluded = expand_paths(Array(options[:exclude]).reject(&:empty?), root: root)
|
|
416
|
+
excluded_paths = excluded.filter_map do |path|
|
|
417
|
+
File.realpath(path)
|
|
418
|
+
rescue StandardError
|
|
419
|
+
nil
|
|
420
|
+
end
|
|
421
|
+
selected = expand_paths(options[:source_paths], root: root).reject do |path|
|
|
422
|
+
relative = Pathname.new(path).relative_path_from(Pathname.new(root)).to_s
|
|
303
423
|
canonical = File.realpath(path)
|
|
304
424
|
relative.match?(%r{\A(?:test|spec|tool|vendor)(?:/|\z)}) ||
|
|
305
|
-
test_paths.include?(canonical) || loaded_paths.include?(canonical)
|
|
425
|
+
test_paths.include?(canonical) || loaded_paths.include?(canonical) || excluded_paths.include?(canonical)
|
|
306
426
|
end
|
|
307
|
-
|
|
427
|
+
options[:excluded_files] = excluded
|
|
428
|
+
options[:selected_source_files] = selected
|
|
429
|
+
Source.new(root: root, limits: options[:limits]).inventory(paths: selected)
|
|
308
430
|
end
|
|
309
431
|
|
|
310
432
|
def empty_evidence(inventory, options)
|
|
@@ -315,18 +437,10 @@ module Branchproof
|
|
|
315
437
|
begin
|
|
316
438
|
directory = Dir.mktmpdir("branchproof-run-")
|
|
317
439
|
config = File.join(directory, "request.json")
|
|
318
|
-
payload =
|
|
319
|
-
inventory: inventory, limits: options[:limits], run_id: evidence.run_id,
|
|
320
|
-
test_files: options[:tests], runner_args: options[:runner_args], project: options[:project],
|
|
321
|
-
result_path: File.join(directory, "result.json"), marker_path: File.join(directory, "complete.marker")
|
|
322
|
-
}
|
|
440
|
+
payload = worker_payload(options, inventory, evidence, directory)
|
|
323
441
|
File.binwrite(config, JSON.generate(normalize(payload)))
|
|
324
442
|
script = "require 'branchproof'; exit(Branchproof::Worker.child_process(ARGV.fetch(0)).to_i)"
|
|
325
|
-
|
|
326
|
-
File.expand_path("..", __dir__), "-e", script, config,
|
|
327
|
-
chdir: options[:project][:root])
|
|
328
|
-
@stderr.write(child_stdout) unless child_stdout.to_s.empty?
|
|
329
|
-
@stderr.write(stderr) unless stderr.to_s.empty?
|
|
443
|
+
stderr, child_status = stream_worker(options, script, config)
|
|
330
444
|
unless File.file?(payload[:marker_path]) && File.file?(payload[:result_path])
|
|
331
445
|
return { status: "ERROR", executed_tests: 0, failed_tests: 0, skipped_tests: 0, finalized: false,
|
|
332
446
|
diagnostics: [{ code: "worker_incomplete", severity: "error", message: stderr.to_s.strip }] }
|
|
@@ -346,6 +460,30 @@ module Branchproof
|
|
|
346
460
|
diagnostics: [{ code: "runner", severity: "error", message: e.message }] }
|
|
347
461
|
end
|
|
348
462
|
|
|
463
|
+
def stream_worker(options, script, config)
|
|
464
|
+
diagnostic_output = +""
|
|
465
|
+
Open3.popen3(options[:project][:environment], RbConfig.ruby, "-I",
|
|
466
|
+
File.expand_path("..", __dir__), "-e", script, config,
|
|
467
|
+
chdir: options[:project][:root]) do |input, output, errors, child|
|
|
468
|
+
input.close
|
|
469
|
+
streams = [output, errors]
|
|
470
|
+
until streams.empty?
|
|
471
|
+
IO.select(streams).first.each do |stream|
|
|
472
|
+
chunk = stream.read_nonblock(16_384, exception: false)
|
|
473
|
+
if chunk.nil?
|
|
474
|
+
streams.delete(stream)
|
|
475
|
+
elsif chunk != :wait_readable
|
|
476
|
+
@stderr.write(chunk)
|
|
477
|
+
@stderr.flush if @stderr.respond_to?(:flush)
|
|
478
|
+
diagnostic_output << chunk
|
|
479
|
+
diagnostic_output = diagnostic_output.byteslice(-16_384, 16_384) if diagnostic_output.bytesize > 16_384
|
|
480
|
+
end
|
|
481
|
+
end
|
|
482
|
+
end
|
|
483
|
+
[diagnostic_output, child.value]
|
|
484
|
+
end
|
|
485
|
+
end
|
|
486
|
+
|
|
349
487
|
def output_report(report, options)
|
|
350
488
|
created = false
|
|
351
489
|
if options[:output]
|
|
@@ -362,6 +500,19 @@ module Branchproof
|
|
|
362
500
|
File.unlink(temporary) if created && temporary && File.file?(temporary)
|
|
363
501
|
end
|
|
364
502
|
|
|
503
|
+
def run_worker?(options)
|
|
504
|
+
!options[:tests].empty? || options[:project][:framework].to_s == "rspec"
|
|
505
|
+
end
|
|
506
|
+
|
|
507
|
+
def worker_payload(options, inventory, evidence, directory)
|
|
508
|
+
{
|
|
509
|
+
inventory: inventory, limits: options[:limits], run_id: evidence.run_id,
|
|
510
|
+
test_files: options[:tests], runner_args: options[:runner_args], project: options[:project],
|
|
511
|
+
test_selection_explicit: options[:explicit_tests],
|
|
512
|
+
result_path: File.join(directory, "result.json"), marker_path: File.join(directory, "complete.marker")
|
|
513
|
+
}
|
|
514
|
+
end
|
|
515
|
+
|
|
365
516
|
def report_string(report, format)
|
|
366
517
|
buffer = StringIO.new
|
|
367
518
|
report.write(io: buffer, format: format)
|
|
@@ -377,8 +528,9 @@ module Branchproof
|
|
|
377
528
|
%w[lib/**/*.rb app/**/*.rb]
|
|
378
529
|
end
|
|
379
530
|
|
|
380
|
-
def default_tests
|
|
381
|
-
|
|
531
|
+
def default_tests(project = nil)
|
|
532
|
+
project ||= Project.new(root: Dir.pwd, mode: "auto", framework: "auto").to_h
|
|
533
|
+
candidates = Array(project[:test_patterns]).flat_map { |pattern| Dir.glob(pattern, base: project[:root]) }
|
|
382
534
|
candidates.map! { |path| File.expand_path(path, Dir.pwd) }
|
|
383
535
|
candidates.reject { |path| default_test_excluded?(path) }.uniq.sort
|
|
384
536
|
end
|
|
@@ -387,7 +539,8 @@ module Branchproof
|
|
|
387
539
|
relative = Pathname.new(path).relative_path_from(Pathname.new(Dir.pwd)).to_s
|
|
388
540
|
segments = relative.split(File::SEPARATOR)
|
|
389
541
|
basename = File.basename(path)
|
|
390
|
-
|
|
542
|
+
%w[test_helper.rb spec_helper.rb rails_helper.rb].include?(basename) ||
|
|
543
|
+
segments.include?("support") || segments.include?("fixtures")
|
|
391
544
|
end
|
|
392
545
|
|
|
393
546
|
def expand_paths(paths, root: Dir.pwd)
|
|
@@ -2,12 +2,13 @@
|
|
|
2
2
|
|
|
3
3
|
# rubocop:disable Metrics/ClassLength, Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
|
|
4
4
|
require "json"
|
|
5
|
+
require "pathname"
|
|
5
6
|
|
|
6
7
|
module Branchproof
|
|
7
8
|
# Compares two complete report documents without loading or executing the project.
|
|
8
9
|
class Comparison
|
|
9
10
|
SCHEMA_VERSION = "1.0"
|
|
10
|
-
SUPPORTED_REPORT_SCHEMAS = %w[1.0 1.1 1.2 1.3].freeze
|
|
11
|
+
SUPPORTED_REPORT_SCHEMAS = %w[1.0 1.1 1.2 1.3 1.4].freeze
|
|
11
12
|
CRITERION_VERSION = "masking_occurrence_v1"
|
|
12
13
|
|
|
13
14
|
def initialize(before:, after:)
|
|
@@ -369,9 +370,17 @@ module Branchproof
|
|
|
369
370
|
tests = Array(value(value(document, :observations), :tests)).to_h { |test| [value(test, :id).to_s, test] }
|
|
370
371
|
rows = index.tests.to_h { |test| [test[:id].to_s, test] }
|
|
371
372
|
keys = tests.to_h do |id, test|
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
373
|
+
adapter = value(test, :adapter).to_s
|
|
374
|
+
path = rows.dig(id, :relative_path)
|
|
375
|
+
key = if adapter.downcase == "rspec"
|
|
376
|
+
example_id = value(test, :example_id).to_s
|
|
377
|
+
# Scoped IDs are positional selectors, valid only within the same spec revision.
|
|
378
|
+
[adapter, rspec_example_path(document, example_id) || path, rspec_scope(example_id),
|
|
379
|
+
value(test, :name), path, value(test, :spec_digest), value(test, :source_digest)]
|
|
380
|
+
else
|
|
381
|
+
[adapter, value(test, :class_name), value(test, :method_name), path]
|
|
382
|
+
end
|
|
383
|
+
[id, key.all? { |part| part.is_a?(String) && !part.empty? } ? key : nil]
|
|
375
384
|
end
|
|
376
385
|
results = Array(value(value(document, :analysis), :decisions)).flat_map do |decision|
|
|
377
386
|
Array(value(decision, :condition_results))
|
|
@@ -447,6 +456,13 @@ module Branchproof
|
|
|
447
456
|
if context(@before)[:test_keys].values.sort_by(&:to_s) != context(@after)[:test_keys].values.sort_by(&:to_s)
|
|
448
457
|
changes << "test population differs"
|
|
449
458
|
end
|
|
459
|
+
if [@before, @after].any? do |document|
|
|
460
|
+
context(document)[:tests].any? do |id, test|
|
|
461
|
+
value(test, :adapter).to_s.downcase == "rspec" && context(document)[:test_keys][id].nil?
|
|
462
|
+
end
|
|
463
|
+
end
|
|
464
|
+
changes << "RSpec example identity is uncertain: missing spec revision or selector evidence"
|
|
465
|
+
end
|
|
450
466
|
changes << "Deltas describe observed runs; nondeterminism and external environment changes are not ruled out."
|
|
451
467
|
changes
|
|
452
468
|
end
|
|
@@ -454,9 +470,10 @@ module Branchproof
|
|
|
454
470
|
def comparability_reasons(changed_paths)
|
|
455
471
|
reasons = []
|
|
456
472
|
reasons << "source files changed: #{changed_paths.sort.join(", ")}" unless changed_paths.empty?
|
|
457
|
-
%i[project_kind runner_args source_patterns test_patterns limits
|
|
458
|
-
|
|
459
|
-
|
|
473
|
+
%i[project_kind framework framework_version rspec_rails_version runner_args source_patterns test_patterns limits
|
|
474
|
+
runtime].each do |key|
|
|
475
|
+
left = key == :framework ? comparison_framework(@before) : metadata(@before, key)
|
|
476
|
+
right = key == :framework ? comparison_framework(@after) : metadata(@after, key)
|
|
460
477
|
if key == :runner_args
|
|
461
478
|
left = comparable_runner_args(left)
|
|
462
479
|
right = comparable_runner_args(right)
|
|
@@ -464,13 +481,47 @@ module Branchproof
|
|
|
464
481
|
reasons << "#{key} differs" if !left.nil? && !right.nil? && left != right
|
|
465
482
|
end
|
|
466
483
|
reasons << "source selection differs" if source_selection(@before) != source_selection(@after)
|
|
484
|
+
reasons << "source exclusion scope differs" if exclusion_scope(@before) != exclusion_scope(@after)
|
|
467
485
|
reasons
|
|
468
486
|
end
|
|
469
487
|
|
|
488
|
+
def comparison_framework(document)
|
|
489
|
+
explicit = metadata(document, :framework)
|
|
490
|
+
return explicit unless explicit.nil? || explicit.to_s.empty?
|
|
491
|
+
|
|
492
|
+
adapters = Array(value(value(document, :observations), :tests)).filter_map do |test|
|
|
493
|
+
adapter = value(test, :adapter).to_s.downcase
|
|
494
|
+
adapter unless adapter.empty? || adapter == "unknown"
|
|
495
|
+
end.uniq
|
|
496
|
+
adapters.length == 1 ? adapters.first : nil
|
|
497
|
+
end
|
|
498
|
+
|
|
499
|
+
def rspec_scope(example_id)
|
|
500
|
+
match = example_id.to_s.match(/(\[[^\]]+\])\z/)
|
|
501
|
+
match && match[1]
|
|
502
|
+
end
|
|
503
|
+
|
|
504
|
+
def rspec_example_path(document, example_id)
|
|
505
|
+
path = example_id.to_s.sub(/\[[^\]]+\]\z/, "")
|
|
506
|
+
return nil if path.empty?
|
|
507
|
+
return path.sub(%r{\A\./}, "") unless path.start_with?("/")
|
|
508
|
+
|
|
509
|
+
root = value(value(document, :source_inventory), :root) || metadata(document, :project_root)
|
|
510
|
+
return nil unless root
|
|
511
|
+
|
|
512
|
+
Pathname.new(path).relative_path_from(Pathname.new(root.to_s)).to_s
|
|
513
|
+
rescue ArgumentError
|
|
514
|
+
nil
|
|
515
|
+
end
|
|
516
|
+
|
|
470
517
|
def source_selection(document)
|
|
471
518
|
value(value(document, :run_metadata), :source_patterns)
|
|
472
519
|
end
|
|
473
520
|
|
|
521
|
+
def exclusion_scope(document)
|
|
522
|
+
Array(value(value(document, :run_metadata), :exclude_patterns))
|
|
523
|
+
end
|
|
524
|
+
|
|
474
525
|
def metadata_requirements
|
|
475
526
|
[@before, @after].each_with_index.filter_map do |document, index|
|
|
476
527
|
metadata = value(document, :run_metadata)
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
require_relative "coverage_policy"
|
|
5
|
+
|
|
6
|
+
module Branchproof
|
|
7
|
+
# Loads and validates the project-local .branchproof.json policy.
|
|
8
|
+
class Configuration
|
|
9
|
+
SCHEMA_VERSION = 1
|
|
10
|
+
PROJECTS = %w[auto ruby rails].freeze
|
|
11
|
+
FRAMEWORKS = %w[auto minitest rspec].freeze
|
|
12
|
+
MINIMUM_CRITERIA = CoveragePolicy::CRITERIA.keys.freeze
|
|
13
|
+
FIELDS = %w[schema_version project framework sources tests exclude minimum].freeze
|
|
14
|
+
|
|
15
|
+
class << self
|
|
16
|
+
def load(path:, root:, explicit: false, disabled: false)
|
|
17
|
+
return nil if disabled
|
|
18
|
+
|
|
19
|
+
root = File.expand_path(root)
|
|
20
|
+
path = File.expand_path(path, root)
|
|
21
|
+
return nil unless ensure_file!(path, explicit)
|
|
22
|
+
|
|
23
|
+
payload = read_payload(path)
|
|
24
|
+
validate(payload, path: path, root: root)
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
private
|
|
28
|
+
|
|
29
|
+
def ensure_file!(path, explicit)
|
|
30
|
+
unless File.exist?(path)
|
|
31
|
+
return nil unless explicit
|
|
32
|
+
|
|
33
|
+
raise ArgumentError, "configuration file does not exist: #{path}"
|
|
34
|
+
end
|
|
35
|
+
return true if File.file?(path)
|
|
36
|
+
|
|
37
|
+
raise ArgumentError, "configuration path is not a file: #{path}"
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def read_payload(path)
|
|
41
|
+
JSON.parse(File.binread(path))
|
|
42
|
+
rescue JSON::ParserError => e
|
|
43
|
+
raise ArgumentError, "invalid JSON configuration: #{e.message}"
|
|
44
|
+
rescue SystemCallError => e
|
|
45
|
+
raise ArgumentError, "configuration could not be read: #{e.message}"
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
def validate(payload, path:, root:)
|
|
49
|
+
validate_shape!(payload)
|
|
50
|
+
validate_values!(payload)
|
|
51
|
+
|
|
52
|
+
payload.each_with_object({ schema_version: SCHEMA_VERSION, path: path, root: root }) do |(key, value), result|
|
|
53
|
+
next if key == "schema_version"
|
|
54
|
+
|
|
55
|
+
result[key.to_sym] = key == "minimum" ? value.transform_keys(&:to_s) : value
|
|
56
|
+
end
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
def validate_shape!(payload)
|
|
60
|
+
raise ArgumentError, "configuration must be a JSON object" unless payload.is_a?(Hash)
|
|
61
|
+
|
|
62
|
+
unknown = payload.keys - FIELDS
|
|
63
|
+
return if unknown.empty? && valid_schema_version?(payload["schema_version"])
|
|
64
|
+
|
|
65
|
+
raise ArgumentError, "configuration has unknown fields: #{unknown.join(", ")}" unless unknown.empty?
|
|
66
|
+
|
|
67
|
+
raise ArgumentError, "configuration schema_version must be #{SCHEMA_VERSION}"
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
def valid_schema_version?(value)
|
|
71
|
+
value.is_a?(Integer) && value == SCHEMA_VERSION
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
def validate_values!(payload)
|
|
75
|
+
validate_enum(payload, "project", PROJECTS)
|
|
76
|
+
validate_enum(payload, "framework", FRAMEWORKS)
|
|
77
|
+
%w[sources tests].each { |field| validate_nonempty_strings(payload, field) if payload.key?(field) }
|
|
78
|
+
validate_strings(payload, "exclude") if payload.key?("exclude")
|
|
79
|
+
validate_minimum(payload["minimum"]) if payload.key?("minimum")
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
def validate_enum(payload, field, values)
|
|
83
|
+
return unless payload.key?(field)
|
|
84
|
+
|
|
85
|
+
value = payload[field]
|
|
86
|
+
raise ArgumentError, "configuration #{field} must be a string" unless value.is_a?(String)
|
|
87
|
+
raise ArgumentError, "configuration #{field} must be #{values.join(", ")}" unless values.include?(value)
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
def validate_nonempty_strings(payload, field)
|
|
91
|
+
value = payload[field]
|
|
92
|
+
valid = value.is_a?(Array) && !value.empty? && value.all? { |item| item.is_a?(String) && !item.empty? }
|
|
93
|
+
raise ArgumentError, "configuration #{field} must be a nonempty array of strings" unless valid
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
def validate_strings(payload, field)
|
|
97
|
+
value = payload[field]
|
|
98
|
+
valid = value.is_a?(Array) && value.all? { |item| item.is_a?(String) && !item.empty? }
|
|
99
|
+
raise ArgumentError, "configuration #{field} must be an array of strings" unless valid
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
def validate_minimum(value)
|
|
103
|
+
raise ArgumentError, "configuration minimum must be an object" unless value.is_a?(Hash)
|
|
104
|
+
|
|
105
|
+
CoveragePolicy.normalize(value)
|
|
106
|
+
rescue ArgumentError => e
|
|
107
|
+
raise ArgumentError, "configuration minimum #{e.message.sub(/\Acoverage minimum for /, "")}"
|
|
108
|
+
end
|
|
109
|
+
end
|
|
110
|
+
end
|
|
111
|
+
end
|