branchproof 0.9.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +42 -0
- data/README.md +230 -28
- data/doc/Branchproof/Configuration.md +28 -0
- data/doc/Branchproof/CoveragePolicy.md +27 -0
- data/doc/Branchproof/Evidence.md +3 -0
- data/doc/Branchproof/FocusedReport.md +1 -1
- data/doc/Branchproof/Project.md +4 -1
- data/doc/Branchproof/RSpecAdapter/ClassRunnerGuard.md +11 -0
- data/doc/Branchproof/RSpecAdapter/ContextLifecycle.md +14 -0
- data/doc/Branchproof/RSpecAdapter/DefaultDiscovery.md +11 -0
- data/doc/Branchproof/RSpecAdapter/ExampleLifecycle.md +11 -0
- data/doc/Branchproof/RSpecAdapter/RunnerGuard.md +11 -0
- data/doc/Branchproof/RSpecAdapter/UnsupportedRunner.md +12 -0
- data/doc/Branchproof/RSpecAdapter.md +47 -0
- data/doc/Branchproof/RailsSupport.md +35 -2
- data/doc/Branchproof/Report.md +11 -2
- data/doc/Branchproof/ReportSelection.md +42 -0
- data/doc/Branchproof/Runtime.md +3 -0
- data/doc/Branchproof/Worker.md +8 -1
- data/doc/Branchproof.md +11 -1
- data/doc/CHANGELOG.md +42 -0
- data/doc/README.md +230 -28
- data/lib/branchproof/cli.rb +191 -38
- data/lib/branchproof/comparison.rb +58 -7
- data/lib/branchproof/configuration.rb +111 -0
- data/lib/branchproof/coverage_policy.rb +169 -0
- data/lib/branchproof/evidence.rb +41 -2
- data/lib/branchproof/focused_report.rb +149 -27
- data/lib/branchproof/project.rb +49 -2
- data/lib/branchproof/rails_support.rb +101 -0
- data/lib/branchproof/report.rb +118 -10
- data/lib/branchproof/report_selection.rb +158 -0
- data/lib/branchproof/rspec_adapter.rb +423 -0
- data/lib/branchproof/runtime.rb +4 -0
- data/lib/branchproof/saved_report.rb +38 -5
- data/lib/branchproof/version.rb +1 -1
- data/lib/branchproof/worker.rb +75 -15
- data/lib/branchproof.rb +4 -0
- data/llms.txt +11 -1
- data/sig/branchproof.rbs +63 -4
- metadata +16 -2
|
@@ -0,0 +1,423 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "records" unless defined?(Branchproof::Records)
|
|
4
|
+
|
|
5
|
+
module Branchproof
|
|
6
|
+
# Bridges one serial RSpec run to Runtime lifecycle ownership.
|
|
7
|
+
class RSpecAdapter
|
|
8
|
+
# Identifies runner capabilities that cannot produce supported evidence.
|
|
9
|
+
class UnsupportedRunner < ArgumentError
|
|
10
|
+
def diagnostic_code = "unsupported_runner"
|
|
11
|
+
end
|
|
12
|
+
|
|
13
|
+
# Restores CLI directory discovery when RSpec runs inside the worker.
|
|
14
|
+
module DefaultDiscovery
|
|
15
|
+
def configure(configuration)
|
|
16
|
+
options[:files_or_directories_to_run] = [options[:default_path] || configuration.default_path].compact
|
|
17
|
+
super
|
|
18
|
+
end
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
# Rejects additional class-level runner entry points.
|
|
22
|
+
module ClassRunnerGuard
|
|
23
|
+
def run(*args, &)
|
|
24
|
+
RSpecAdapter.runner_adapter&.reject_execution!("nested or repeated RSpec runs are unsupported")
|
|
25
|
+
super
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
# Allows only the worker-owned native runner instance.
|
|
30
|
+
module RunnerGuard
|
|
31
|
+
def run(*args, &)
|
|
32
|
+
RSpecAdapter.runner_adapter&.enter_runner!(self)
|
|
33
|
+
super
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
# Assigns observation phases while the reporter owns test registration.
|
|
38
|
+
module ExampleLifecycle
|
|
39
|
+
def run(*args, &)
|
|
40
|
+
adapter = RSpecAdapter.active_adapter
|
|
41
|
+
return super unless adapter
|
|
42
|
+
|
|
43
|
+
adapter.send(:begin_example, self)
|
|
44
|
+
super
|
|
45
|
+
ensure
|
|
46
|
+
adapter&.send(:unattributed)
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
private
|
|
50
|
+
|
|
51
|
+
def run_before_example(*args, &)
|
|
52
|
+
adapter = RSpecAdapter.active_adapter
|
|
53
|
+
adapter&.send(:begin_attempt, self)
|
|
54
|
+
result = super
|
|
55
|
+
adapter&.send(:phase, self, "body")
|
|
56
|
+
result
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
def run_after_example(*args, &)
|
|
60
|
+
RSpecAdapter.active_adapter&.send(:phase, self, "teardown")
|
|
61
|
+
super
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
# Keeps group-level hooks outside individual example ownership.
|
|
66
|
+
module ContextLifecycle
|
|
67
|
+
def run_before_context_hooks(*args, &)
|
|
68
|
+
adapter = RSpecAdapter.active_adapter
|
|
69
|
+
return super unless adapter
|
|
70
|
+
|
|
71
|
+
adapter.send(:without_owner) { super }
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
def run_after_context_hooks(*args, &)
|
|
75
|
+
adapter = RSpecAdapter.active_adapter
|
|
76
|
+
return super unless adapter
|
|
77
|
+
|
|
78
|
+
adapter.send(:without_owner) { super }
|
|
79
|
+
end
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
class << self
|
|
83
|
+
attr_accessor :active_adapter, :runner_adapter
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
FORBIDDEN_OPTIONS = {
|
|
87
|
+
dry_run: "dry-run",
|
|
88
|
+
bisect: "bisect",
|
|
89
|
+
drb: "DRb",
|
|
90
|
+
runner: "custom runner"
|
|
91
|
+
}.freeze
|
|
92
|
+
|
|
93
|
+
attr_reader :tests, :late_execution_error
|
|
94
|
+
|
|
95
|
+
def initialize(runtime:)
|
|
96
|
+
@runtime = runtime
|
|
97
|
+
@tests = {}
|
|
98
|
+
@example_records = {}.compare_by_identity
|
|
99
|
+
@file_digests = {}
|
|
100
|
+
@running = false
|
|
101
|
+
@completed = false
|
|
102
|
+
@completion_started = false
|
|
103
|
+
@registered = false
|
|
104
|
+
@current_context = nil
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
# rubocop:disable-next Metrics/ParameterLists
|
|
108
|
+
def run(test_files:, runner_args:, on_complete:, before_load: nil, after_load: nil,
|
|
109
|
+
test_selection_explicit: false)
|
|
110
|
+
raise ArgumentError, "test_files must be an Array" unless test_files.is_a?(Array)
|
|
111
|
+
raise ArgumentError, "runner_args must be an Array" unless runner_args.is_a?(Array)
|
|
112
|
+
raise ArgumentError, "on_complete must respond to call" unless on_complete.respond_to?(:call)
|
|
113
|
+
raise UnsupportedRunner, "RSpec adapter cannot run more than once" if @running || @registered
|
|
114
|
+
raise UnsupportedRunner, "nested RSpec runs are unsupported" if self.class.active_adapter
|
|
115
|
+
raise UnsupportedRunner, "TEST_ENV_NUMBER is unsupported" if ENV.key?("TEST_ENV_NUMBER")
|
|
116
|
+
|
|
117
|
+
require "rspec/core"
|
|
118
|
+
require "rspec/core/runner"
|
|
119
|
+
unless Gem::Requirement.new("~> 3.13.0").satisfied_by?(Gem::Version.new(RSpec::Core::Version::STRING))
|
|
120
|
+
raise UnsupportedRunner, "unsupported RSpec version #{RSpec::Core::Version::STRING}; use 3.13.x"
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
RSpec::Core::Runner.disable_autorun!
|
|
124
|
+
options = RSpec::Core::ConfigurationOptions.new(runner_args)
|
|
125
|
+
configured_files = Array(options.options[:files_or_directories_to_run]).map(&:to_s)
|
|
126
|
+
if test_selection_explicit && configured_files.any?
|
|
127
|
+
raise ArgumentError, "explicit test selection conflicts with configured RSpec selectors"
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
options.options[:files_or_directories_to_run] = test_files.map(&:to_s) if test_selection_explicit
|
|
131
|
+
options.singleton_class.prepend(DefaultDiscovery) if !test_selection_explicit && configured_files.empty?
|
|
132
|
+
reject_options!(options)
|
|
133
|
+
|
|
134
|
+
@running = true
|
|
135
|
+
|
|
136
|
+
@registered = true
|
|
137
|
+
@completion_callback = on_complete
|
|
138
|
+
@unmatched_test_selection = test_selection_explicit && test_files.empty?
|
|
139
|
+
if @unmatched_test_selection
|
|
140
|
+
complete(status: 0, finalized: true)
|
|
141
|
+
return 0
|
|
142
|
+
end
|
|
143
|
+
self.class.active_adapter = self
|
|
144
|
+
self.class.runner_adapter = self
|
|
145
|
+
install_lifecycle_hooks
|
|
146
|
+
install_runner_guard
|
|
147
|
+
runner = RSpec::Core::Runner.new(options)
|
|
148
|
+
@native_runner = runner
|
|
149
|
+
adapter = self
|
|
150
|
+
runner.define_singleton_method(:setup) do |*setup_args|
|
|
151
|
+
super(*setup_args)
|
|
152
|
+
return if world.wants_to_quit
|
|
153
|
+
|
|
154
|
+
adapter.validate_runner!
|
|
155
|
+
after_load&.call
|
|
156
|
+
adapter.send(:install_reporter_listener, configuration.reporter)
|
|
157
|
+
end
|
|
158
|
+
before_load&.call
|
|
159
|
+
status = runner.run($stderr, $stdout)
|
|
160
|
+
capture_runner_state(runner)
|
|
161
|
+
complete(status: status, finalized: true)
|
|
162
|
+
status
|
|
163
|
+
rescue StandardError => e
|
|
164
|
+
raise unless @running
|
|
165
|
+
raise if @completion_started
|
|
166
|
+
|
|
167
|
+
@run_error = e
|
|
168
|
+
capture_runner_state(runner) if runner
|
|
169
|
+
status = 2
|
|
170
|
+
complete(status: status, finalized: false)
|
|
171
|
+
status
|
|
172
|
+
ensure
|
|
173
|
+
if self.class.active_adapter.equal?(self)
|
|
174
|
+
unattributed
|
|
175
|
+
self.class.active_adapter = nil
|
|
176
|
+
end
|
|
177
|
+
@running = false
|
|
178
|
+
end
|
|
179
|
+
|
|
180
|
+
def validate_runner!
|
|
181
|
+
return nil unless defined?(RSpec.configuration)
|
|
182
|
+
|
|
183
|
+
configuration = RSpec.configuration
|
|
184
|
+
reject_configuration!(configuration)
|
|
185
|
+
if configuration.respond_to?(:parallelize) && configuration.parallelize
|
|
186
|
+
raise UnsupportedRunner,
|
|
187
|
+
"parallel test scheduling is unsupported"
|
|
188
|
+
end
|
|
189
|
+
|
|
190
|
+
nil
|
|
191
|
+
end
|
|
192
|
+
|
|
193
|
+
def example_started(notification)
|
|
194
|
+
example = notification.example
|
|
195
|
+
record = test_record(example)
|
|
196
|
+
record[:status] = "running"
|
|
197
|
+
register_test(record)
|
|
198
|
+
end
|
|
199
|
+
|
|
200
|
+
def example_finished(notification)
|
|
201
|
+
example = notification.example
|
|
202
|
+
record = test_record(example)
|
|
203
|
+
record[:name] = example.full_description.to_s
|
|
204
|
+
record[:status] = status_for(example)
|
|
205
|
+
register_test(record)
|
|
206
|
+
end
|
|
207
|
+
|
|
208
|
+
def reject_execution!(message)
|
|
209
|
+
@run_error ||= UnsupportedRunner.new(message)
|
|
210
|
+
@late_execution_error = @run_error if @completion_started
|
|
211
|
+
raise @run_error
|
|
212
|
+
end
|
|
213
|
+
|
|
214
|
+
def enter_runner!(runner)
|
|
215
|
+
unless runner.equal?(@native_runner) && !@runner_entered
|
|
216
|
+
reject_execution!("nested or repeated RSpec runs are unsupported")
|
|
217
|
+
end
|
|
218
|
+
@runner_entered = true
|
|
219
|
+
end
|
|
220
|
+
|
|
221
|
+
private
|
|
222
|
+
|
|
223
|
+
def capture_runner_state(runner)
|
|
224
|
+
@selected_test_files = runner.configuration.loaded_spec_files.map { |path| File.expand_path(path.to_s) }
|
|
225
|
+
@selected_example_ids = runner.world.filtered_examples.values.flatten.map(&:id)
|
|
226
|
+
@seed = runner.configuration.seed
|
|
227
|
+
@non_example_failure = runner.world.non_example_failure
|
|
228
|
+
end
|
|
229
|
+
|
|
230
|
+
def reject_options!(options)
|
|
231
|
+
FORBIDDEN_OPTIONS.each do |key, label|
|
|
232
|
+
value = options.options[key]
|
|
233
|
+
next unless value && value != false && value != []
|
|
234
|
+
|
|
235
|
+
raise UnsupportedRunner, "#{label} is unsupported"
|
|
236
|
+
end
|
|
237
|
+
end
|
|
238
|
+
|
|
239
|
+
def reject_configuration!(configuration)
|
|
240
|
+
if configuration.respond_to?(:dry_run?) && configuration.dry_run?
|
|
241
|
+
raise UnsupportedRunner,
|
|
242
|
+
"dry-run is unsupported"
|
|
243
|
+
end
|
|
244
|
+
raise UnsupportedRunner, "bisect is unsupported" if configuration.respond_to?(:bisect) && configuration.bisect
|
|
245
|
+
raise UnsupportedRunner, "DRb is unsupported" if defined?(DRb) && RSpec::Core::Runner.send(:running_in_drb?)
|
|
246
|
+
end
|
|
247
|
+
|
|
248
|
+
def install_reporter_listener(reporter)
|
|
249
|
+
@defined_example_count = RSpec.world.filtered_examples.keys.sum { |group| group.examples.length }
|
|
250
|
+
listener = self
|
|
251
|
+
reporter.register_listener(listener, :example_started, :example_finished)
|
|
252
|
+
end
|
|
253
|
+
|
|
254
|
+
def install_runner_guard
|
|
255
|
+
return if RSpec::Core::Runner.singleton_class.instance_variable_defined?(:@branchproof_runner_guard)
|
|
256
|
+
|
|
257
|
+
RSpec::Core::Runner.singleton_class.prepend(ClassRunnerGuard)
|
|
258
|
+
RSpec::Core::Runner.prepend(RunnerGuard)
|
|
259
|
+
RSpec::Core::Runner.singleton_class.instance_variable_set(:@branchproof_runner_guard, true)
|
|
260
|
+
end
|
|
261
|
+
|
|
262
|
+
def install_lifecycle_hooks
|
|
263
|
+
return if RSpec::Core::Example.instance_variable_defined?(:@branchproof_lifecycle_hooks)
|
|
264
|
+
|
|
265
|
+
RSpec::Core::Example.prepend(ExampleLifecycle)
|
|
266
|
+
RSpec::Core::ExampleGroup.singleton_class.prepend(ContextLifecycle)
|
|
267
|
+
RSpec::Core::Example.instance_variable_set(:@branchproof_lifecycle_hooks, true)
|
|
268
|
+
end
|
|
269
|
+
|
|
270
|
+
def begin_example(example)
|
|
271
|
+
validate_runner!
|
|
272
|
+
seen = (@seen_examples ||= {}.compare_by_identity)
|
|
273
|
+
reject_execution!("repeated RSpec example attempts are unsupported") if seen[example]
|
|
274
|
+
|
|
275
|
+
seen[example] = true
|
|
276
|
+
phase(example, "setup")
|
|
277
|
+
rescue ArgumentError => e
|
|
278
|
+
reject_execution!(e.message)
|
|
279
|
+
end
|
|
280
|
+
|
|
281
|
+
def begin_attempt(example)
|
|
282
|
+
@attempts ||= {}
|
|
283
|
+
id = test_id(example)
|
|
284
|
+
if @attempts[id]
|
|
285
|
+
@run_error ||= UnsupportedRunner.new("repeated RSpec example attempts are unsupported")
|
|
286
|
+
raise @run_error
|
|
287
|
+
end
|
|
288
|
+
@attempts[id] = true
|
|
289
|
+
end
|
|
290
|
+
|
|
291
|
+
def without_owner
|
|
292
|
+
previous = @current_context
|
|
293
|
+
unattributed
|
|
294
|
+
yield
|
|
295
|
+
ensure
|
|
296
|
+
@current_context = previous
|
|
297
|
+
@runtime.context(test_id: previous&.first, phase: previous&.last || "unattributed")
|
|
298
|
+
end
|
|
299
|
+
|
|
300
|
+
def status_for(example)
|
|
301
|
+
result = example.execution_result
|
|
302
|
+
status = result.status&.to_sym
|
|
303
|
+
return "failed" if (result.respond_to?(:pending_fixed?) && result.pending_fixed?) ||
|
|
304
|
+
(example.respond_to?(:pending?) && example.pending? && status == :failed)
|
|
305
|
+
return "failed" if status == :failed
|
|
306
|
+
return "skipped" if example.respond_to?(:skipped?) && example.skipped?
|
|
307
|
+
return "skipped" if status == :pending
|
|
308
|
+
|
|
309
|
+
status == :passed ? "passed" : "running"
|
|
310
|
+
end
|
|
311
|
+
|
|
312
|
+
def test_record(example)
|
|
313
|
+
@example_records[example] ||= begin
|
|
314
|
+
source = source_for(example)
|
|
315
|
+
id = Branchproof::Records.id(adapter: "rspec", example_id: example.id.to_s, source: source)
|
|
316
|
+
@tests[id] = {
|
|
317
|
+
id: id, adapter: "rspec", name: example.full_description.to_s,
|
|
318
|
+
example_id: example.id.to_s, class_name: nil, method_name: nil,
|
|
319
|
+
spec_digest: file_digest(example.id.to_s.sub(/\[[\d:]+\]\z/, "")),
|
|
320
|
+
source_digest: file_digest(source[:path]),
|
|
321
|
+
source: source, status: "running", phase_counts: {}
|
|
322
|
+
}
|
|
323
|
+
end
|
|
324
|
+
end
|
|
325
|
+
|
|
326
|
+
def test_id(example)
|
|
327
|
+
test_record(example)[:id]
|
|
328
|
+
end
|
|
329
|
+
|
|
330
|
+
def file_digest(path)
|
|
331
|
+
expanded = File.expand_path(path)
|
|
332
|
+
return @file_digests[expanded] if @file_digests.key?(expanded)
|
|
333
|
+
|
|
334
|
+
@file_digests[expanded] = File.file?(expanded) ? Digest::SHA256.file(expanded).hexdigest : nil
|
|
335
|
+
end
|
|
336
|
+
|
|
337
|
+
def source_for(example)
|
|
338
|
+
path = example.metadata[:absolute_file_path] || example.file_path
|
|
339
|
+
{ path: File.expand_path(path.to_s), line: example.metadata[:line_number].to_i }
|
|
340
|
+
end
|
|
341
|
+
|
|
342
|
+
def phase(example, name)
|
|
343
|
+
id = test_id(example)
|
|
344
|
+
@current_context = [id, name]
|
|
345
|
+
@runtime.context(test_id: id, phase: name)
|
|
346
|
+
end
|
|
347
|
+
|
|
348
|
+
def unattributed
|
|
349
|
+
@current_context = nil
|
|
350
|
+
@runtime.context(test_id: nil, phase: "unattributed")
|
|
351
|
+
end
|
|
352
|
+
|
|
353
|
+
def register_test(test)
|
|
354
|
+
@runtime.register_test(test: test)
|
|
355
|
+
end
|
|
356
|
+
|
|
357
|
+
def complete(status:, finalized:)
|
|
358
|
+
return if @completed || @completion_started
|
|
359
|
+
|
|
360
|
+
@completion_started = true
|
|
361
|
+
apply_canonical_phase_counts
|
|
362
|
+
tests = @tests.values
|
|
363
|
+
failed = tests.count { |test| test[:status] == "failed" }
|
|
364
|
+
skipped = tests.count { |test| test[:status] == "skipped" }
|
|
365
|
+
diagnostics = completion_diagnostics(status)
|
|
366
|
+
errored = diagnostics.any? { |diagnostic| diagnostic[:severity] == "error" }
|
|
367
|
+
result = {
|
|
368
|
+
status: if errored
|
|
369
|
+
"ERROR"
|
|
370
|
+
elsif tests.empty?
|
|
371
|
+
"INCOMPLETE"
|
|
372
|
+
elsif failed.zero?
|
|
373
|
+
"PASSED"
|
|
374
|
+
else
|
|
375
|
+
"FAILED"
|
|
376
|
+
end,
|
|
377
|
+
executed_tests: tests.length, failed_tests: failed, skipped_tests: skipped,
|
|
378
|
+
exit_status: status.to_i, finalized: finalized && !errored, seed: @seed, diagnostics: diagnostics,
|
|
379
|
+
selected_test_files: @selected_test_files || [], selected_example_ids: @selected_example_ids || [], tests: tests
|
|
380
|
+
}
|
|
381
|
+
@completion_callback.call(result)
|
|
382
|
+
@completed = true
|
|
383
|
+
end
|
|
384
|
+
|
|
385
|
+
def completion_diagnostics(native_status)
|
|
386
|
+
if @run_error
|
|
387
|
+
code = if defined?(Branchproof::RailsSupport::Error) && @run_error.is_a?(Branchproof::RailsSupport::Error)
|
|
388
|
+
"rails_boot"
|
|
389
|
+
elsif @run_error.respond_to?(:diagnostic_code)
|
|
390
|
+
@run_error.diagnostic_code
|
|
391
|
+
else
|
|
392
|
+
"rspec_run"
|
|
393
|
+
end
|
|
394
|
+
return [{ code: code, severity: "error", message: "#{@run_error.class}: #{@run_error.message}" }]
|
|
395
|
+
end
|
|
396
|
+
if @non_example_failure || @tests.values.any? { |test| test[:status] == "running" } ||
|
|
397
|
+
(native_status.to_i != 0 && @tests.values.none? { |test| test[:status] == "failed" })
|
|
398
|
+
return [{ code: "rspec_run", severity: "error",
|
|
399
|
+
message: "RSpec failed outside examples; see runner output" }]
|
|
400
|
+
end
|
|
401
|
+
|
|
402
|
+
return [] unless @tests.empty?
|
|
403
|
+
|
|
404
|
+
code, message = if @unmatched_test_selection
|
|
405
|
+
["unmatched_test_selection", "No files matched --test; check the path or glob"]
|
|
406
|
+
elsif @defined_example_count.to_i.positive?
|
|
407
|
+
["empty_example_selection",
|
|
408
|
+
"RSpec filters selected zero examples; check tags, descriptions, and selectors"]
|
|
409
|
+
else
|
|
410
|
+
["empty_suite", "RSpec discovered no examples; check the default path and configured patterns"]
|
|
411
|
+
end
|
|
412
|
+
[{ code: code, severity: "warning", message: message }]
|
|
413
|
+
end
|
|
414
|
+
|
|
415
|
+
def apply_canonical_phase_counts
|
|
416
|
+
canonical_by_id = @runtime.test_phase_counts
|
|
417
|
+
@tests.each do |id, test|
|
|
418
|
+
counts = canonical_by_id[id]
|
|
419
|
+
test[:phase_counts] = counts unless counts.nil?
|
|
420
|
+
end
|
|
421
|
+
end
|
|
422
|
+
end
|
|
423
|
+
end
|
data/lib/branchproof/runtime.rb
CHANGED
|
@@ -7,13 +7,14 @@
|
|
|
7
7
|
require "json"
|
|
8
8
|
# rubocop:disable-next Lint/RedundantRequireStatement -- supports standalone core entry
|
|
9
9
|
require "set"
|
|
10
|
+
require_relative "coverage_policy"
|
|
10
11
|
require_relative "decision_table"
|
|
11
12
|
|
|
12
13
|
module Branchproof
|
|
13
14
|
# Reads and validates a persisted JSON report without loading the project.
|
|
14
15
|
class SavedReport
|
|
15
|
-
SUPPORTED_SCHEMAS = %w[1.0 1.1 1.2 1.3].freeze
|
|
16
|
-
STRICT_FLOW_SCHEMAS = %w[1.2 1.3].freeze
|
|
16
|
+
SUPPORTED_SCHEMAS = %w[1.0 1.1 1.2 1.3 1.4].freeze
|
|
17
|
+
STRICT_FLOW_SCHEMAS = %w[1.2 1.3 1.4].freeze
|
|
17
18
|
DECISION_TABLE_STATUSES = %w[calculated not_calculated].freeze
|
|
18
19
|
RULE_CONDITION_VALUES = DecisionTable::CONDITION_VALUES
|
|
19
20
|
RULE_COVERAGE_STATUSES = DecisionTable::COVERAGE_STATUSES
|
|
@@ -60,6 +61,7 @@ module Branchproof
|
|
|
60
61
|
validate_completeness(@document["completeness"])
|
|
61
62
|
validate_completeness(@document.dig("observations", "completeness"))
|
|
62
63
|
validate_completeness(@document.dig("analysis", "completeness")) if @document["analysis"]
|
|
64
|
+
validate_coverage_policy
|
|
63
65
|
validate_optional_sections
|
|
64
66
|
validate_analysis_coverage
|
|
65
67
|
@document
|
|
@@ -218,7 +220,9 @@ module Branchproof
|
|
|
218
220
|
fail_with("nonboolean condition results must be empty") unless results.empty?
|
|
219
221
|
validate_nonboolean_analysis(decision, inventory_decision)
|
|
220
222
|
elsif decision.key?("decision_table")
|
|
221
|
-
|
|
223
|
+
unless %w[1.3 1.4].include?(@schema_version)
|
|
224
|
+
fail_with("decision tables are unsupported in legacy report schemas")
|
|
225
|
+
end
|
|
222
226
|
validate_decision_table(decision["decision_table"], inventory_decision)
|
|
223
227
|
end
|
|
224
228
|
seen_conditions = {}
|
|
@@ -626,6 +630,28 @@ module Branchproof
|
|
|
626
630
|
fail_with("document contains non-string object keys") unless all_hashes_have_string_keys?(@document)
|
|
627
631
|
end
|
|
628
632
|
|
|
633
|
+
def validate_coverage_policy
|
|
634
|
+
policy = @document["coverage_policy"]
|
|
635
|
+
fail_with("missing field: coverage_policy") if @schema_version == "1.4" && !policy
|
|
636
|
+
return if policy.nil?
|
|
637
|
+
|
|
638
|
+
fail_with("coverage_policy must be an object") unless hash_with_string_keys?(policy)
|
|
639
|
+
minimum = policy["minimum"]
|
|
640
|
+
fail_with("coverage_policy minimum must be an object") unless hash_with_string_keys?(minimum)
|
|
641
|
+
normalized = CoveragePolicy.normalize(minimum)
|
|
642
|
+
fail_with("coverage_policy status must be a string") unless policy["status"].is_a?(String)
|
|
643
|
+
fail_with("coverage_policy gates must be an array") unless policy["gates"].is_a?(Array)
|
|
644
|
+
expected = CoveragePolicy.new(minimum: normalized).call(
|
|
645
|
+
document: { baseline: @document["baseline"], completeness: @document["completeness"],
|
|
646
|
+
observations: @document["observations"], analysis: @document["analysis"] }
|
|
647
|
+
)
|
|
648
|
+
actual = JSON.parse(JSON.generate(policy))
|
|
649
|
+
expected = JSON.parse(JSON.generate(expected))
|
|
650
|
+
fail_with("coverage_policy does not match report coverage") unless actual == expected
|
|
651
|
+
rescue ArgumentError => e
|
|
652
|
+
fail_with(e.message.sub(/\Ainvalid saved report: /, ""))
|
|
653
|
+
end
|
|
654
|
+
|
|
629
655
|
def validate_phases(vector)
|
|
630
656
|
phases = vector["phases_by_test"]
|
|
631
657
|
return if phases.nil?
|
|
@@ -638,7 +664,9 @@ module Branchproof
|
|
|
638
664
|
end
|
|
639
665
|
|
|
640
666
|
def validate_test(test)
|
|
641
|
-
%w[name class_name method_name source_path].each
|
|
667
|
+
%w[adapter name class_name method_name source_path example_id].each do |field|
|
|
668
|
+
validate_string_field(test, field, nullable: true)
|
|
669
|
+
end
|
|
642
670
|
validate_integer_field(test, "line", nullable: true)
|
|
643
671
|
phase_counts = test["phase_counts"]
|
|
644
672
|
if phase_counts
|
|
@@ -726,7 +754,12 @@ module Branchproof
|
|
|
726
754
|
string_fields.each do |field|
|
|
727
755
|
fail_with("run_metadata #{field} must be a string") if metadata.key?(field) && !metadata[field].is_a?(String)
|
|
728
756
|
end
|
|
729
|
-
|
|
757
|
+
nullable_string_fields = %w[framework framework_version rspec_rails_version rails_version]
|
|
758
|
+
nullable_string_fields.each do |field|
|
|
759
|
+
fail_with("run_metadata #{field} must be a string") if metadata.key?(field) &&
|
|
760
|
+
!metadata[field].nil? && !metadata[field].is_a?(String)
|
|
761
|
+
end
|
|
762
|
+
array_fields = %w[source_patterns test_patterns test_files runner_args selected_test_files selected_example_ids]
|
|
730
763
|
array_fields.each do |field|
|
|
731
764
|
if metadata.key?(field) && !strings?(metadata[field])
|
|
732
765
|
fail_with("run_metadata #{field} must be an array of strings")
|
data/lib/branchproof/version.rb
CHANGED
data/lib/branchproof/worker.rb
CHANGED
|
@@ -8,19 +8,18 @@ require "stringio"
|
|
|
8
8
|
require "fileutils"
|
|
9
9
|
|
|
10
10
|
module Branchproof
|
|
11
|
-
# Runs one isolated
|
|
11
|
+
# Runs one isolated test worker and atomically exports its result.
|
|
12
12
|
module Worker
|
|
13
13
|
module_function
|
|
14
14
|
|
|
15
15
|
def child_process(config_path)
|
|
16
|
+
$stdout.sync = true
|
|
17
|
+
$stderr.sync = true
|
|
16
18
|
payload = JSON.parse(File.binread(config_path))
|
|
17
19
|
project = symbolize(payload.fetch("project"))
|
|
18
20
|
prepend_load_paths(project)
|
|
19
21
|
inventory = symbolize(payload.fetch("inventory"))
|
|
20
22
|
limits = symbolize(payload.fetch("limits"))
|
|
21
|
-
require "minitest"
|
|
22
|
-
require "minitest/test"
|
|
23
|
-
require_relative "minitest_adapter"
|
|
24
23
|
evidence = Branchproof::Evidence.new(inventory: inventory, limits: limits, run_id: payload.fetch("run_id"))
|
|
25
24
|
runtime = Branchproof::Runtime
|
|
26
25
|
runtime.boot(evidence: evidence)
|
|
@@ -28,23 +27,38 @@ module Branchproof
|
|
|
28
27
|
status = loader.install
|
|
29
28
|
return write_failure(payload, "loader", status) unless status[:status].to_sym == :installed
|
|
30
29
|
|
|
31
|
-
adapter =
|
|
30
|
+
adapter = adapter_for(project, runtime)
|
|
31
|
+
guard_late_rspec_execution(adapter, payload) if project[:framework].to_s == "rspec"
|
|
32
32
|
rails_metadata = nil
|
|
33
33
|
ARGV.replace(payload.fetch("runner_args"))
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
34
|
+
callbacks = if project[:framework].to_s == "rspec"
|
|
35
|
+
{ test_selection_explicit: payload.fetch("test_selection_explicit", false), after_load: lambda {
|
|
36
|
+
require_relative "rails_support"
|
|
37
|
+
rails_metadata = Branchproof::RailsSupport.validate_rspec!(project: project)
|
|
38
|
+
} }
|
|
39
|
+
else
|
|
40
|
+
{}
|
|
41
|
+
end
|
|
42
|
+
before_load = lambda do
|
|
43
|
+
rails_metadata = boot_project(project) unless project[:framework].to_s == "rspec"
|
|
44
|
+
end
|
|
45
|
+
on_complete = lambda do |baseline|
|
|
46
|
+
result = completion_result(baseline: baseline, project: project, rails_metadata: rails_metadata,
|
|
47
|
+
evidence: runtime.snapshot, tests: adapter.tests.values,
|
|
48
|
+
diagnostics: loader.diagnostics)
|
|
49
|
+
write_completion(payload, result)
|
|
50
|
+
end
|
|
51
|
+
exit_status = adapter.run(test_files: payload.fetch("test_files"), runner_args: payload.fetch("runner_args"),
|
|
52
|
+
before_load: before_load, on_complete: on_complete, **callbacks)
|
|
53
|
+
return exit_status if project[:framework].to_s == "rspec"
|
|
54
|
+
|
|
43
55
|
adapter.validate_runner!
|
|
44
56
|
Minitest.autorun
|
|
45
57
|
nil
|
|
46
58
|
rescue StandardError => e
|
|
47
|
-
code = if e.
|
|
59
|
+
code = if e.respond_to?(:diagnostic_code)
|
|
60
|
+
e.diagnostic_code
|
|
61
|
+
elsif e.message.match?(/parallel|runner|bisect|DRb|dry.run/)
|
|
48
62
|
"unsupported_runner"
|
|
49
63
|
elsif defined?(Branchproof::RailsSupport::Error) && e.is_a?(Branchproof::RailsSupport::Error)
|
|
50
64
|
"rails_boot"
|
|
@@ -55,6 +69,44 @@ module Branchproof
|
|
|
55
69
|
2
|
|
56
70
|
end
|
|
57
71
|
|
|
72
|
+
def adapter_for(project, runtime)
|
|
73
|
+
if project[:framework].to_s == "rspec"
|
|
74
|
+
begin
|
|
75
|
+
require "rspec/core"
|
|
76
|
+
rescue LoadError => e
|
|
77
|
+
raise unless e.path == "rspec/core"
|
|
78
|
+
|
|
79
|
+
error = ArgumentError.new("RSpec is not available; add rspec to the application's test bundle")
|
|
80
|
+
def error.diagnostic_code = "rspec_missing"
|
|
81
|
+
raise error
|
|
82
|
+
end
|
|
83
|
+
require_relative "rspec_adapter"
|
|
84
|
+
Branchproof::RSpecAdapter.new(runtime: runtime)
|
|
85
|
+
else
|
|
86
|
+
require "minitest"
|
|
87
|
+
require "minitest/test"
|
|
88
|
+
require_relative "minitest_adapter"
|
|
89
|
+
Branchproof::MinitestAdapter.new(runtime: runtime)
|
|
90
|
+
end
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
# Installed before application hooks, so this runs after their at_exit work.
|
|
94
|
+
# A rescued second runner must still invalidate the already-exported result.
|
|
95
|
+
def guard_late_rspec_execution(adapter, payload)
|
|
96
|
+
at_exit do
|
|
97
|
+
error = adapter.late_execution_error
|
|
98
|
+
if error
|
|
99
|
+
result = symbolize(JSON.parse(File.binread(payload.fetch("result_path"))))
|
|
100
|
+
diagnostic = { code: "unsupported_runner", severity: "error", message: error.message }
|
|
101
|
+
result.merge!(status: "ERROR", finalized: false, exit_status: 2,
|
|
102
|
+
diagnostics: Array(result[:diagnostics]) + [diagnostic])
|
|
103
|
+
result[:evidence] = incomplete_evidence(result[:evidence], [diagnostic]) if result[:evidence]
|
|
104
|
+
write_completion(payload, result)
|
|
105
|
+
exit(2)
|
|
106
|
+
end
|
|
107
|
+
end
|
|
108
|
+
end
|
|
109
|
+
|
|
58
110
|
def legacy_project
|
|
59
111
|
{ "kind" => "ruby", "root" => Dir.pwd,
|
|
60
112
|
"load_paths" => [File.join(Dir.pwd, "lib"), File.join(Dir.pwd, "test")],
|
|
@@ -76,7 +128,14 @@ module Branchproof
|
|
|
76
128
|
end
|
|
77
129
|
|
|
78
130
|
def project_metadata(project, rails_metadata)
|
|
131
|
+
framework = project[:framework].to_s.empty? ? "minitest" : project[:framework].to_s
|
|
132
|
+
version = if framework == "rspec" && defined?(RSpec::Core::Version::STRING)
|
|
133
|
+
RSpec::Core::Version::STRING
|
|
134
|
+
elsif framework == "minitest" && defined?(Minitest::VERSION)
|
|
135
|
+
Minitest::VERSION
|
|
136
|
+
end
|
|
79
137
|
metadata = { kind: project[:kind].to_s, root: project[:root].to_s,
|
|
138
|
+
framework: framework, framework_version: version,
|
|
80
139
|
load_paths: Array(project[:load_paths]).map(&:to_s), serial_policy: { mode: "single_process", workers: 1 } }
|
|
81
140
|
metadata.merge!(rails_metadata) if rails_metadata
|
|
82
141
|
metadata
|
|
@@ -85,6 +144,7 @@ module Branchproof
|
|
|
85
144
|
# This result boundary intentionally carries the complete worker payload.
|
|
86
145
|
# rubocop:disable-next Metrics/ParameterLists
|
|
87
146
|
def completion_result(baseline:, project:, rails_metadata:, evidence:, tests:, diagnostics:)
|
|
147
|
+
diagnostics = Array(baseline[:diagnostics]) + diagnostics
|
|
88
148
|
result = baseline.merge(project: project_metadata(project, rails_metadata), evidence: evidence,
|
|
89
149
|
tests: tests, diagnostics: diagnostics)
|
|
90
150
|
return result unless diagnostics.any? { |diagnostic| diagnostic[:severity].to_s == "error" }
|
data/lib/branchproof.rb
CHANGED
|
@@ -9,11 +9,15 @@ require_relative "branchproof/project"
|
|
|
9
9
|
# Public namespace for source inventory and one-run MC/DC reporting.
|
|
10
10
|
module Branchproof
|
|
11
11
|
class Error < StandardError; end
|
|
12
|
+
autoload :Configuration, "branchproof/configuration"
|
|
13
|
+
autoload :CoveragePolicy, "branchproof/coverage_policy"
|
|
14
|
+
autoload :ReportSelection, "branchproof/report_selection"
|
|
12
15
|
autoload :CLI, "branchproof/cli"
|
|
13
16
|
autoload :Instrumenter, "branchproof/instrumenter"
|
|
14
17
|
autoload :Loader, "branchproof/loader"
|
|
15
18
|
autoload :Runtime, "branchproof/runtime"
|
|
16
19
|
autoload :MinitestAdapter, "branchproof/minitest_adapter"
|
|
20
|
+
autoload :RSpecAdapter, "branchproof/rspec_adapter"
|
|
17
21
|
autoload :Evidence, "branchproof/evidence"
|
|
18
22
|
autoload :Analyzer, "branchproof/analyzer"
|
|
19
23
|
autoload :Constraints, "branchproof/constraints"
|