ripr 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +5 -0
- data/LICENSE.txt +21 -0
- data/README.md +146 -0
- data/exe/ripr +6 -0
- data/lib/ripr/accept_ledger.rb +57 -0
- data/lib/ripr/cli.rb +160 -0
- data/lib/ripr/config.rb +102 -0
- data/lib/ripr/core/analyzer.rb +179 -0
- data/lib/ripr/core/classifier.rb +18 -0
- data/lib/ripr/core/instrumenter.rb +110 -0
- data/lib/ripr/core/operators.rb +41 -0
- data/lib/ripr/core/reuse_planner.rb +20 -0
- data/lib/ripr/engine/baseline_cache.rb +95 -0
- data/lib/ripr/engine/child.rb +103 -0
- data/lib/ripr/engine/result_cache.rb +83 -0
- data/lib/ripr/engine/runner.rb +224 -0
- data/lib/ripr/engine/scheduler.rb +81 -0
- data/lib/ripr/git.rb +58 -0
- data/lib/ripr/integration/minitest.rb +72 -0
- data/lib/ripr/integration/rspec.rb +88 -0
- data/lib/ripr/loader.rb +165 -0
- data/lib/ripr/reporters.rb +139 -0
- data/lib/ripr/runtime.rb +205 -0
- data/lib/ripr/snapshot.rb +41 -0
- data/lib/ripr/version.rb +5 -0
- data/lib/ripr.rb +8 -0
- data/schema/agent-v1.json +74 -0
- data/skills/mutation-test-loop/SKILL.md +28 -0
- metadata +94 -0
|
@@ -0,0 +1,224 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../core/analyzer"
|
|
4
|
+
require_relative "../core/instrumenter"
|
|
5
|
+
require_relative "../core/classifier"
|
|
6
|
+
require_relative "../runtime"
|
|
7
|
+
require_relative "../loader"
|
|
8
|
+
require_relative "../git"
|
|
9
|
+
require_relative "../accept_ledger"
|
|
10
|
+
require_relative "child"
|
|
11
|
+
require_relative "baseline_cache"
|
|
12
|
+
require_relative "result_cache"
|
|
13
|
+
require_relative "scheduler"
|
|
14
|
+
require "pathname"
|
|
15
|
+
require "tempfile"
|
|
16
|
+
|
|
17
|
+
module Ripr
|
|
18
|
+
module Engine
|
|
19
|
+
class Runner
|
|
20
|
+
attr_reader :mutants, :warnings, :baseline, :framework, :sources, :static, :metrics
|
|
21
|
+
|
|
22
|
+
def initialize(config, paths = [])
|
|
23
|
+
@config = config
|
|
24
|
+
@paths = paths
|
|
25
|
+
@warnings = []
|
|
26
|
+
@mutants = []
|
|
27
|
+
@sources = {}
|
|
28
|
+
@table = {}
|
|
29
|
+
@codes = {}
|
|
30
|
+
@static = 0
|
|
31
|
+
@metrics = { "baseline_reused" => false, "baseline_tests_run" => 0, "reused" => 0 }
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def plan
|
|
35
|
+
return @mutants if @planned
|
|
36
|
+
|
|
37
|
+
patterns = if @paths.empty?
|
|
38
|
+
@config[:include]
|
|
39
|
+
else
|
|
40
|
+
@paths.map { |p| File.directory?(p) ? "#{p}/**/*.rb" : p }
|
|
41
|
+
end
|
|
42
|
+
files = patterns.flat_map { |pattern| Dir.glob(pattern) }.uniq.sort.select { |p| File.file?(p) && p.end_with?(".rb") }
|
|
43
|
+
files.map! { |path| relative(path) }
|
|
44
|
+
files.uniq!
|
|
45
|
+
files.reject! { |path| @config[:exclude].any? { |pattern| File.fnmatch?(pattern, path, File::FNM_PATHNAME) } }
|
|
46
|
+
files.each do |path|
|
|
47
|
+
source = File.read(path)
|
|
48
|
+
@sources[path] = source
|
|
49
|
+
analysis = Core::Analyzer.call(source, path, operators: @config[:operators], arid_callees: @config[:arid_callees])
|
|
50
|
+
instrumented = Core::Instrumenter.call(source, path, analysis.fetch(:sites))
|
|
51
|
+
@codes[File.expand_path(path)] = instrumented.fetch(:code) unless instrumented.fetch(:site_table).empty?
|
|
52
|
+
@table.merge!(instrumented.fetch(:site_table))
|
|
53
|
+
analysis.fetch(:mutants).each do |mutant|
|
|
54
|
+
mutant[:invalid] = true if instrumented.fetch(:invalid).include?(mutant[:id])
|
|
55
|
+
end
|
|
56
|
+
@mutants.concat(analysis.fetch(:mutants).map { |mutant| stringify(mutant) })
|
|
57
|
+
@warnings.concat(analysis.fetch(:warnings) + instrumented.fetch(:warnings))
|
|
58
|
+
@static += analysis.fetch(:static)
|
|
59
|
+
end
|
|
60
|
+
acceptances = AcceptLedger.new.read.to_h { |entry| [entry.fetch("full_id"), entry] }
|
|
61
|
+
@mutants.each do |mutant|
|
|
62
|
+
entry = acceptances[mutant.fetch("full_id")]
|
|
63
|
+
next unless entry
|
|
64
|
+
|
|
65
|
+
if entry["method_hash"] == mutant["method_hash"]
|
|
66
|
+
mutant["accepted"] ||= entry.fetch("reason")
|
|
67
|
+
else
|
|
68
|
+
@warnings << "stale acceptance #{mutant.fetch('id')}: method changed; review the reason again"
|
|
69
|
+
end
|
|
70
|
+
end
|
|
71
|
+
@planned = true
|
|
72
|
+
@mutants
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
def call(&)
|
|
76
|
+
previous_ripr = ENV.fetch("RIPR", nil)
|
|
77
|
+
plan unless @planned
|
|
78
|
+
selected = select_mutants(@mutants)
|
|
79
|
+
return selected.map { |m| m.merge("status" => "planned") } if @config[:dry_run]
|
|
80
|
+
return [] if selected.empty?
|
|
81
|
+
|
|
82
|
+
reject_preloaded_sources
|
|
83
|
+
ENV["RIPR"] = "1"
|
|
84
|
+
@loader = Loader.new(@codes).install
|
|
85
|
+
R.configure(@table, samples: @config[:samples])
|
|
86
|
+
# The probe still records unattributed reach for conservative baseline reuse.
|
|
87
|
+
probe_table = @table.slice(*selected.map { |mutant| mutant.fetch("site") }.uniq)
|
|
88
|
+
R.probe = Probe.new(probe_table, samples: @config[:samples])
|
|
89
|
+
adapter = load_tests
|
|
90
|
+
load_reached = R.probe.snapshot.fetch("load_reached")
|
|
91
|
+
R.probe = nil
|
|
92
|
+
@cache = ResultCache.new(@config, sources: @sources, test_files: @test_files, framework: @framework, mutants: @mutants,
|
|
93
|
+
test_ids: adapter.tests.map { |test| test.fetch("id") }, probe_sites: probe_table.keys)
|
|
94
|
+
prepare_baseline(adapter, probe_table, load_reached)
|
|
95
|
+
scheduler = Scheduler.new(@config, @baseline, @table, @cache, warnings: @warnings, metrics: @metrics)
|
|
96
|
+
results = scheduler.call(selected, adapter, &)
|
|
97
|
+
@cache.write(results, @baseline, @metrics)
|
|
98
|
+
results
|
|
99
|
+
ensure
|
|
100
|
+
@loader&.uninstall
|
|
101
|
+
R.reset
|
|
102
|
+
previous_ripr.nil? ? ENV.delete("RIPR") : ENV["RIPR"] = previous_ripr
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
private
|
|
106
|
+
|
|
107
|
+
def reject_preloaded_sources
|
|
108
|
+
loaded = $LOADED_FEATURES.filter_map do |feature|
|
|
109
|
+
File.realpath(feature)
|
|
110
|
+
rescue Errno::ENOENT, Errno::ENOTDIR
|
|
111
|
+
nil
|
|
112
|
+
end
|
|
113
|
+
@codes.each do |path, code|
|
|
114
|
+
next unless loaded.include?(File.realpath(path))
|
|
115
|
+
next if Loader.instrumented?(path, code)
|
|
116
|
+
|
|
117
|
+
raise ArgumentError,
|
|
118
|
+
"target #{relative(path)} was loaded before ripr; start ripr without pre-requiring target source"
|
|
119
|
+
end
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
def select_mutants(mutants)
|
|
123
|
+
result = mutants
|
|
124
|
+
if @config[:since] && !@config[:full]
|
|
125
|
+
changed = begin
|
|
126
|
+
Git.new.changed_lines(@config[:since])
|
|
127
|
+
rescue Error => e
|
|
128
|
+
raise ArgumentError, e.message
|
|
129
|
+
end
|
|
130
|
+
result = result.select do |mutant|
|
|
131
|
+
location = mutant.fetch("location")
|
|
132
|
+
range = if @config[:scope] == "method"
|
|
133
|
+
mutant.fetch("method_start")..mutant.fetch("method_end")
|
|
134
|
+
else
|
|
135
|
+
location.fetch("line")..location.fetch("end_line")
|
|
136
|
+
end
|
|
137
|
+
changed.fetch(location.fetch("file"), []).any? { |line| range.cover?(line) }
|
|
138
|
+
end
|
|
139
|
+
end
|
|
140
|
+
unless @config[:only].empty?
|
|
141
|
+
missing = @config[:only] - mutants.flat_map { |m| [m["id"], m["full_id"]] }
|
|
142
|
+
raise ArgumentError, "unknown mutant ids: #{missing.join(', ')}" unless missing.empty?
|
|
143
|
+
|
|
144
|
+
result = result.select { |m| @config[:only].include?(m["id"]) || @config[:only].include?(m["full_id"]) }
|
|
145
|
+
end
|
|
146
|
+
result
|
|
147
|
+
end
|
|
148
|
+
|
|
149
|
+
def load_tests
|
|
150
|
+
%w[lib test spec].reverse_each { |directory| $LOAD_PATH.unshift(File.expand_path(directory)) }
|
|
151
|
+
@framework = @config[:framework]
|
|
152
|
+
@framework = Dir.glob("spec/**/*_spec.rb").empty? ? "minitest" : "rspec" if @framework == "auto"
|
|
153
|
+
patterns = @config[:tests] || (@framework == "rspec" ? "spec/**/*_spec.rb" : "test/**/*_test.rb")
|
|
154
|
+
files = Array(patterns).flat_map { |pattern| Dir.glob(pattern) }.map { |file| relative(file) }.uniq.sort
|
|
155
|
+
@test_files = files
|
|
156
|
+
raise ArgumentError, "no test files found" if files.empty?
|
|
157
|
+
|
|
158
|
+
require_relative "../integration/#{@framework}"
|
|
159
|
+
klass = @framework == "rspec" ? Integration::RSpec : Integration::Minitest
|
|
160
|
+
adapter = klass.new(files, seed: @config[:seed])
|
|
161
|
+
capture_load_output { adapter.load }
|
|
162
|
+
raise BaselineFailure, "no tests were discovered" if adapter.tests.empty?
|
|
163
|
+
|
|
164
|
+
adapter
|
|
165
|
+
end
|
|
166
|
+
|
|
167
|
+
# rubocop:disable-next Style/GlobalStdStream -- Capture descriptor 1, including direct STDOUT writes.
|
|
168
|
+
def capture_load_output
|
|
169
|
+
Tempfile.create("ripr-load") do |output|
|
|
170
|
+
STDOUT.flush
|
|
171
|
+
descriptor = STDOUT.dup
|
|
172
|
+
stream = $stdout
|
|
173
|
+
begin
|
|
174
|
+
STDOUT.reopen(output)
|
|
175
|
+
$stdout = STDOUT
|
|
176
|
+
yield
|
|
177
|
+
ensure
|
|
178
|
+
STDOUT.flush
|
|
179
|
+
STDOUT.reopen(descriptor)
|
|
180
|
+
$stdout = stream
|
|
181
|
+
descriptor.close
|
|
182
|
+
output.rewind
|
|
183
|
+
IO.copy_stream(output, STDERR)
|
|
184
|
+
end
|
|
185
|
+
end
|
|
186
|
+
end
|
|
187
|
+
|
|
188
|
+
def prepare_baseline(adapter, probe_table, load_reached)
|
|
189
|
+
tests = adapter.tests.map { |test| test.merge("file" => relative(test.fetch("file"))) }
|
|
190
|
+
cache = BaselineCache.new(@cache.previous["baseline"], env_hash: @cache.baseline_env_hash,
|
|
191
|
+
source_hash: @cache.source_hash, test_hashes: @cache.test_hashes,
|
|
192
|
+
tests: tests, full: @config[:full])
|
|
193
|
+
fresh = unless cache.test_ids == []
|
|
194
|
+
Child.call(timeout_ms: 60_000) do
|
|
195
|
+
probe = Probe.new(probe_table, samples: @config[:samples])
|
|
196
|
+
R.probe = probe
|
|
197
|
+
adapter.run(cache.test_ids).merge("probe" => probe.snapshot)
|
|
198
|
+
end
|
|
199
|
+
end
|
|
200
|
+
if fresh && !fresh["success"]
|
|
201
|
+
failures = fresh["failures"] || [fresh["error"] || fresh["status"]]
|
|
202
|
+
messages = failures.map { |failure| failure.is_a?(Hash) ? failure["message"] : failure }
|
|
203
|
+
raise BaselineFailure, "baseline failed: #{messages.join('; ')}"
|
|
204
|
+
end
|
|
205
|
+
fresh&.fetch("tests")&.each_value { |test| test["file"] = relative(test.fetch("file")) }
|
|
206
|
+
@baseline = cache.combine(fresh, load_reached: load_reached)
|
|
207
|
+
@metrics["baseline_reused"] = cache.reused_count.positive?
|
|
208
|
+
@metrics["baseline_tests_run"] = fresh ? fresh.fetch("tests").length : 0
|
|
209
|
+
end
|
|
210
|
+
|
|
211
|
+
def relative(path)
|
|
212
|
+
Pathname.new(File.expand_path(path)).relative_path_from(Pathname.new(Dir.pwd)).to_s
|
|
213
|
+
end
|
|
214
|
+
|
|
215
|
+
def stringify(value)
|
|
216
|
+
case value
|
|
217
|
+
when Hash then value.to_h { |key, child| [key.to_s, stringify(child)] }
|
|
218
|
+
when Array then value.map { |child| stringify(child) }
|
|
219
|
+
else value
|
|
220
|
+
end
|
|
221
|
+
end
|
|
222
|
+
end
|
|
223
|
+
end
|
|
224
|
+
end
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "child"
|
|
4
|
+
|
|
5
|
+
module Ripr
|
|
6
|
+
module Engine
|
|
7
|
+
class Scheduler
|
|
8
|
+
def initialize(config, baseline, table, cache, warnings:, metrics:)
|
|
9
|
+
@config = config
|
|
10
|
+
@baseline = baseline
|
|
11
|
+
@table = table
|
|
12
|
+
@cache = cache
|
|
13
|
+
@warnings = warnings
|
|
14
|
+
@metrics = metrics
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
def call(selected, adapter, &progress)
|
|
18
|
+
probe = @baseline.fetch("probe")
|
|
19
|
+
ready = []
|
|
20
|
+
results = selected.filter_map do |mutant|
|
|
21
|
+
reach = probe.fetch("reach").fetch(mutant.fetch("site"), [])
|
|
22
|
+
infect = probe.fetch("infect").fetch(mutant.fetch("id"), [])
|
|
23
|
+
unknown_reach = probe.fetch("load_reached").include?(mutant.fetch("site"))
|
|
24
|
+
status = if mutant["accepted"] then "accepted"
|
|
25
|
+
elsif mutant["invalid"] then "invalid"
|
|
26
|
+
elsif reach.empty? && !unknown_reach then "not_reached"
|
|
27
|
+
elsif mutant["mode"] == "value" && infect.empty? && !unknown_reach
|
|
28
|
+
"not_infected"
|
|
29
|
+
end
|
|
30
|
+
data = mutant.merge("load_reached" => unknown_reach,
|
|
31
|
+
"observed" => probe.fetch("samples").fetch(mutant.fetch("site"), []),
|
|
32
|
+
"covering_tests" => reach.filter_map { |id| @baseline.fetch("tests")[id]&.merge("id" => id) })
|
|
33
|
+
cached = @cache.result_for(mutant) unless %w[accepted invalid].include?(status)
|
|
34
|
+
verify = %w[not_reached not_infected].include?(status) &&
|
|
35
|
+
(!@config[:only].empty? || (@config[:verify_not_infected] && status == "not_infected"))
|
|
36
|
+
if cached
|
|
37
|
+
@metrics["reused"] += 1
|
|
38
|
+
cached.merge("load_reached" => unknown_reach)
|
|
39
|
+
elsif status && !verify
|
|
40
|
+
data.merge("status" => status)
|
|
41
|
+
else
|
|
42
|
+
ids = if unknown_reach
|
|
43
|
+
@baseline.fetch("tests").keys
|
|
44
|
+
else
|
|
45
|
+
(mutant["mode"] == "value" ? infect : reach)
|
|
46
|
+
end
|
|
47
|
+
ids = reach if ids.empty?
|
|
48
|
+
ids = @baseline.fetch("tests").keys if ids.empty?
|
|
49
|
+
ids.sort_by! { |id| @baseline.fetch("tests").fetch(id).fetch("duration_ms") }
|
|
50
|
+
ready << data.merge("test_ids" => ids, "verification_status" => verify ? status : nil)
|
|
51
|
+
nil
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
cost = lambda do |mutant|
|
|
55
|
+
mutant.fetch("test_ids").sum { |id| @baseline.fetch("tests").fetch(id).fetch("duration_ms") }
|
|
56
|
+
end
|
|
57
|
+
timeout = ->(mutant) { @config[:timeout].fetch("base_ms") + (@config[:timeout].fetch("factor") * cost.call(mutant)) }
|
|
58
|
+
completed = Child.batch(ready.sort_by(&cost), jobs: @config[:jobs], timeout: timeout, on_complete: progress) do |mutant|
|
|
59
|
+
site_ids = @table.fetch(mutant.fetch("site"))
|
|
60
|
+
R.activate(mutant.fetch("site"), site_ids.index(mutant.fetch("id")) + 1)
|
|
61
|
+
adapter.run(mutant.fetch("test_ids"), fail_fast: true)
|
|
62
|
+
end
|
|
63
|
+
completed.each do |mutant, outcome|
|
|
64
|
+
status = outcome["status"] || (outcome["success"] ? "not_detected" : "killed")
|
|
65
|
+
verification = mutant["verification_status"]
|
|
66
|
+
if verification
|
|
67
|
+
status = verification if outcome["success"]
|
|
68
|
+
@warnings << "verification discrepancy #{mutant.fetch('id')}: expected #{verification}, got killed" if status == "killed"
|
|
69
|
+
end
|
|
70
|
+
failure = outcome["failures"]&.first
|
|
71
|
+
result = mutant.except("test_ids", "verification_status").merge("status" => status,
|
|
72
|
+
"killed_by" => failure.is_a?(Hash) ? failure["id"] : failure,
|
|
73
|
+
"error" => outcome["error"])
|
|
74
|
+
result["verified"] = true if verification
|
|
75
|
+
results << result
|
|
76
|
+
end
|
|
77
|
+
results.sort_by { |mutant| mutant.fetch("full_id") }
|
|
78
|
+
end
|
|
79
|
+
end
|
|
80
|
+
end
|
|
81
|
+
end
|
data/lib/ripr/git.rb
ADDED
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "open3"
|
|
4
|
+
require_relative "../ripr"
|
|
5
|
+
|
|
6
|
+
module Ripr
|
|
7
|
+
class Git
|
|
8
|
+
def initialize(root = Dir.pwd)
|
|
9
|
+
@root = File.expand_path(root)
|
|
10
|
+
end
|
|
11
|
+
|
|
12
|
+
def changed_lines(ref, paths = nil)
|
|
13
|
+
base = execute("merge-base", "--", ref, "HEAD").strip
|
|
14
|
+
selected = Array(paths)
|
|
15
|
+
entries = execute("diff", "--relative", "--name-status", "-z", "--find-renames", base, "--", *selected).split("\0")
|
|
16
|
+
changed = {}
|
|
17
|
+
until entries.empty?
|
|
18
|
+
status = entries.shift
|
|
19
|
+
original = entries.shift
|
|
20
|
+
destination = status.start_with?("R", "C") ? entries.shift : original
|
|
21
|
+
next if status.start_with?("D") || !File.file?(File.join(@root, destination))
|
|
22
|
+
|
|
23
|
+
diff_paths = status.start_with?("R", "C") ? [original, destination] : [destination]
|
|
24
|
+
diff = execute("diff", "--relative", "--unified=0", "--no-color", "--no-ext-diff", "--no-textconv", "--find-renames", base, "--",
|
|
25
|
+
*diff_paths)
|
|
26
|
+
lines = hunk_lines(diff)
|
|
27
|
+
changed[destination] = lines unless lines.empty?
|
|
28
|
+
end
|
|
29
|
+
execute("ls-files", "--others", "--exclude-standard", "-z", "--", *selected).split("\0").each do |path|
|
|
30
|
+
absolute = File.join(@root, path)
|
|
31
|
+
next unless File.file?(absolute)
|
|
32
|
+
|
|
33
|
+
lines = File.binread(absolute).lines.count
|
|
34
|
+
changed[path] = (1..lines).to_a unless lines.zero?
|
|
35
|
+
end
|
|
36
|
+
changed
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
private
|
|
40
|
+
|
|
41
|
+
def execute(*arguments)
|
|
42
|
+
output, error, status = Open3.capture3("git", "-C", @root, *arguments)
|
|
43
|
+
raise Error, "git #{arguments.first} failed: #{error.strip}" unless status.success?
|
|
44
|
+
|
|
45
|
+
output
|
|
46
|
+
rescue Errno::ENOENT => e
|
|
47
|
+
raise Error, "git is unavailable: #{e.message}"
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
def hunk_lines(diff)
|
|
51
|
+
diff.scan(/^@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@/).flat_map do |start, length|
|
|
52
|
+
first = start.to_i
|
|
53
|
+
count = length.nil? ? 1 : length.to_i
|
|
54
|
+
count.zero? ? [first, first + 1] : (first...(first + count)).to_a
|
|
55
|
+
end.uniq.sort
|
|
56
|
+
end
|
|
57
|
+
end
|
|
58
|
+
end
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "minitest"
|
|
4
|
+
require_relative "../runtime"
|
|
5
|
+
|
|
6
|
+
module Ripr
|
|
7
|
+
module Integration
|
|
8
|
+
class Minitest
|
|
9
|
+
def initialize(test_files, seed: 0)
|
|
10
|
+
@test_files = test_files.map { |path| File.expand_path(path) }
|
|
11
|
+
@seed = seed
|
|
12
|
+
@cases = {}
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
def load
|
|
16
|
+
return self if @loaded
|
|
17
|
+
|
|
18
|
+
::Minitest.singleton_class.prepend(Module.new do
|
|
19
|
+
def autorun; end
|
|
20
|
+
end)
|
|
21
|
+
existing = ::Minitest::Runnable.runnables.dup
|
|
22
|
+
@test_files.each { |file| require file }
|
|
23
|
+
::Minitest.seed = @seed
|
|
24
|
+
srand(@seed)
|
|
25
|
+
(::Minitest::Runnable.runnables - existing).each do |klass|
|
|
26
|
+
klass.runnable_methods.sort.each do |method|
|
|
27
|
+
file, line = klass.instance_method(method).source_location
|
|
28
|
+
@cases["#{klass}##{method}"] = [klass, method, File.expand_path(file), line]
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
@loaded = true
|
|
32
|
+
self
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def tests
|
|
36
|
+
@cases.map do |id, (_, _, file, line)|
|
|
37
|
+
{ "id" => id, "file" => file.dup, "line" => line }
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
def run(ids = nil, fail_fast: false)
|
|
42
|
+
ids ||= @cases.keys.sort
|
|
43
|
+
unknown = ids.reject { |id| @cases.key?(id) }
|
|
44
|
+
raise ArgumentError, "unknown test IDs: #{unknown.join(', ')}" unless unknown.empty?
|
|
45
|
+
|
|
46
|
+
result = { "success" => !ids.empty?, "tests" => {}, "failures" => [] }
|
|
47
|
+
result["failures"] << { "message" => "No tests selected" } if ids.empty?
|
|
48
|
+
srand(@seed)
|
|
49
|
+
ids.each do |id|
|
|
50
|
+
klass, method, file, line = @cases.fetch(id)
|
|
51
|
+
R.current_test = id
|
|
52
|
+
started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
53
|
+
test = if ::Minitest.respond_to?(:run_one_method)
|
|
54
|
+
::Minitest.run_one_method(klass, method)
|
|
55
|
+
else
|
|
56
|
+
klass.new(method).run
|
|
57
|
+
end
|
|
58
|
+
duration = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - started) * 1000
|
|
59
|
+
result["tests"][id] = { "file" => file.dup, "line" => line, "duration_ms" => duration }
|
|
60
|
+
next if test.passed? || test.skipped?
|
|
61
|
+
|
|
62
|
+
result["success"] = false
|
|
63
|
+
result["failures"] << { "id" => id, "message" => test.failures.map(&:message).join("\n") }
|
|
64
|
+
break if fail_fast
|
|
65
|
+
ensure
|
|
66
|
+
R.current_test = nil
|
|
67
|
+
end
|
|
68
|
+
result
|
|
69
|
+
end
|
|
70
|
+
end
|
|
71
|
+
end
|
|
72
|
+
end
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "rspec"
|
|
4
|
+
require "stringio"
|
|
5
|
+
require_relative "../runtime"
|
|
6
|
+
|
|
7
|
+
module Ripr
|
|
8
|
+
module Integration
|
|
9
|
+
class RSpec
|
|
10
|
+
def initialize(test_files, seed: 0)
|
|
11
|
+
@test_files = test_files.map { |path| File.expand_path(path) }
|
|
12
|
+
@seed = seed
|
|
13
|
+
@executed = {}
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
def load
|
|
17
|
+
return self if @loaded
|
|
18
|
+
|
|
19
|
+
::RSpec::Core::Runner.disable_autorun!
|
|
20
|
+
adapter = self
|
|
21
|
+
::RSpec.configure do |config|
|
|
22
|
+
config.around(:each) { |example| adapter.record(example) }
|
|
23
|
+
end
|
|
24
|
+
options = ::RSpec::Core::ConfigurationOptions.new(["--seed", @seed.to_s, "--format", "progress", *@test_files])
|
|
25
|
+
@runner = ::RSpec::Core::Runner.new(options)
|
|
26
|
+
@output = StringIO.new
|
|
27
|
+
@runner.setup(@output, @output)
|
|
28
|
+
world = ::RSpec.world
|
|
29
|
+
@load_failed = world.wants_to_quit || world.non_example_failure
|
|
30
|
+
@groups = world.all_example_groups
|
|
31
|
+
@examples = @groups.flat_map { |group| world.filtered_examples[group] }
|
|
32
|
+
@loaded = true
|
|
33
|
+
self
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def tests
|
|
37
|
+
@examples.map do |example|
|
|
38
|
+
{ "id" => example.id, "file" => File.expand_path(example.metadata.fetch(:file_path)),
|
|
39
|
+
"line" => example.metadata.fetch(:line_number) }
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def run(ids = nil, fail_fast: false)
|
|
44
|
+
return { "success" => false, "tests" => {}, "failures" => [{ "message" => @output.string }] } if @load_failed
|
|
45
|
+
|
|
46
|
+
ids ||= @examples.map(&:id)
|
|
47
|
+
unknown = ids - @examples.map(&:id)
|
|
48
|
+
raise ArgumentError, "unknown test IDs: #{unknown.join(', ')}" unless unknown.empty?
|
|
49
|
+
return { "success" => false, "tests" => {}, "failures" => [{ "message" => "No tests selected" }] } if ids.empty?
|
|
50
|
+
|
|
51
|
+
@executed = {}
|
|
52
|
+
::RSpec.configuration.fail_fast = fail_fast
|
|
53
|
+
::RSpec.world.wants_to_quit = false
|
|
54
|
+
::RSpec.world.non_example_failure = false
|
|
55
|
+
@groups.each do |group|
|
|
56
|
+
::RSpec.world.filtered_examples[group] = @examples.select { |example| example.example_group == group && ids.include?(example.id) }
|
|
57
|
+
end
|
|
58
|
+
ranks = ids.each_with_index.to_h
|
|
59
|
+
ordering = lambda do |items|
|
|
60
|
+
items.sort_by do |item|
|
|
61
|
+
examples = item.respond_to?(:id) ? [item] : item.descendant_filtered_examples
|
|
62
|
+
examples.map { |example| ranks.fetch(example.id, ids.length) }.min || ids.length
|
|
63
|
+
end
|
|
64
|
+
end
|
|
65
|
+
::RSpec.configuration.ordering_registry.register(:global, ::RSpec::Core::Ordering::Custom.new(ordering))
|
|
66
|
+
status = @runner.run_specs(::RSpec.world.ordered_example_groups)
|
|
67
|
+
failures = @examples.filter_map do |example|
|
|
68
|
+
next unless @executed.key?(example.id) && example.execution_result.status == :failed
|
|
69
|
+
|
|
70
|
+
{ "id" => example.id, "message" => example.execution_result.exception.message }
|
|
71
|
+
end
|
|
72
|
+
failures << { "message" => @output.string } if !status.zero? && failures.empty?
|
|
73
|
+
{ "success" => status.zero?, "tests" => @executed, "failures" => failures }
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
def record(example)
|
|
77
|
+
R.current_test = example.id
|
|
78
|
+
started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
79
|
+
example.run
|
|
80
|
+
ensure
|
|
81
|
+
@executed[example.id] = { "file" => File.expand_path(example.metadata.fetch(:file_path)),
|
|
82
|
+
"line" => example.metadata.fetch(:line_number),
|
|
83
|
+
"duration_ms" => (Process.clock_gettime(Process::CLOCK_MONOTONIC) - started) * 1000 }
|
|
84
|
+
R.current_test = nil
|
|
85
|
+
end
|
|
86
|
+
end
|
|
87
|
+
end
|
|
88
|
+
end
|
data/lib/ripr/loader.rb
ADDED
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "monitor"
|
|
4
|
+
|
|
5
|
+
module Ripr
|
|
6
|
+
class Loader
|
|
7
|
+
@active_loaders = []
|
|
8
|
+
|
|
9
|
+
class << self
|
|
10
|
+
attr_reader :active_loaders
|
|
11
|
+
|
|
12
|
+
def instrumented?(path, expected_code)
|
|
13
|
+
active_loaders.any? { |loader| loader.instrumented?(path, expected_code) }
|
|
14
|
+
end
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
def initialize(code_by_path)
|
|
18
|
+
@code = code_by_path.to_h { |path, code| [File.expand_path(path), code] }
|
|
19
|
+
@paths = @code.keys.to_h { |path| [File.realpath(path), path] }
|
|
20
|
+
@hooks = []
|
|
21
|
+
@loading = []
|
|
22
|
+
@required = {}
|
|
23
|
+
@compiled = {}
|
|
24
|
+
@lock = Monitor.new
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def install
|
|
28
|
+
return self unless @hooks.empty?
|
|
29
|
+
|
|
30
|
+
if ENV["RIPR_LOADER"] == "require"
|
|
31
|
+
install_require_hook(Kernel, private_methods: true)
|
|
32
|
+
install_require_hook(Kernel.singleton_class, private_methods: false)
|
|
33
|
+
else
|
|
34
|
+
install_iseq_hook
|
|
35
|
+
end
|
|
36
|
+
Loader.active_loaders << self
|
|
37
|
+
self
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def uninstall
|
|
41
|
+
@hooks.each do |hook|
|
|
42
|
+
(hook.instance_methods(false) + hook.private_instance_methods(false)).each do |name|
|
|
43
|
+
hook.send(:remove_method, name)
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
@hooks.clear
|
|
47
|
+
@compiled.clear
|
|
48
|
+
Loader.active_loaders.delete(self)
|
|
49
|
+
self
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
def instrumented?(path, expected_code)
|
|
53
|
+
@compiled[File.realpath(path)] == expected_code
|
|
54
|
+
rescue Errno::ENOENT, Errno::ENOTDIR
|
|
55
|
+
false
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
def code_for(path)
|
|
59
|
+
@code[@paths[File.realpath(path)]]
|
|
60
|
+
rescue Errno::ENOENT, Errno::ENOTDIR
|
|
61
|
+
nil
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
def resolve_require(feature, extension: true)
|
|
65
|
+
feature = File.path(feature)
|
|
66
|
+
names = if !extension || feature.end_with?(".rb")
|
|
67
|
+
[feature]
|
|
68
|
+
else
|
|
69
|
+
["#{feature}.rb", "#{feature}.so", "#{feature}.bundle"]
|
|
70
|
+
end
|
|
71
|
+
directories = feature.start_with?("/", "./", "../") ? [Dir.pwd] : $LOAD_PATH
|
|
72
|
+
directories += [Dir.pwd] unless extension
|
|
73
|
+
directories.each do |directory|
|
|
74
|
+
names.each do |name|
|
|
75
|
+
path = File.expand_path(name, directory)
|
|
76
|
+
next unless File.file?(path)
|
|
77
|
+
|
|
78
|
+
return code_for(path) ? path : nil
|
|
79
|
+
end
|
|
80
|
+
end
|
|
81
|
+
nil
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
def require_target(path)
|
|
85
|
+
@lock.synchronize do
|
|
86
|
+
canonical = File.realpath(path)
|
|
87
|
+
return false if $LOADED_FEATURES.include?(path) || $LOADED_FEATURES.include?(@required[canonical]) || @loading.include?(canonical)
|
|
88
|
+
|
|
89
|
+
@loading << canonical
|
|
90
|
+
begin
|
|
91
|
+
evaluate(path)
|
|
92
|
+
$LOADED_FEATURES << path
|
|
93
|
+
@required[canonical] = path
|
|
94
|
+
true
|
|
95
|
+
ensure
|
|
96
|
+
@loading.delete(canonical)
|
|
97
|
+
end
|
|
98
|
+
end
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
def compile(path)
|
|
102
|
+
code = code_for(path)
|
|
103
|
+
instruction_sequence = RubyVM::InstructionSequence.compile(code, path, path)
|
|
104
|
+
@compiled[File.realpath(path)] = code
|
|
105
|
+
instruction_sequence
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
def evaluate(path)
|
|
109
|
+
compile(path).eval
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
private
|
|
113
|
+
|
|
114
|
+
def install_iseq_hook
|
|
115
|
+
loader = self
|
|
116
|
+
hook = Module.new do
|
|
117
|
+
define_method(:load_iseq) do |path|
|
|
118
|
+
code = loader.code_for(path)
|
|
119
|
+
if code
|
|
120
|
+
loader.compile(path)
|
|
121
|
+
else
|
|
122
|
+
defined?(super) ? super(path) : nil
|
|
123
|
+
end
|
|
124
|
+
end
|
|
125
|
+
end
|
|
126
|
+
RubyVM::InstructionSequence.singleton_class.prepend(hook)
|
|
127
|
+
@hooks << hook
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
def install_require_hook(target, private_methods:)
|
|
131
|
+
loader = self
|
|
132
|
+
hook = Module.new do
|
|
133
|
+
define_method(:require) do |feature|
|
|
134
|
+
path = loader.resolve_require(feature)
|
|
135
|
+
path ? loader.require_target(path) : super(feature)
|
|
136
|
+
end
|
|
137
|
+
|
|
138
|
+
define_method(:require_relative) do |feature|
|
|
139
|
+
location = caller_locations(1, 1).first
|
|
140
|
+
base = location&.absolute_path
|
|
141
|
+
raise LoadError, "cannot infer basepath" unless base
|
|
142
|
+
|
|
143
|
+
require(File.expand_path(File.path(feature), File.dirname(base)))
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
define_method(:load) do |feature, wrap = false|
|
|
147
|
+
path = loader.resolve_require(feature, extension: false)
|
|
148
|
+
if path
|
|
149
|
+
if wrap
|
|
150
|
+
raise ArgumentError,
|
|
151
|
+
"ripr require loader does not support wrapped load for instrumented files; use load_iseq"
|
|
152
|
+
end
|
|
153
|
+
loader.evaluate(path)
|
|
154
|
+
true
|
|
155
|
+
else
|
|
156
|
+
super(feature, wrap)
|
|
157
|
+
end
|
|
158
|
+
end
|
|
159
|
+
private :require, :require_relative, :load if private_methods
|
|
160
|
+
end
|
|
161
|
+
target.prepend(hook)
|
|
162
|
+
@hooks << hook
|
|
163
|
+
end
|
|
164
|
+
end
|
|
165
|
+
end
|