polyrun 3.0.1 → 3.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +12 -0
- data/README.md +4 -2
- data/docs/QUALITY.md +80 -0
- data/docs/QUALITY_CONFIGURATION.md +95 -0
- data/lib/polyrun/cli/ci_shard_run_command.rb +1 -1
- data/lib/polyrun/cli/graph_command.rb +68 -0
- data/lib/polyrun/cli/help.rb +5 -1
- data/lib/polyrun/cli/helpers.rb +3 -2
- data/lib/polyrun/cli/mutation_command.rb +67 -0
- data/lib/polyrun/cli/quality_commands.rb +65 -0
- data/lib/polyrun/cli/run_shards_plan_options.rb +1 -1
- data/lib/polyrun/cli/tool_options.rb +48 -0
- data/lib/polyrun/cli.rb +17 -1
- data/lib/polyrun/config/effective.rb +9 -0
- data/lib/polyrun/config/resolver.rb +9 -0
- data/lib/polyrun/config/schema.rb +8 -3
- data/lib/polyrun/config/suite.rb +6 -1
- data/lib/polyrun/config/tool_defaults.rb +42 -0
- data/lib/polyrun/config/tool_validation.rb +79 -0
- data/lib/polyrun/config/tools.rb +120 -0
- data/lib/polyrun/config.rb +6 -4
- data/lib/polyrun/graph/client.rb +64 -0
- data/lib/polyrun/graph/error.rb +13 -0
- data/lib/polyrun/graph/index.rb +75 -0
- data/lib/polyrun/graph/inspect.rb +58 -0
- data/lib/polyrun/graph/plan.rb +134 -0
- data/lib/polyrun/graph/planner.rb +117 -0
- data/lib/polyrun/graph/syntax.rb +114 -0
- data/lib/polyrun/graph/transaction.rb +81 -0
- data/lib/polyrun/graph/worker.rb +55 -0
- data/lib/polyrun/graph/workspace.rb +117 -0
- data/lib/polyrun/minitest_support.rb +1 -1
- data/lib/polyrun/mutation/candidates.rb +91 -0
- data/lib/polyrun/mutation/operators.rb +51 -0
- data/lib/polyrun/mutation/runner.rb +131 -0
- data/lib/polyrun/mutation/subjects.rb +88 -0
- data/lib/polyrun/quality/command.rb +84 -0
- data/lib/polyrun/quality/measure.rb +38 -0
- data/lib/polyrun/quality/observe.rb +53 -0
- data/lib/polyrun/quality/observe_hook.rb +103 -0
- data/lib/polyrun/quality/sandbox.rb +70 -0
- data/lib/polyrun/rspec_support.rb +1 -1
- data/lib/polyrun/spec_quality/config.rb +8 -3
- data/lib/polyrun/spec_quality.rb +3 -4
- data/lib/polyrun/version.rb +1 -1
- data/polyrun.gemspec +1 -1
- data/sig/polyrun/config.rbs +2 -2
- metadata +29 -1
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
require "tmpdir"
|
|
2
|
+
require_relative "../quality/command"
|
|
3
|
+
require_relative "../quality/sandbox"
|
|
4
|
+
require_relative "candidates"
|
|
5
|
+
|
|
6
|
+
module Polyrun
|
|
7
|
+
module Mutation
|
|
8
|
+
class Runner
|
|
9
|
+
def initialize(root: Dir.pwd, workers: nil, timeout: nil, arguments: [], max_mutations: nil,
|
|
10
|
+
max_report_bytes: nil, operators: nil, settings: {})
|
|
11
|
+
overrides = {"workers" => workers, "timeout" => timeout, "max_mutations" => max_mutations, "max_report_bytes" => max_report_bytes}.compact
|
|
12
|
+
@settings = Config::Tools.runtime("mutation", settings.merge(overrides))
|
|
13
|
+
raise ArgumentError, "workers exceed configured maximum" if @settings["workers"] > @settings["max_workers"]
|
|
14
|
+
|
|
15
|
+
@root, @workers, @timeout, @arguments = File.realpath(root), @settings["workers"], @settings["timeout"], arguments
|
|
16
|
+
@max_mutations, @max_report_bytes = @settings["max_mutations"], @settings["max_report_bytes"]
|
|
17
|
+
@operators = operators || Operators.select(@settings["operators"])
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
def call(subjects, shard: 1, total: 1)
|
|
21
|
+
subjects = select_subjects(subjects, shard, total)
|
|
22
|
+
return aggregate(subjects, [], nil) if subjects.empty?
|
|
23
|
+
raise ArgumentError, "a test command is required after --" if @arguments.empty?
|
|
24
|
+
|
|
25
|
+
mutations = Candidates.new(@root, operators: @operators, limit: @max_mutations, settings: @settings).call(subjects)
|
|
26
|
+
raise ArgumentError, "no supported mutations for selected subjects" if mutations.empty?
|
|
27
|
+
budget = {remaining: @max_report_bytes, mutex: Mutex.new}
|
|
28
|
+
Dir.mktmpdir("polyrun-mutations") do |template|
|
|
29
|
+
project = Quality::Sandbox.new(@root, settings: @settings["staging"])
|
|
30
|
+
enforce_copy_budget!(project.bytes, mutations.size)
|
|
31
|
+
|
|
32
|
+
project.copy_to(template)
|
|
33
|
+
baseline = retain_result(execute(template, nil), budget)
|
|
34
|
+
unless baseline["exitstatus"].zero? && baseline["code"] != "REPORT_LIMIT"
|
|
35
|
+
return aggregate(subjects, [], baseline).merge("ok" => false, "code" => "BASELINE_FAILED", "killed" => nil, "survived" => nil)
|
|
36
|
+
end
|
|
37
|
+
results = parallel(mutations, template, budget)
|
|
38
|
+
aggregate(subjects, results, baseline)
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
private
|
|
43
|
+
|
|
44
|
+
def enforce_copy_budget!(bytes, mutations)
|
|
45
|
+
raise ArgumentError, "mutation copy budget exceeded; narrow subjects or raise max_copy_bytes" if bytes * (mutations + 2) > @settings["max_copy_bytes"]
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
def parallel(mutations, template, budget)
|
|
49
|
+
pending = ::Queue.new
|
|
50
|
+
results = Array.new(mutations.size)
|
|
51
|
+
mutations.each_with_index { |mutation, index| pending << [mutation, index] }
|
|
52
|
+
threads = Array.new([@workers, mutations.size].min) do
|
|
53
|
+
Thread.new do # rubocop:disable ThreadSafety/NewThread -- bounded subprocess workers with isolated copies
|
|
54
|
+
loop do
|
|
55
|
+
mutation, index = pending.pop(true)
|
|
56
|
+
result = execute(template, mutation)
|
|
57
|
+
results[index] = retain_result(result, budget)
|
|
58
|
+
rescue ThreadError
|
|
59
|
+
break
|
|
60
|
+
end
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
threads.each(&:value)
|
|
64
|
+
results
|
|
65
|
+
ensure
|
|
66
|
+
threads&.each { |thread| thread.kill if thread.alive? }
|
|
67
|
+
threads&.each(&:join)
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
def execute(template, mutation)
|
|
71
|
+
Dir.mktmpdir("polyrun-mutation") do |directory|
|
|
72
|
+
project = Quality::Sandbox.new(template, settings: @settings["staging"])
|
|
73
|
+
project.copy_to(directory)
|
|
74
|
+
apply_mutation(directory, mutation) if mutation
|
|
75
|
+
command = @arguments.map { |argument| argument.start_with?("#{@root}/") ? File.join(directory, argument.delete_prefix("#{@root}/")) : argument }
|
|
76
|
+
result = Quality::Command.call(command, root: directory, timeout: @timeout, max_output: @settings["max_output_bytes"], env: Quality::Sandbox.environment(directory))
|
|
77
|
+
result.merge!(mutation) if mutation
|
|
78
|
+
result["code"] = {0 => mutation ? "SURVIVED" : "BASELINE_OK", 1 => mutation ? "KILLED" : "BASELINE_FAILED",
|
|
79
|
+
122 => "OUTPUT_LIMIT", 124 => "TIMEOUT", 127 => "COMMAND_UNAVAILABLE"}.fetch(result["exitstatus"], "COMMAND_FAILED")
|
|
80
|
+
result
|
|
81
|
+
end
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
def retain_result(result, budget)
|
|
85
|
+
budget[:mutex].synchronize do
|
|
86
|
+
size = result["stdout"].bytesize + result["stderr"].bytesize
|
|
87
|
+
if size > budget[:remaining]
|
|
88
|
+
result.merge!("stdout" => "", "stderr" => "", "code" => "REPORT_LIMIT")
|
|
89
|
+
else
|
|
90
|
+
budget[:remaining] -= size
|
|
91
|
+
end
|
|
92
|
+
end
|
|
93
|
+
result
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
def apply_mutation(directory, mutation)
|
|
97
|
+
file = File.join(directory, mutation["path"])
|
|
98
|
+
source = File.binread(file)
|
|
99
|
+
raise ArgumentError, "source changed before mutation testing" unless Digest::SHA256.hexdigest(source) == mutation["original_hash"]
|
|
100
|
+
|
|
101
|
+
source[mutation["start"]...mutation["end"]] = mutation["replacement"].b
|
|
102
|
+
File.binwrite(file, source)
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
def select_subjects(subjects, shard, total)
|
|
106
|
+
unless shard.is_a?(Integer) && total.is_a?(Integer) && total.positive? && (1..total).cover?(shard)
|
|
107
|
+
raise ArgumentError, "shard must be between 1 and total"
|
|
108
|
+
end
|
|
109
|
+
raise ArgumentError, "at least one subject is required" unless subjects.is_a?(Array) && !subjects.empty?
|
|
110
|
+
raise ArgumentError, "too many mutation subjects" if subjects.size > @settings["max_subjects"]
|
|
111
|
+
unless subjects.all? { |subject| valid_subject?(subject) }
|
|
112
|
+
raise ArgumentError, "invalid mutation subject"
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
subjects.uniq.sort.each_with_index.filter_map { |subject, index| subject if index % total == shard - 1 }
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
def valid_subject?(subject)
|
|
119
|
+
subject.is_a?(String) && !subject.empty? && !subject.start_with?("-") && !subject.match?(/[\s\x00]/)
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
def aggregate(subjects, results, baseline)
|
|
123
|
+
complete = results.all? { |result| %w[KILLED SURVIVED].include?(result["code"]) }
|
|
124
|
+
killed = complete ? results.count { |result| result["code"] == "KILLED" } : nil
|
|
125
|
+
survived = complete ? results.count { |result| result["code"] == "SURVIVED" } : nil
|
|
126
|
+
{"ok" => complete && survived.zero?, "engine" => "polyrun", "subjects" => subjects, "mutations" => results.size,
|
|
127
|
+
"killed" => killed, "survived" => survived, "baseline" => baseline, "results" => results}
|
|
128
|
+
end
|
|
129
|
+
end
|
|
130
|
+
end
|
|
131
|
+
end
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
require "prism"
|
|
2
|
+
require_relative "../graph/plan"
|
|
3
|
+
|
|
4
|
+
module Polyrun
|
|
5
|
+
module Mutation
|
|
6
|
+
module Subjects
|
|
7
|
+
module_function
|
|
8
|
+
|
|
9
|
+
def from_observation(path, max_bytes: 16_777_216)
|
|
10
|
+
raise ArgumentError, "observation is too large" if File.size(path) > max_bytes
|
|
11
|
+
|
|
12
|
+
observation = JSON.parse(File.read(path))
|
|
13
|
+
unless observation.is_a?(Hash) && observation["provenance"] == "observed" && observation["calls"].is_a?(Array)
|
|
14
|
+
raise ArgumentError, "invalid observation"
|
|
15
|
+
end
|
|
16
|
+
raise ArgumentError, "truncated observation requires explicit subjects" if observation["truncated"]
|
|
17
|
+
unless observation["calls"].all? { |row| row.is_a?(Hash) && row["subject"].is_a?(String) }
|
|
18
|
+
raise ArgumentError, "observation subjects must be strings"
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
observation["calls"].map { |row| row["subject"] }
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
def from_plan(root, id, settings: {})
|
|
25
|
+
plan = Graph::Plan.new(root, id, settings: settings)
|
|
26
|
+
plan.verify_originals!
|
|
27
|
+
plan.metadata.fetch("pre_hashes").keys.flat_map do |relative|
|
|
28
|
+
original = File.binread(File.join(root, relative))
|
|
29
|
+
staged = File.binread(File.join(plan.tree, relative))
|
|
30
|
+
next [] if original == staged
|
|
31
|
+
|
|
32
|
+
before = definitions(original)
|
|
33
|
+
after = definitions(staged)
|
|
34
|
+
(before.keys | after.keys).select { |subject| before[subject] != after[subject] }
|
|
35
|
+
end.uniq.sort
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
def definitions(source)
|
|
39
|
+
parsed = Prism.parse(source)
|
|
40
|
+
raise ArgumentError, "cannot select subjects from invalid Ruby" if parsed.failure?
|
|
41
|
+
|
|
42
|
+
result = {}
|
|
43
|
+
walk(parsed.value, [], false, result)
|
|
44
|
+
result
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def walk(node, owners, singleton, result)
|
|
48
|
+
if node.is_a?(Prism::ClassNode) || node.is_a?(Prism::ModuleNode)
|
|
49
|
+
owners = scope_owners(node, owners)
|
|
50
|
+
singleton = false
|
|
51
|
+
elsif node.is_a?(Prism::SingletonClassNode)
|
|
52
|
+
raise ArgumentError, "only self singleton classes support subject selection" unless node.expression.is_a?(Prism::SelfNode)
|
|
53
|
+
|
|
54
|
+
singleton = true
|
|
55
|
+
elsif node.is_a?(Prism::DefNode)
|
|
56
|
+
add_definition(node, owners, singleton, result)
|
|
57
|
+
return
|
|
58
|
+
end
|
|
59
|
+
node.compact_child_nodes.each { |child| walk(child, owners, singleton, result) }
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
def scope_owners(node, owners)
|
|
63
|
+
name = node.constant_path.full_name
|
|
64
|
+
if !owners.empty? && name.include?("::") && !name.start_with?("::")
|
|
65
|
+
raise ArgumentError, "qualified nested owners require explicit mutation subjects"
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
name.start_with?("::") ? [name.delete_prefix("::")] : owners + [name]
|
|
69
|
+
rescue Prism::ConstantPathNode::DynamicPartsInConstantPathError
|
|
70
|
+
raise ArgumentError, "dynamic constant owners require explicit mutation subjects"
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
def add_definition(node, owners, singleton, result)
|
|
74
|
+
raise ArgumentError, "top-level methods require explicit mutation subjects" if owners.empty?
|
|
75
|
+
if node.receiver && !node.receiver.is_a?(Prism::SelfNode)
|
|
76
|
+
raise ArgumentError, "dynamic method owners require explicit mutation subjects"
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
separator = (singleton || node.receiver) ? "." : "#"
|
|
80
|
+
subject = "#{owners.join("::")}#{separator}#{node.name}"
|
|
81
|
+
raise ArgumentError, "redefined methods require explicit mutation subjects" if result.key?(subject)
|
|
82
|
+
|
|
83
|
+
result[subject] = node.location.slice
|
|
84
|
+
end
|
|
85
|
+
private_class_method :definitions, :walk, :scope_owners, :add_definition
|
|
86
|
+
end
|
|
87
|
+
end
|
|
88
|
+
end
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
require "io/wait"
|
|
2
|
+
|
|
3
|
+
module Polyrun
|
|
4
|
+
module Quality
|
|
5
|
+
module Command
|
|
6
|
+
DEFAULT_OUTPUT_LIMIT = 1_048_576
|
|
7
|
+
|
|
8
|
+
module_function
|
|
9
|
+
|
|
10
|
+
# rubocop:disable Metrics/AbcSize -- keep descriptor ownership and process cleanup in the same scope
|
|
11
|
+
def call(argv, root:, timeout:, env: {}, max_output: DEFAULT_OUTPUT_LIMIT)
|
|
12
|
+
timeout = Float(timeout)
|
|
13
|
+
validate!(argv, timeout, max_output)
|
|
14
|
+
|
|
15
|
+
started = monotonic
|
|
16
|
+
streams = []
|
|
17
|
+
2.times { streams << IO.pipe }
|
|
18
|
+
output = {"stdout" => +"".b, "stderr" => +"".b}
|
|
19
|
+
pid = Process.spawn(env, [argv.first, argv.first], *argv.drop(1), chdir: root, in: File::NULL, out: streams[0][1], err: streams[1][1], pgroup: true)
|
|
20
|
+
streams.each { |_, writer| writer.close }
|
|
21
|
+
code = collect(pid, streams.map(&:first), output, started + timeout, max_output)
|
|
22
|
+
output.each_value { |value| value.force_encoding(Encoding::UTF_8).scrub!("") }
|
|
23
|
+
output.merge("exitstatus" => code, "wall_s" => monotonic - started)
|
|
24
|
+
rescue Errno::ENOENT => e
|
|
25
|
+
{"stdout" => "", "stderr" => e.message, "exitstatus" => 127, "wall_s" => monotonic - started}
|
|
26
|
+
ensure
|
|
27
|
+
terminate(pid) if pid
|
|
28
|
+
streams&.flatten&.each { |stream| stream.close unless stream.closed? }
|
|
29
|
+
end
|
|
30
|
+
# rubocop:enable Metrics/AbcSize
|
|
31
|
+
|
|
32
|
+
def collect(pid, readers, output, deadline, limit)
|
|
33
|
+
active = readers.dup
|
|
34
|
+
status = nil
|
|
35
|
+
loop do
|
|
36
|
+
status ||= Process.waitpid2(pid, Process::WNOHANG)&.last
|
|
37
|
+
return status.exitstatus || 128 + status.termsig if status && active.empty?
|
|
38
|
+
return 124 if monotonic >= deadline
|
|
39
|
+
|
|
40
|
+
return 122 if drain(active, readers, output, deadline, limit)
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
def drain(active, readers, output, deadline, limit)
|
|
45
|
+
ready = IO.select(active, nil, nil, (deadline - monotonic).clamp(0.0001, 0.001))&.first || []
|
|
46
|
+
ready.each do |reader|
|
|
47
|
+
chunk = reader.read_nonblock(16_384, exception: false)
|
|
48
|
+
if chunk.nil?
|
|
49
|
+
active.delete(reader)
|
|
50
|
+
elsif chunk != :wait_readable
|
|
51
|
+
key = (reader == readers.first) ? "stdout" : "stderr"
|
|
52
|
+
remaining = limit - output.values.sum(&:bytesize)
|
|
53
|
+
output[key] << chunk.byteslice(0, remaining)
|
|
54
|
+
return true if chunk.bytesize > remaining
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
false
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
def validate!(argv, timeout, max_output)
|
|
61
|
+
raise ArgumentError, "timeout must be finite and positive" unless timeout.finite? && timeout.positive?
|
|
62
|
+
raise ArgumentError, "command must not be empty" unless argv.is_a?(Array) && !argv.empty?
|
|
63
|
+
raise ArgumentError, "output limit must be positive" unless max_output.is_a?(Integer) && max_output.positive?
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
def terminate(pid)
|
|
67
|
+
Process.kill("KILL", -pid)
|
|
68
|
+
rescue Errno::ESRCH
|
|
69
|
+
nil
|
|
70
|
+
ensure
|
|
71
|
+
begin
|
|
72
|
+
Process.waitpid(pid)
|
|
73
|
+
rescue Errno::ECHILD
|
|
74
|
+
nil
|
|
75
|
+
end
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
def monotonic
|
|
79
|
+
Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
80
|
+
end
|
|
81
|
+
private_class_method :collect, :drain, :validate!, :terminate, :monotonic
|
|
82
|
+
end
|
|
83
|
+
end
|
|
84
|
+
end
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
require_relative "command"
|
|
2
|
+
require_relative "../benchmark/pressure/process_sample"
|
|
3
|
+
|
|
4
|
+
module Polyrun
|
|
5
|
+
module Quality
|
|
6
|
+
module Measure
|
|
7
|
+
module_function
|
|
8
|
+
|
|
9
|
+
def call(command, root:, timeout:, label:, ceilings: {}, max_output: Command::DEFAULT_OUTPUT_LIMIT)
|
|
10
|
+
validate_ceilings!(ceilings)
|
|
11
|
+
before = Process.times
|
|
12
|
+
result = Command.call(command, root: root, timeout: timeout, max_output: max_output)
|
|
13
|
+
metrics = process_metrics(before).merge("wall_s" => result["wall_s"])
|
|
14
|
+
failures = ceilings.keys.select { |key| !metrics[key].is_a?(Numeric) || metrics[key] > ceilings[key] }
|
|
15
|
+
result.merge("ok" => result["exitstatus"].zero? && failures.empty?, "label" => label, "metrics" => metrics,
|
|
16
|
+
"failures" => failures, "memory_probe_class" => "orchestrator_sample", "network" => "not_measured", "energy_proxy" => "cpu_seconds")
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
def process_metrics(before)
|
|
20
|
+
after = Process.times
|
|
21
|
+
rss = Benchmark::Pressure::ProcessSample.rss_bytes
|
|
22
|
+
{
|
|
23
|
+
"cpu_s" => after.utime + after.stime - before.utime - before.stime,
|
|
24
|
+
"child_cpu_s" => after.cutime + after.cstime - before.cutime - before.cstime,
|
|
25
|
+
"orchestrator_rss_bytes" => rss.positive? ? rss : nil
|
|
26
|
+
}
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def validate_ceilings!(ceilings)
|
|
30
|
+
raise ArgumentError, "ceilings must be an object" unless ceilings.is_a?(Hash)
|
|
31
|
+
unless ceilings.values.all? { |value| value.is_a?(Numeric) && value.finite? && value >= 0 }
|
|
32
|
+
raise ArgumentError, "ceilings must be finite nonnegative numbers"
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
private_class_method :process_metrics, :validate_ceilings!
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
end
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
require "json"
|
|
2
|
+
require "tmpdir"
|
|
3
|
+
require_relative "command"
|
|
4
|
+
|
|
5
|
+
module Polyrun
|
|
6
|
+
module Quality
|
|
7
|
+
module Observe
|
|
8
|
+
module_function
|
|
9
|
+
|
|
10
|
+
def call(command, root:, output:, timeout: 300, limit: 10_000, max_output: Command::DEFAULT_OUTPUT_LIMIT, max_artifact_bytes: 16_777_216)
|
|
11
|
+
raise ArgumentError, "max entries must be positive" unless limit.is_a?(Integer) && limit.positive?
|
|
12
|
+
|
|
13
|
+
output = File.expand_path(output)
|
|
14
|
+
Dir.mktmpdir("polyrun-observe", File.dirname(output)) do |directory|
|
|
15
|
+
artifact = File.join(directory, "trace.json")
|
|
16
|
+
result = Command.call(command, root: root, timeout: timeout, max_output: max_output, env: environment(root, artifact, limit))
|
|
17
|
+
present = File.file?(artifact)
|
|
18
|
+
if present
|
|
19
|
+
validate_artifact!(artifact, max_artifact_bytes)
|
|
20
|
+
File.rename(artifact, output)
|
|
21
|
+
end
|
|
22
|
+
result["ok"] = result["exitstatus"].zero? && present
|
|
23
|
+
result["code"] = if !result["exitstatus"].zero?
|
|
24
|
+
"COMMAND_FAILED"
|
|
25
|
+
elsif present
|
|
26
|
+
"OBSERVED"
|
|
27
|
+
else
|
|
28
|
+
"TRACE_MISSING"
|
|
29
|
+
end
|
|
30
|
+
result
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def environment(root, artifact, limit)
|
|
35
|
+
library = File.expand_path("../..", __dir__)
|
|
36
|
+
{"POLYRUN_OBSERVE_ROOT" => File.realpath(root), "POLYRUN_OBSERVE_OUTPUT" => artifact,
|
|
37
|
+
"POLYRUN_OBSERVE_LIMIT" => limit.to_s, "POLYRUN_OBSERVE_RUBYOPT" => ENV["RUBYOPT"],
|
|
38
|
+
"RUBYLIB" => [library, ENV["RUBYLIB"]].compact.join(File::PATH_SEPARATOR),
|
|
39
|
+
"RUBYOPT" => "#{ENV["RUBYOPT"]} -rpolyrun/quality/observe_hook"}
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
def validate_artifact!(path, max_bytes)
|
|
43
|
+
raise ArgumentError, "observation artifact is too large" if File.size(path) > max_bytes
|
|
44
|
+
|
|
45
|
+
trace = JSON.parse(File.read(path))
|
|
46
|
+
unless trace.is_a?(Hash) && trace["calls"].is_a?(Array) && trace["provenance"] == "observed"
|
|
47
|
+
raise ArgumentError, "invalid observation artifact"
|
|
48
|
+
end
|
|
49
|
+
end
|
|
50
|
+
private_class_method :environment, :validate_artifact!
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
end
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
require "json"
|
|
2
|
+
|
|
3
|
+
module Polyrun
|
|
4
|
+
module Quality
|
|
5
|
+
module ObserveHook
|
|
6
|
+
class Recorder
|
|
7
|
+
def initialize(root, output, limit)
|
|
8
|
+
@root = File.realpath(root) + "/"
|
|
9
|
+
@output = output
|
|
10
|
+
@limit = limit
|
|
11
|
+
@calls = {}
|
|
12
|
+
@paths = {}
|
|
13
|
+
@owners = {}
|
|
14
|
+
@entries = 0
|
|
15
|
+
@truncated = false
|
|
16
|
+
@process = Process.pid
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
def record(event)
|
|
20
|
+
path = relative_path(event.path)
|
|
21
|
+
return unless path
|
|
22
|
+
|
|
23
|
+
owner = event.defined_class
|
|
24
|
+
name = event.method_id
|
|
25
|
+
methods = @calls.dig(path, owner)
|
|
26
|
+
count = methods && methods[name]
|
|
27
|
+
if count
|
|
28
|
+
methods[name] = count + 1
|
|
29
|
+
elsif @entries < @limit
|
|
30
|
+
identity = owner_identity(event)
|
|
31
|
+
return unless identity
|
|
32
|
+
|
|
33
|
+
@owners[owner] = identity
|
|
34
|
+
methods ||= ((@calls[path] ||= {})[owner] ||= {})
|
|
35
|
+
methods[name] = 1
|
|
36
|
+
@entries += 1
|
|
37
|
+
else
|
|
38
|
+
@truncated = true
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
def write
|
|
43
|
+
return unless Process.pid == @process
|
|
44
|
+
|
|
45
|
+
rows = @calls.flat_map do |path, owners|
|
|
46
|
+
owners.flat_map do |owner, methods|
|
|
47
|
+
name, separator = @owners.fetch(owner)
|
|
48
|
+
methods.map do |method, count|
|
|
49
|
+
{"path" => path, "owner" => name, "method" => method.to_s, "subject" => "#{name}#{separator}#{method}", "count" => count}
|
|
50
|
+
end
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
File.write(@output, JSON.generate("calls" => rows, "truncated" => @truncated, "provenance" => "observed"))
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
private
|
|
57
|
+
|
|
58
|
+
def relative_path(source)
|
|
59
|
+
path = @paths[source]
|
|
60
|
+
if path.nil?
|
|
61
|
+
resolved = File.file?(source) ? File.realpath(source) : nil
|
|
62
|
+
path = resolved&.start_with?(@root) ? resolved.delete_prefix(@root) : false
|
|
63
|
+
@paths[source] = path if @paths.size < @limit
|
|
64
|
+
end
|
|
65
|
+
path
|
|
66
|
+
rescue Errno::ENOENT
|
|
67
|
+
false
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
def owner_identity(event)
|
|
71
|
+
owner = event.defined_class
|
|
72
|
+
name = owner.name
|
|
73
|
+
separator = "#"
|
|
74
|
+
if owner.singleton_class? && event.self.is_a?(Module)
|
|
75
|
+
name = event.self.name
|
|
76
|
+
separator = "."
|
|
77
|
+
end
|
|
78
|
+
[name, separator] if name.is_a?(String) && !name.empty?
|
|
79
|
+
end
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
module_function
|
|
83
|
+
|
|
84
|
+
def install
|
|
85
|
+
root = ENV.delete("POLYRUN_OBSERVE_ROOT")
|
|
86
|
+
output = ENV.delete("POLYRUN_OBSERVE_OUTPUT")
|
|
87
|
+
limit = Integer(ENV.delete("POLYRUN_OBSERVE_LIMIT") || "10000")
|
|
88
|
+
ENV["RUBYOPT"] = ENV.delete("POLYRUN_OBSERVE_RUBYOPT")
|
|
89
|
+
return unless root && output
|
|
90
|
+
|
|
91
|
+
recorder = Recorder.new(root, output, limit)
|
|
92
|
+
trace = TracePoint.new(:call) { |event| recorder.record(event) }
|
|
93
|
+
at_exit do
|
|
94
|
+
trace.disable
|
|
95
|
+
recorder.write
|
|
96
|
+
end
|
|
97
|
+
trace.enable
|
|
98
|
+
end
|
|
99
|
+
end
|
|
100
|
+
end
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
Polyrun::Quality::ObserveHook.install
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
require "digest"
|
|
2
|
+
require "fileutils"
|
|
3
|
+
require "find"
|
|
4
|
+
require_relative "../config/tools"
|
|
5
|
+
|
|
6
|
+
module Polyrun
|
|
7
|
+
module Quality
|
|
8
|
+
class Sandbox
|
|
9
|
+
attr_reader :bytes
|
|
10
|
+
|
|
11
|
+
def initialize(root, settings: {})
|
|
12
|
+
@settings = Config::Tools.merge(Config::ToolDefaults::STAGING, Config::Tools.stringify(settings))
|
|
13
|
+
errors = Config::ToolValidation.check(@settings, Config::ToolDefaults::STAGING, "staging")
|
|
14
|
+
raise ArgumentError, errors.join("; ") unless errors.empty?
|
|
15
|
+
|
|
16
|
+
@root = File.realpath(root)
|
|
17
|
+
@bytes = 0
|
|
18
|
+
@files = inventory
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
def copy_to(destination)
|
|
22
|
+
@files.each do |relative, expected|
|
|
23
|
+
source = File.join(@root, relative)
|
|
24
|
+
raise ArgumentError, "project changed while staging" unless File.file?(source) && !File.symlink?(source) && Digest::SHA256.file(source).hexdigest == expected
|
|
25
|
+
|
|
26
|
+
target = File.join(destination, relative)
|
|
27
|
+
FileUtils.mkdir_p(File.dirname(target))
|
|
28
|
+
FileUtils.copy_file(source, target, true)
|
|
29
|
+
raise ArgumentError, "project changed while staging" unless Digest::SHA256.file(target).hexdigest == expected
|
|
30
|
+
end
|
|
31
|
+
destination
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def self.environment(destination)
|
|
35
|
+
{"BUNDLE_GEMFILE" => File.file?(File.join(destination, "Gemfile")) ? File.join(destination, "Gemfile") : nil,
|
|
36
|
+
"RUBYLIB" => [File.join(destination, "lib"), ENV["RUBYLIB"]].compact.join(File::PATH_SEPARATOR)}
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
private
|
|
40
|
+
|
|
41
|
+
def inventory
|
|
42
|
+
files = {}
|
|
43
|
+
Find.find(@root) do |path|
|
|
44
|
+
next if path == @root
|
|
45
|
+
|
|
46
|
+
name = File.basename(path)
|
|
47
|
+
relative = path.delete_prefix("#{@root}/")
|
|
48
|
+
if excluded?(name) || @settings["exclude_paths"].any? { |pattern| File.fnmatch?(pattern, relative, File::FNM_PATHNAME) }
|
|
49
|
+
Find.prune if File.directory?(path)
|
|
50
|
+
next
|
|
51
|
+
end
|
|
52
|
+
raise ArgumentError, "symbolic links are not supported in staged projects" if File.symlink?(path)
|
|
53
|
+
next unless File.file?(path)
|
|
54
|
+
|
|
55
|
+
@bytes += File.size(path)
|
|
56
|
+
raise ArgumentError, "staged project exceeds configured file or byte limit" if @bytes > @settings["max_bytes"] || files.size >= @settings["max_files"]
|
|
57
|
+
|
|
58
|
+
files[path.delete_prefix("#{@root}/")] = Digest::SHA256.file(path).hexdigest
|
|
59
|
+
end
|
|
60
|
+
files
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
def excluded?(name)
|
|
64
|
+
return true if name.start_with?(".") && !@settings["include_hidden"].include?(name)
|
|
65
|
+
|
|
66
|
+
@settings["exclude"].any? { |pattern| File.fnmatch?(pattern, name) } || name.end_with?(".key")
|
|
67
|
+
end
|
|
68
|
+
end
|
|
69
|
+
end
|
|
70
|
+
end
|
|
@@ -50,7 +50,7 @@ module Polyrun
|
|
|
50
50
|
# Writes {WorkerPing} after suite start, before/after each example (+location+ is file:line from metadata).
|
|
51
51
|
# Keeps +--worker-idle-timeout+ sensitive to example progress (not only a background thread).
|
|
52
52
|
def install_spec_quality!(only_if: nil, root: nil, output_path: nil)
|
|
53
|
-
pred = only_if || -> { Polyrun::SpecQuality.enabled? }
|
|
53
|
+
pred = only_if || -> { Polyrun::SpecQuality.enabled?(root: root || Dir.pwd) }
|
|
54
54
|
return unless pred.call
|
|
55
55
|
|
|
56
56
|
require_relative "spec_quality/rspec_hook"
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
require "yaml"
|
|
2
|
+
require_relative "../config"
|
|
2
3
|
|
|
3
4
|
module Polyrun
|
|
4
5
|
module SpecQuality
|
|
@@ -25,10 +26,12 @@ module Polyrun
|
|
|
25
26
|
|
|
26
27
|
module_function
|
|
27
28
|
|
|
28
|
-
def enabled?(env = ENV)
|
|
29
|
+
def enabled?(env = ENV, root: Dir.pwd)
|
|
29
30
|
return false if disabled?(env)
|
|
30
31
|
|
|
31
|
-
truthy?(env["POLYRUN_SPEC_QUALITY"]) || truthy?(env["POLYRUN_SPEC_QUALITY_FRAGMENTS"])
|
|
32
|
+
return true if truthy?(env["POLYRUN_SPEC_QUALITY"]) || truthy?(env["POLYRUN_SPEC_QUALITY_FRAGMENTS"])
|
|
33
|
+
|
|
34
|
+
Polyrun::Config::Tools.spec_quality(root: root, env: env)["enabled"] == true
|
|
32
35
|
end
|
|
33
36
|
|
|
34
37
|
def disabled?(env = ENV)
|
|
@@ -38,7 +41,9 @@ module Polyrun
|
|
|
38
41
|
def load(root:, config_path: nil, env: ENV, **overrides)
|
|
39
42
|
root = File.expand_path(root)
|
|
40
43
|
file_cfg = load_yaml(root, config_path)
|
|
41
|
-
|
|
44
|
+
shared = Polyrun::Config::Tools.spec_quality(root: root, env: env).except("enabled")
|
|
45
|
+
merged = DEFAULTS.merge(stringify_keys(file_cfg)).merge(shared)
|
|
46
|
+
merged.merge!(stringify_keys(file_cfg)) if config_path
|
|
42
47
|
apply_env!(merged, env)
|
|
43
48
|
merged.merge!(stringify_keys(overrides).transform_keys(&:to_s))
|
|
44
49
|
merged["root"] = root
|
data/lib/polyrun/spec_quality.rb
CHANGED
|
@@ -37,8 +37,8 @@ module Polyrun
|
|
|
37
37
|
instance_variable_defined?(:@config) && @config
|
|
38
38
|
end
|
|
39
39
|
|
|
40
|
-
def enabled?
|
|
41
|
-
Config.enabled? && !Config.disabled?
|
|
40
|
+
def enabled?(root: Dir.pwd)
|
|
41
|
+
Config.enabled?(ENV, root: root) && !Config.disabled?
|
|
42
42
|
end
|
|
43
43
|
|
|
44
44
|
attr_reader :config
|
|
@@ -130,8 +130,7 @@ module Polyrun
|
|
|
130
130
|
|
|
131
131
|
def spec_quality_requested_for_quick?(root = Dir.pwd)
|
|
132
132
|
return false if Config.disabled?
|
|
133
|
-
return true if
|
|
134
|
-
return true if %w[1 true yes].include?(ENV["POLYRUN_SPEC_QUALITY_FRAGMENTS"]&.to_s&.downcase)
|
|
133
|
+
return true if Config.enabled?(ENV, root: root)
|
|
135
134
|
|
|
136
135
|
path = File.join(File.expand_path(root), Config::DEFAULT_CONFIG_RELATIVE)
|
|
137
136
|
return false unless File.file?(path)
|
data/lib/polyrun/version.rb
CHANGED
data/polyrun.gemspec
CHANGED
|
@@ -11,7 +11,7 @@ Gem::Specification.new do |spec|
|
|
|
11
11
|
spec.required_ruby_version = ">= 3.4"
|
|
12
12
|
|
|
13
13
|
spec.files = (
|
|
14
|
-
Dir["lib/**/*", "sig/**/*.rbs", "bin/polyrun", "README.md", "CHANGELOG.md", "docs/SETUP_PROFILE.md", "LICENSE", "CONTRIBUTING.md", "CODE_OF_CONDUCT.md", "SECURITY.md", "polyrun.gemspec"] +
|
|
14
|
+
Dir["lib/**/*", "sig/**/*.rbs", "bin/polyrun", "README.md", "CHANGELOG.md", "docs/SETUP_PROFILE.md", "docs/QUALITY.md", "docs/QUALITY_CONFIGURATION.md", "LICENSE", "CONTRIBUTING.md", "CODE_OF_CONDUCT.md", "SECURITY.md", "polyrun.gemspec"] +
|
|
15
15
|
Dir["ext/**/extconf.rb", "ext/**/*.c", "ext/**/*.h"]
|
|
16
16
|
).uniq.reject { |f| File.directory?(f) }
|
|
17
17
|
spec.bindir = "bin"
|
data/sig/polyrun/config.rbs
CHANGED
|
@@ -5,8 +5,8 @@ module Polyrun
|
|
|
5
5
|
attr_reader path: String?
|
|
6
6
|
attr_reader raw: untyped
|
|
7
7
|
|
|
8
|
-
def self.load: (?path: String?) -> Config
|
|
9
|
-
def self.resolve_path: (String? explicit) -> String?
|
|
8
|
+
def self.load: (?path: String?, ?suite: String?, ?root: String) -> Config
|
|
9
|
+
def self.resolve_path: (String? explicit, ?root: String) -> String?
|
|
10
10
|
|
|
11
11
|
def initialize: (path: String?, raw: untyped) -> void
|
|
12
12
|
|