polyrun 2.1.3 → 2.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +10 -1
- data/CONTRIBUTING.md +22 -0
- data/docs/SETUP_PROFILE.md +1 -1
- data/ext/polyrun_coverage_merge/coverage_merge.c +492 -0
- data/ext/polyrun_coverage_merge/extconf.rb +3 -0
- data/lib/polyrun/benchmark/profile.rb +191 -0
- data/lib/polyrun/benchmark/report.rb +87 -0
- data/lib/polyrun/cli/benchmark_commands.rb +66 -0
- data/lib/polyrun/cli/coverage_commands.rb +4 -4
- data/lib/polyrun/cli/coverage_merge_io.rb +30 -4
- data/lib/polyrun/cli/failure_commands.rb +10 -3
- data/lib/polyrun/cli/help.rb +12 -8
- data/lib/polyrun/cli/report_commands.rb +25 -11
- data/lib/polyrun/cli/run_shards_parallel_children.rb +1 -1
- data/lib/polyrun/cli/run_shards_plan_options.rb +1 -1
- data/lib/polyrun/cli/spec_quality_commands.rb +16 -7
- data/lib/polyrun/cli.rb +8 -1
- data/lib/polyrun/coverage/example_diff.rb +127 -60
- data/lib/polyrun/coverage/example_diff_snapshot.rb +44 -0
- data/lib/polyrun/coverage/example_diff_track_filter.rb +51 -0
- data/lib/polyrun/coverage/file_stats_report.rb +36 -0
- data/lib/polyrun/coverage/formatter.rb +22 -1
- data/lib/polyrun/coverage/merge/formatters.rb +11 -3
- data/lib/polyrun/coverage/merge_merge_two.rb +112 -63
- data/lib/polyrun/coverage/merge_native.rb +37 -0
- data/lib/polyrun/coverage/reporting.rb +1 -1
- data/lib/polyrun/export/csv.rb +25 -0
- data/lib/polyrun/export/markdown.rb +47 -0
- data/lib/polyrun/hooks/shell_runner.rb +33 -0
- data/lib/polyrun/hooks.rb +4 -7
- data/lib/polyrun/reporting/failure_merge.rb +48 -12
- data/lib/polyrun/reporting/junit.rb +8 -7
- data/lib/polyrun/reporting/junit_report.rb +87 -0
- data/lib/polyrun/rspec/example_debug.rb +1 -1
- data/lib/polyrun/rspec/sharded_formatter_compat.rb +30 -7
- data/lib/polyrun/spec_quality/profile.rb +24 -12
- data/lib/polyrun/spec_quality/report.rb +93 -0
- data/lib/polyrun/spec_quality.rb +19 -11
- data/lib/polyrun/timing/summary.rb +55 -5
- data/lib/polyrun/version.rb +1 -1
- data/polyrun.gemspec +8 -1
- metadata +58 -2
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
require "fileutils"
|
|
2
|
+
require "json"
|
|
3
|
+
require "open3"
|
|
4
|
+
|
|
5
|
+
module Polyrun
|
|
6
|
+
module Benchmark
|
|
7
|
+
# Records benchmark output during performance specs and writes profile artifacts.
|
|
8
|
+
# Set POLYRUN_BENCH=1 to echo lines to stdout. JSON sidecar enables report-benchmark exports.
|
|
9
|
+
module Profile
|
|
10
|
+
module_function
|
|
11
|
+
|
|
12
|
+
def reset!
|
|
13
|
+
lines_storage.clear
|
|
14
|
+
metrics_storage.clear
|
|
15
|
+
current_section_storage.replace(["default"])
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
def log(message = "")
|
|
19
|
+
line = message.to_s
|
|
20
|
+
lines_storage << line
|
|
21
|
+
section = detect_section_title(line) || current_section_storage.first
|
|
22
|
+
current_section_storage[0] = section if detect_section_title(line)
|
|
23
|
+
$stdout.puts(line) if verbose? && !line.empty?
|
|
24
|
+
line
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def record_metric!(name:, value:, unit: "seconds", section: nil)
|
|
28
|
+
section_name = section || current_section_storage.first
|
|
29
|
+
metric = {
|
|
30
|
+
"section" => section_name.to_s,
|
|
31
|
+
"name" => name.to_s,
|
|
32
|
+
"value" => value,
|
|
33
|
+
"unit" => unit.to_s
|
|
34
|
+
}
|
|
35
|
+
metrics_storage << metric
|
|
36
|
+
log(format_metric_line(metric))
|
|
37
|
+
metric
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def snapshot
|
|
41
|
+
{
|
|
42
|
+
"meta" => profile_meta,
|
|
43
|
+
"lines" => lines_storage.dup,
|
|
44
|
+
"metrics" => metrics_storage.dup
|
|
45
|
+
}
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
def write!(repository_root: default_repository_root)
|
|
49
|
+
data = snapshot
|
|
50
|
+
return if data["lines"].empty? && data["metrics"].empty?
|
|
51
|
+
|
|
52
|
+
log_path = output_path(repository_root: repository_root, extension: "log")
|
|
53
|
+
json_path = output_path(repository_root: repository_root, extension: "json")
|
|
54
|
+
FileUtils.mkdir_p(File.dirname(log_path))
|
|
55
|
+
File.write(log_path, profile_header(data["meta"]) + data["lines"].join("\n") + "\n")
|
|
56
|
+
File.write(json_path, JSON.pretty_generate(data))
|
|
57
|
+
export_sidecars!(data, json_path)
|
|
58
|
+
$stdout.puts("\nBenchmark profile written to #{log_path}") if verbose?
|
|
59
|
+
log_path
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
def export_sidecars!(data, json_path)
|
|
63
|
+
formats = export_formats
|
|
64
|
+
return if formats.empty?
|
|
65
|
+
|
|
66
|
+
require_relative "report"
|
|
67
|
+
base = json_path.sub(/\.json\z/, "")
|
|
68
|
+
formats.each do |format|
|
|
69
|
+
path = "#{base}.#{export_extension(format)}"
|
|
70
|
+
File.write(path, Report.render(data, format: format))
|
|
71
|
+
end
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
def export_formats
|
|
75
|
+
raw = ENV["POLYRUN_BENCH_FORMATS"]
|
|
76
|
+
return [] if raw.nil? || raw.to_s.strip.empty?
|
|
77
|
+
|
|
78
|
+
raw.split(",").map(&:strip).reject(&:empty?)
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
def export_extension(format)
|
|
82
|
+
case format.to_s.downcase
|
|
83
|
+
when "markdown", "md" then "md"
|
|
84
|
+
when "text", "console", "txt" then "txt"
|
|
85
|
+
else format.to_s.downcase
|
|
86
|
+
end
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
def verbose?
|
|
90
|
+
%w[1 true yes].include?(ENV["POLYRUN_BENCH"]&.to_s&.downcase)
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
def output_path(repository_root: default_repository_root, extension: "log", commit_sha: nil, working_tree_clean: nil, timestamp: nil)
|
|
94
|
+
commit_identifier = commit_sha || self.commit_sha(repository_root: repository_root)
|
|
95
|
+
clean_tree = working_tree_clean.nil? ? working_tree_clean?(repository_root: repository_root) : working_tree_clean
|
|
96
|
+
filename = if clean_tree
|
|
97
|
+
"profile_#{commit_identifier}.#{extension}"
|
|
98
|
+
else
|
|
99
|
+
recorded_at = timestamp || self.timestamp
|
|
100
|
+
"profile_#{commit_identifier}_#{recorded_at}.#{extension}"
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
File.join(repository_root, "tmp", "benchmarks", filename)
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
def profile_meta(repository_root: default_repository_root)
|
|
107
|
+
{
|
|
108
|
+
"commit" => commit_sha(repository_root: repository_root),
|
|
109
|
+
"recorded_at" => Time.now.utc.iso8601,
|
|
110
|
+
"working_tree_clean" => working_tree_clean?(repository_root: repository_root),
|
|
111
|
+
"ruby" => RUBY_VERSION,
|
|
112
|
+
"polyrun_version" => Polyrun::VERSION
|
|
113
|
+
}
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
def profile_header(meta)
|
|
117
|
+
[
|
|
118
|
+
"# Benchmark profile",
|
|
119
|
+
"# commit: #{meta["commit"]}",
|
|
120
|
+
"# recorded_at: #{meta["recorded_at"]}",
|
|
121
|
+
"# working_tree_clean: #{meta["working_tree_clean"]}",
|
|
122
|
+
"# ruby: #{meta["ruby"]}",
|
|
123
|
+
""
|
|
124
|
+
].join("\n")
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
def commit_sha(repository_root: default_repository_root)
|
|
128
|
+
git_command("git rev-parse HEAD", repository_root: repository_root) || "unknown"
|
|
129
|
+
end
|
|
130
|
+
|
|
131
|
+
def working_tree_clean?(repository_root: default_repository_root)
|
|
132
|
+
git_command("git status --porcelain", repository_root: repository_root).to_s.empty?
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
def timestamp
|
|
136
|
+
Time.now.utc.strftime("%Y%m%d%H%M%S")
|
|
137
|
+
end
|
|
138
|
+
|
|
139
|
+
def git_command(command, repository_root:)
|
|
140
|
+
stdout, status = Open3.capture2(command, chdir: repository_root)
|
|
141
|
+
return nil unless status.success?
|
|
142
|
+
|
|
143
|
+
stdout.strip
|
|
144
|
+
rescue
|
|
145
|
+
nil
|
|
146
|
+
end
|
|
147
|
+
|
|
148
|
+
def detect_section_title(line)
|
|
149
|
+
return unless line.is_a?(String)
|
|
150
|
+
return if line.strip.empty?
|
|
151
|
+
return if line.start_with?("#")
|
|
152
|
+
return unless line.end_with?(":")
|
|
153
|
+
|
|
154
|
+
line.strip.delete_suffix(":")
|
|
155
|
+
end
|
|
156
|
+
|
|
157
|
+
def format_metric_line(metric)
|
|
158
|
+
value = metric["value"]
|
|
159
|
+
unit = metric["unit"]
|
|
160
|
+
suffix = (unit == "seconds") ? "s" : " #{unit}"
|
|
161
|
+
format(" %s: %s%s", metric["name"], value, suffix)
|
|
162
|
+
end
|
|
163
|
+
|
|
164
|
+
STORAGE_KEY = :polyrun_benchmark_profile_storage
|
|
165
|
+
|
|
166
|
+
def storage
|
|
167
|
+
Thread.current[STORAGE_KEY] ||= {
|
|
168
|
+
"lines" => [],
|
|
169
|
+
"metrics" => [],
|
|
170
|
+
"current_section" => ["default"]
|
|
171
|
+
}
|
|
172
|
+
end
|
|
173
|
+
|
|
174
|
+
def lines_storage
|
|
175
|
+
storage["lines"]
|
|
176
|
+
end
|
|
177
|
+
|
|
178
|
+
def metrics_storage
|
|
179
|
+
storage["metrics"]
|
|
180
|
+
end
|
|
181
|
+
|
|
182
|
+
def current_section_storage
|
|
183
|
+
storage["current_section"]
|
|
184
|
+
end
|
|
185
|
+
|
|
186
|
+
def default_repository_root
|
|
187
|
+
Dir.pwd
|
|
188
|
+
end
|
|
189
|
+
end
|
|
190
|
+
end
|
|
191
|
+
end
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
require "json"
|
|
2
|
+
|
|
3
|
+
require_relative "../export/csv"
|
|
4
|
+
require_relative "../export/markdown"
|
|
5
|
+
|
|
6
|
+
module Polyrun
|
|
7
|
+
module Benchmark
|
|
8
|
+
# Formats benchmark profile JSON for humans and downstream tools.
|
|
9
|
+
module Report
|
|
10
|
+
METRIC_HEADERS = %w[section name value unit].freeze
|
|
11
|
+
LINE_HEADERS = %w[index text].freeze
|
|
12
|
+
|
|
13
|
+
module_function
|
|
14
|
+
|
|
15
|
+
def load(path)
|
|
16
|
+
ext = File.extname(path).downcase
|
|
17
|
+
if ext == ".json"
|
|
18
|
+
JSON.parse(File.read(path))
|
|
19
|
+
else
|
|
20
|
+
{"meta" => {}, "lines" => File.read(path).lines.map(&:chomp), "metrics" => []}
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
def render(data, format: "text")
|
|
25
|
+
case format.to_s.downcase
|
|
26
|
+
when "text", "console", "txt" then format_text(data)
|
|
27
|
+
when "json" then JSON.pretty_generate(data)
|
|
28
|
+
when "csv" then format_csv(data)
|
|
29
|
+
when "markdown", "md" then format_markdown(data)
|
|
30
|
+
else
|
|
31
|
+
raise Polyrun::Error, "report-benchmark: unknown format #{format.inspect} (use text, json, csv, or markdown)"
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def format_text(data)
|
|
36
|
+
meta = data["meta"] || {}
|
|
37
|
+
header = [
|
|
38
|
+
"Benchmark profile",
|
|
39
|
+
"commit: #{meta["commit"]}",
|
|
40
|
+
"recorded_at: #{meta["recorded_at"]}",
|
|
41
|
+
"ruby: #{meta["ruby"]}",
|
|
42
|
+
""
|
|
43
|
+
]
|
|
44
|
+
(header + Array(data["lines"])).join("\n") + "\n"
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def format_csv(data)
|
|
48
|
+
metrics = Array(data["metrics"])
|
|
49
|
+
if metrics.any?
|
|
50
|
+
rows = metrics.map { |metric| [metric["section"], metric["name"], metric["value"], metric["unit"]] }
|
|
51
|
+
return Export::Csv.generate(METRIC_HEADERS, rows)
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
rows = Array(data["lines"]).each_with_index.map { |line, index| [index + 1, line] }
|
|
55
|
+
Export::Csv.generate(LINE_HEADERS, rows)
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
def format_markdown(data)
|
|
59
|
+
meta = data["meta"] || {}
|
|
60
|
+
sections = [
|
|
61
|
+
{
|
|
62
|
+
heading: "Metadata",
|
|
63
|
+
headers: %w[key value],
|
|
64
|
+
rows: meta.sort_by { |key, _| key.to_s }
|
|
65
|
+
}
|
|
66
|
+
]
|
|
67
|
+
|
|
68
|
+
metrics = Array(data["metrics"])
|
|
69
|
+
if metrics.any?
|
|
70
|
+
sections << {
|
|
71
|
+
heading: "Metrics",
|
|
72
|
+
headers: METRIC_HEADERS,
|
|
73
|
+
rows: metrics.map { |metric| [metric["section"], metric["name"], metric["value"], metric["unit"]] }
|
|
74
|
+
}
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
line_rows = Array(data["lines"]).reject(&:empty?).zip
|
|
78
|
+
sections << {
|
|
79
|
+
heading: "Log",
|
|
80
|
+
headers: %w[line],
|
|
81
|
+
rows: line_rows.empty? ? [["(empty)"]] : line_rows
|
|
82
|
+
}
|
|
83
|
+
Export::Markdown.document("Polyrun benchmark profile", sections)
|
|
84
|
+
end
|
|
85
|
+
end
|
|
86
|
+
end
|
|
87
|
+
end
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
require "optparse"
|
|
2
|
+
|
|
3
|
+
require_relative "../benchmark/profile"
|
|
4
|
+
require_relative "../benchmark/report"
|
|
5
|
+
|
|
6
|
+
module Polyrun
|
|
7
|
+
class CLI
|
|
8
|
+
module BenchmarkCommands
|
|
9
|
+
private
|
|
10
|
+
|
|
11
|
+
def cmd_report_benchmark(argv)
|
|
12
|
+
input = nil
|
|
13
|
+
out_file = nil
|
|
14
|
+
report_format = "text"
|
|
15
|
+
OptionParser.new do |opts|
|
|
16
|
+
opts.banner = "usage: polyrun report-benchmark -i FILE [-o PATH] [--format text|json|csv|markdown]"
|
|
17
|
+
opts.on("-i", "--input PATH", "Benchmark profile .json or .log from tmp/benchmarks/") { |v| input = v }
|
|
18
|
+
opts.on("-o", "--output PATH", "Write report to file instead of stdout") { |v| out_file = v }
|
|
19
|
+
opts.on("--format VAL", "text (default), json, csv, or markdown") { |v| report_format = v }
|
|
20
|
+
end.parse!(argv)
|
|
21
|
+
input ||= argv.first
|
|
22
|
+
|
|
23
|
+
unless input && File.file?(input)
|
|
24
|
+
Polyrun::Log.warn "report-benchmark: need -i FILE"
|
|
25
|
+
return 2
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
data = Polyrun::Benchmark::Report.load(File.expand_path(input))
|
|
29
|
+
begin
|
|
30
|
+
text = Polyrun::Benchmark::Report.render(data, format: report_format)
|
|
31
|
+
rescue Polyrun::Error => e
|
|
32
|
+
Polyrun::Log.warn e.message.to_s
|
|
33
|
+
return 2
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
if out_file
|
|
37
|
+
File.write(File.expand_path(out_file), text)
|
|
38
|
+
Polyrun::Log.puts File.expand_path(out_file)
|
|
39
|
+
else
|
|
40
|
+
Polyrun::Log.print text
|
|
41
|
+
end
|
|
42
|
+
0
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def cmd_bench(argv)
|
|
46
|
+
tag = "benchmark"
|
|
47
|
+
paths = %w[spec/performance/benchmark_spec.rb spec/performance/benchmark_merge_spec.rb]
|
|
48
|
+
env = {
|
|
49
|
+
"BENCH_FILES" => ENV.fetch("BENCH_FILES", "110"),
|
|
50
|
+
"BENCH_LINES" => ENV.fetch("BENCH_LINES", "310"),
|
|
51
|
+
"BENCH_FRAGMENTS" => ENV.fetch("BENCH_FRAGMENTS", "8"),
|
|
52
|
+
"BENCH_MERGE_REPS" => ENV.fetch("BENCH_MERGE_REPS", "3"),
|
|
53
|
+
"BENCH_LINE_COUNT_REPS" => ENV.fetch("BENCH_LINE_COUNT_REPS", "5"),
|
|
54
|
+
"BENCH_PEEK_REPS" => ENV.fetch("BENCH_PEEK_REPS", "200")
|
|
55
|
+
}
|
|
56
|
+
formats = ENV["POLYRUN_BENCH_FORMATS"]
|
|
57
|
+
env["POLYRUN_BENCH_FORMATS"] = formats if formats && !formats.to_s.strip.empty?
|
|
58
|
+
bench_argv = paths + ["--tag", tag]
|
|
59
|
+
bench_argv.concat(argv) unless argv.empty?
|
|
60
|
+
Polyrun::Log.warn "polyrun bench: running performance specs (#{paths.join(", ")})" if @verbose
|
|
61
|
+
success = system(env, "bundle", "exec", "rspec", *bench_argv)
|
|
62
|
+
success ? 0 : ($?.exitstatus || 1)
|
|
63
|
+
end
|
|
64
|
+
end
|
|
65
|
+
end
|
|
66
|
+
end
|
|
@@ -11,7 +11,7 @@ module Polyrun
|
|
|
11
11
|
|
|
12
12
|
private
|
|
13
13
|
|
|
14
|
-
def cmd_merge_coverage(argv, _config_path)
|
|
14
|
+
def cmd_merge_coverage(argv, _config_path, print_console_summary: true)
|
|
15
15
|
inputs, output, formats = merge_coverage_parse_argv(argv)
|
|
16
16
|
if inputs.empty?
|
|
17
17
|
Polyrun::Log.warn "merge-coverage: need at least one existing -i FILE (after glob expansion)"
|
|
@@ -36,7 +36,7 @@ module Polyrun
|
|
|
36
36
|
payload = Polyrun::Coverage::Merge.to_simplecov_json(merged, meta: r[:meta], groups: r[:groups])
|
|
37
37
|
out_abs = File.expand_path(output)
|
|
38
38
|
merge_coverage_write_json_payload(out_abs, payload)
|
|
39
|
-
merge_coverage_write_format_outputs(merged, r, out_abs, formats)
|
|
39
|
+
merge_coverage_write_format_outputs(merged, r, out_abs, formats, print_console_summary: print_console_summary)
|
|
40
40
|
|
|
41
41
|
elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - t0
|
|
42
42
|
merge_coverage_log_finish(elapsed, inputs)
|
|
@@ -63,7 +63,7 @@ module Polyrun
|
|
|
63
63
|
formats = Polyrun::Coverage::Reporting::DEFAULT_FORMATS.dup
|
|
64
64
|
|
|
65
65
|
parser = OptionParser.new do |opts|
|
|
66
|
-
opts.banner = "usage: polyrun report-coverage -i FILE [-o DIR] [--basename NAME] [--format json,lcov,cobertura,console,html]"
|
|
66
|
+
opts.banner = "usage: polyrun report-coverage -i FILE [-o DIR] [--basename NAME] [--format json,lcov,cobertura,console,html,csv,markdown]"
|
|
67
67
|
opts.on("-i", "--input PATH", "Merged or raw SimpleCov JSON") { |v| input = v }
|
|
68
68
|
opts.on("-o", "--output DIR", "Output directory") { |v| output_dir = v }
|
|
69
69
|
opts.on("--basename NAME", "File name prefix") { |v| basename = v }
|
|
@@ -122,7 +122,7 @@ module Polyrun
|
|
|
122
122
|
files.each { |f| merge_argv.push("-i", f) }
|
|
123
123
|
merge_argv += ["-o", output, "--format", format_list]
|
|
124
124
|
Polyrun::Debug.time("merge-coverage (parent after workers)") do
|
|
125
|
-
cmd_merge_coverage(merge_argv, config_path)
|
|
125
|
+
cmd_merge_coverage(merge_argv, config_path, print_console_summary: coverage_console_output?)
|
|
126
126
|
end
|
|
127
127
|
end
|
|
128
128
|
|
|
@@ -13,7 +13,7 @@ module Polyrun
|
|
|
13
13
|
output = "coverage/polyrun-merged.json"
|
|
14
14
|
formats = ["json"]
|
|
15
15
|
parser = OptionParser.new do |opts|
|
|
16
|
-
opts.banner = "usage: polyrun merge-coverage -i FILE [-i FILE] [-o PATH] [--format json,lcov,cobertura,console,html]"
|
|
16
|
+
opts.banner = "usage: polyrun merge-coverage -i FILE [-i FILE] [-o PATH] [--format json,lcov,cobertura,console,html,csv,markdown]"
|
|
17
17
|
opts.on("-i", "--input FILE", "Coverage JSON (repeatable; globs ok)") do |f|
|
|
18
18
|
expand_merge_input_pattern(f).each { |x| inputs << x }
|
|
19
19
|
end
|
|
@@ -39,11 +39,13 @@ module Polyrun
|
|
|
39
39
|
end
|
|
40
40
|
end
|
|
41
41
|
|
|
42
|
-
def merge_coverage_write_format_outputs(merged, r, out_abs, formats)
|
|
42
|
+
def merge_coverage_write_format_outputs(merged, r, out_abs, formats, print_console_summary: true)
|
|
43
43
|
write_merge_lcov(merged, out_abs) if formats.include?("lcov")
|
|
44
44
|
write_merge_cobertura(merged, r, out_abs) if formats.include?("cobertura")
|
|
45
|
-
write_merge_console(merged, out_abs) if formats.include?("console")
|
|
45
|
+
write_merge_console(merged, out_abs, print_summary: print_console_summary) if formats.include?("console")
|
|
46
46
|
write_merge_html(merged, out_abs) if formats.include?("html")
|
|
47
|
+
write_merge_csv(merged, out_abs) if formats.include?("csv")
|
|
48
|
+
write_merge_markdown(merged, out_abs) if formats.include?("markdown")
|
|
47
49
|
end
|
|
48
50
|
|
|
49
51
|
def write_merge_lcov(merged, out_abs)
|
|
@@ -67,21 +69,45 @@ module Polyrun
|
|
|
67
69
|
r[:meta]["polyrun_coverage_root"] || r[:meta][:polyrun_coverage_root]
|
|
68
70
|
end
|
|
69
71
|
|
|
70
|
-
def write_merge_console(merged, out_abs)
|
|
72
|
+
def write_merge_console(merged, out_abs, print_summary: true)
|
|
71
73
|
sum_path = out_abs.sub(/\.json\z/, "-summary.txt")
|
|
72
74
|
sum_path = "#{out_abs}-summary.txt" if sum_path == out_abs
|
|
73
75
|
summary = Polyrun::Coverage::Merge.console_summary(merged)
|
|
74
76
|
summary_text = Polyrun::Coverage::Merge.format_console_summary(summary)
|
|
75
77
|
Polyrun::Debug.time("write console summary") { File.write(sum_path, summary_text) }
|
|
78
|
+
return unless print_summary
|
|
79
|
+
|
|
76
80
|
Polyrun::Log.print summary_text
|
|
77
81
|
end
|
|
78
82
|
|
|
83
|
+
def coverage_console_output?
|
|
84
|
+
@verbose ||
|
|
85
|
+
%w[1 true yes].include?(ENV["POLYRUN_COVERAGE_VERBOSE"]&.to_s&.downcase) ||
|
|
86
|
+
Polyrun::Debug.enabled?
|
|
87
|
+
end
|
|
88
|
+
|
|
79
89
|
def write_merge_html(merged, out_abs)
|
|
80
90
|
html_path = out_abs.sub(/\.json\z/, ".html")
|
|
81
91
|
html_path = "#{out_abs}.html" if html_path == out_abs
|
|
82
92
|
Polyrun::Debug.time("write HTML report") { File.write(html_path, Polyrun::Coverage::Merge.emit_html(merged)) }
|
|
83
93
|
end
|
|
84
94
|
|
|
95
|
+
def write_merge_csv(merged, out_abs)
|
|
96
|
+
csv_path = out_abs.sub(/\.json\z/, ".csv")
|
|
97
|
+
csv_path = "#{out_abs}.csv" if csv_path == out_abs
|
|
98
|
+
Polyrun::Debug.time("write CSV report") do
|
|
99
|
+
File.write(csv_path, Polyrun::Coverage::FileStatsReport.emit_csv(merged))
|
|
100
|
+
end
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
def write_merge_markdown(merged, out_abs)
|
|
104
|
+
markdown_path = out_abs.sub(/\.json\z/, ".md")
|
|
105
|
+
markdown_path = "#{out_abs}.md" if markdown_path == out_abs
|
|
106
|
+
Polyrun::Debug.time("write Markdown report") do
|
|
107
|
+
File.write(markdown_path, Polyrun::Coverage::FileStatsReport.emit_markdown(merged))
|
|
108
|
+
end
|
|
109
|
+
end
|
|
110
|
+
|
|
85
111
|
def merge_coverage_log_finish(elapsed, inputs)
|
|
86
112
|
thr = merge_slow_warn_threshold_seconds
|
|
87
113
|
merge_coverage_warn_if_slow(elapsed, thr, inputs) if thr && elapsed > thr
|
|
@@ -36,12 +36,12 @@ module Polyrun
|
|
|
36
36
|
output = File.join("tmp", "polyrun_failures", "merged.jsonl")
|
|
37
37
|
format = "jsonl"
|
|
38
38
|
OptionParser.new do |opts|
|
|
39
|
-
opts.banner = "usage: polyrun merge-failures -i FILE [-i FILE] [-o PATH] [--format jsonl|json]"
|
|
39
|
+
opts.banner = "usage: polyrun merge-failures -i FILE [-i FILE] [-o PATH] [--format jsonl|json|csv|markdown]"
|
|
40
40
|
opts.on("-i", "--input FILE", "JSONL fragment or RSpec JSON (repeatable; globs ok)") do |f|
|
|
41
41
|
expand_merge_input_pattern(f).each { |x| inputs << x }
|
|
42
42
|
end
|
|
43
43
|
opts.on("-o", "--output PATH", String) { |v| output = v }
|
|
44
|
-
opts.on("--format VAL", "jsonl (default) or
|
|
44
|
+
opts.on("--format VAL", "jsonl (default), json, csv, or markdown") { |v| format = v }
|
|
45
45
|
end.parse!(argv)
|
|
46
46
|
inputs.uniq!
|
|
47
47
|
inputs.select! { |p| File.file?(p) }
|
|
@@ -75,6 +75,8 @@ module Polyrun
|
|
|
75
75
|
return "jsonl" if f.empty?
|
|
76
76
|
return "jsonl" if f == "jsonl"
|
|
77
77
|
return "json" if f == "json"
|
|
78
|
+
return "csv" if f == "csv"
|
|
79
|
+
return "markdown" if %w[markdown md].include?(f)
|
|
78
80
|
|
|
79
81
|
Polyrun::Log.warn "polyrun run-shards: unknown merge_failures_format=#{ctx[:merge_failures_format].inspect}; using jsonl"
|
|
80
82
|
"jsonl"
|
|
@@ -84,7 +86,12 @@ module Polyrun
|
|
|
84
86
|
raw = ctx[:merge_failures_output]
|
|
85
87
|
return File.expand_path(raw) if raw && !raw.to_s.strip.empty?
|
|
86
88
|
|
|
87
|
-
ext =
|
|
89
|
+
ext = case fmt
|
|
90
|
+
when "json" then "json"
|
|
91
|
+
when "csv" then "csv"
|
|
92
|
+
when "markdown", "md" then "md"
|
|
93
|
+
else "jsonl"
|
|
94
|
+
end
|
|
88
95
|
File.expand_path(File.join("tmp", "polyrun_failures", "merged.#{ext}"))
|
|
89
96
|
end
|
|
90
97
|
end
|
data/lib/polyrun/cli/help.rb
CHANGED
|
@@ -13,9 +13,11 @@ module Polyrun
|
|
|
13
13
|
-h, --help
|
|
14
14
|
|
|
15
15
|
Trace timing (stderr): DEBUG=1 or POLYRUN_DEBUG=1
|
|
16
|
-
Coverage: POLYRUN_COVERAGE=1 (or config/polyrun_coverage.yml + POLYRUN_QUICK_COVERAGE=1); POLYRUN_COVERAGE_DISABLE=1 skips; POLYRUN_COVERAGE_BRANCHES=1 for branch data in fragments
|
|
16
|
+
Coverage: POLYRUN_COVERAGE=1 (or config/polyrun_coverage.yml + POLYRUN_QUICK_COVERAGE=1); POLYRUN_COVERAGE_DISABLE=1 skips; POLYRUN_COVERAGE_BRANCHES=1 for branch data in fragments; POLYRUN_COVERAGE_VERBOSE=1 for per-worker and merged console summaries
|
|
17
|
+
Benchmark profiles: POLYRUN_BENCH=1 (stdout); JSON sidecar in tmp/benchmarks/; POLYRUN_BENCH_FORMATS=csv,markdown for extra exports; report-benchmark / bench
|
|
18
|
+
Hooks shell output: POLYRUN_HOOKS_VERBOSE=1 or -v / POLYRUN_VERBOSE=1 (or DEBUG); failures always print
|
|
17
19
|
Merge profiling (stderr): POLYRUN_PROFILE_MERGE=1 (or verbose / DEBUG)
|
|
18
|
-
Post-merge formats (run-shards): POLYRUN_MERGE_FORMATS (default: json,lcov,cobertura,console,html)
|
|
20
|
+
Post-merge formats (run-shards): POLYRUN_MERGE_FORMATS (default: json,lcov,cobertura,console,html,csv,markdown)
|
|
19
21
|
Start skips: POLYRUN_SKIP_BUILD_SPEC_PATHS=1, POLYRUN_START_SKIP_PREPARE=1, POLYRUN_START_SKIP_DATABASES=1
|
|
20
22
|
Paths build skip: POLYRUN_SKIP_PATHS_BUILD=1
|
|
21
23
|
Slow merge warning (seconds, default 10; 0 disables): POLYRUN_MERGE_SLOW_WARN_SECONDS
|
|
@@ -25,7 +27,7 @@ module Polyrun
|
|
|
25
27
|
Per-worker idle timeout: --worker-idle-timeout SEC or POLYRUN_WORKER_IDLE_TIMEOUT_SEC after a progress ping (POLYRUN_WORKER_PING_FILE). Enable pings in test setup. Exit 125. Optional periodic pings: POLYRUN_WORKER_PING_THREAD=1 (POLYRUN_WORKER_PING_INTERVAL_SEC).
|
|
26
28
|
Worker output routing (opt-in): POLYRUN_WORKER_OUTPUT_ROUTING=1 or POLYRUN_WORKER_LOG_DIR; per-shard logs under tmp/polyrun/workers (POLYRUN_WORKER_OUTPUT_PREFIX=0 for log-only)
|
|
27
29
|
Example debug (RSpec, opt-in): POLYRUN_EXAMPLE_DEBUG=1; POLYRUN_DEBUG_SQL / POLYRUN_DEBUG_TRACE
|
|
28
|
-
Sharded formatter compat:
|
|
30
|
+
Sharded formatter compat: silences per-worker seed, summary, and pending lines under POLYRUN_SHARD_* (see docs/SETUP_PROFILE.md)
|
|
29
31
|
Orchestration warnings on process stderr: POLYRUN_ORCHESTRATION_STDERR=1
|
|
30
32
|
Spec quality (opt-in): POLYRUN_SPEC_QUALITY=1; run-shards --merge-spec-quality; merge-spec-quality / report-spec-quality
|
|
31
33
|
Partition timing granularity (default file): POLYRUN_TIMING_GRANULARITY=file|example (experimental; see partition.timing_granularity)
|
|
@@ -35,8 +37,8 @@ module Polyrun
|
|
|
35
37
|
version print version
|
|
36
38
|
plan emit partition manifest JSON
|
|
37
39
|
prepare run prepare recipe: default | assets (optional prepare.command overrides bin/rails assets:precompile) | shell (prepare.command required)
|
|
38
|
-
merge-coverage merge SimpleCov JSON fragments (json/lcov/cobertura/console)
|
|
39
|
-
merge-failures merge per-shard failure JSONL fragments or RSpec JSON files (jsonl/json)
|
|
40
|
+
merge-coverage merge SimpleCov JSON fragments (json/lcov/cobertura/console/html/csv/markdown)
|
|
41
|
+
merge-failures merge per-shard failure JSONL fragments or RSpec JSON files (jsonl/json/csv/markdown)
|
|
40
42
|
run-shards fan out N parallel OS processes (POLYRUN_SHARD_*; not Ruby threads); optional --merge-coverage / --merge-failures / --merge-spec-quality
|
|
41
43
|
parallel-rspec run-shards + merge-coverage (defaults to: bundle exec rspec after --)
|
|
42
44
|
start parallel-rspec; auto-runs prepare (shell/assets) and db:setup-* when polyrun.yml configures them; legacy script/build_spec_paths.rb if paths_build absent
|
|
@@ -49,11 +51,13 @@ module Polyrun
|
|
|
49
51
|
quick quick test runner (describe/it, before/after, let, expect…to, assert_*; optional capybara!)
|
|
50
52
|
hook run <phase> run one shell hook from polyrun.yml hooks: (e.g. before_suite); optional --shard/--total
|
|
51
53
|
report-coverage write all coverage formats from one JSON file
|
|
52
|
-
report-junit RSpec JSON or Polyrun testcase JSON → JUnit XML
|
|
53
|
-
report-timing
|
|
54
|
+
report-junit RSpec JSON or Polyrun testcase JSON → JUnit XML, CSV, or Markdown
|
|
55
|
+
report-timing slow-file summary from merged timing JSON (text/csv/markdown)
|
|
56
|
+
report-benchmark benchmark profile export from tmp/benchmarks/profile_*.json
|
|
57
|
+
bench run performance benchmark specs (same as rake bench_performance)
|
|
54
58
|
merge-timing merge polyrun_timing_*.json shards
|
|
55
59
|
merge-spec-quality merge polyrun-spec-quality-fragment-*.jsonl shards
|
|
56
|
-
report-spec-quality spec quality report from merged JSON (
|
|
60
|
+
report-spec-quality spec quality report from merged JSON (text/json/csv/markdown)
|
|
57
61
|
config print effective config by dotted path (loaded YAML plus merged prepare.env, resolved partition shard fields, workers)
|
|
58
62
|
env print shard + database env (see polyrun.yml databases)
|
|
59
63
|
db:setup-template migrate template DB (PostgreSQL)
|
|
@@ -7,7 +7,7 @@ module Polyrun
|
|
|
7
7
|
private
|
|
8
8
|
|
|
9
9
|
def cmd_report_junit(argv)
|
|
10
|
-
inputs, output = report_junit_parse_inputs(argv)
|
|
10
|
+
inputs, output, report_format = report_junit_parse_inputs(argv)
|
|
11
11
|
inputs.uniq!
|
|
12
12
|
if inputs.empty?
|
|
13
13
|
Polyrun::Log.warn "report-junit: need -i FILE (existing path after glob expansion)"
|
|
@@ -17,12 +17,12 @@ module Polyrun
|
|
|
17
17
|
inputs = inputs.map { |p| File.expand_path(p) }
|
|
18
18
|
return 2 unless report_junit_inputs_exist?(inputs)
|
|
19
19
|
|
|
20
|
-
out = report_junit_resolved_output(inputs, output)
|
|
20
|
+
out = report_junit_resolved_output(inputs, output, report_format)
|
|
21
21
|
path =
|
|
22
22
|
if inputs.size == 1
|
|
23
|
-
Polyrun::Reporting::Junit.write_from_json_file(inputs.first, output_path: out)
|
|
23
|
+
Polyrun::Reporting::Junit.write_from_json_file(inputs.first, output_path: out, format: report_format)
|
|
24
24
|
else
|
|
25
|
-
Polyrun::Reporting::Junit.merge_rspec_json_files(inputs, output_path: out)
|
|
25
|
+
Polyrun::Reporting::Junit.merge_rspec_json_files(inputs, output_path: out, format: report_format)
|
|
26
26
|
end
|
|
27
27
|
Polyrun::Log.puts path
|
|
28
28
|
0
|
|
@@ -31,17 +31,19 @@ module Polyrun
|
|
|
31
31
|
def report_junit_parse_inputs(argv)
|
|
32
32
|
inputs = []
|
|
33
33
|
output = nil
|
|
34
|
+
report_format = "xml"
|
|
34
35
|
OptionParser.new do |opts|
|
|
35
|
-
opts.banner = "usage: polyrun report-junit -i FILE [-i FILE]... [-o PATH]"
|
|
36
|
+
opts.banner = "usage: polyrun report-junit -i FILE [-i FILE]... [-o PATH] [--format xml|csv|markdown]"
|
|
36
37
|
opts.on("-i", "--input PATH", "RSpec JSON (repeatable; globs ok; multiple files merge examples)") do |v|
|
|
37
38
|
expand_merge_input_pattern(v).each { |x| inputs << x }
|
|
38
39
|
end
|
|
39
|
-
opts.on("-o", "--output PATH", "Default: <dir of first input>/junit
|
|
40
|
+
opts.on("-o", "--output PATH", "Default: <dir of first input>/junit.<ext>") { |v| output = v }
|
|
41
|
+
opts.on("--format VAL", "xml (default), csv, or markdown") { |v| report_format = v }
|
|
40
42
|
end.parse!(argv)
|
|
41
43
|
if inputs.empty? && argv.first
|
|
42
44
|
expand_merge_input_pattern(argv.first).each { |x| inputs << x }
|
|
43
45
|
end
|
|
44
|
-
[inputs, output]
|
|
46
|
+
[inputs, output, report_format]
|
|
45
47
|
end
|
|
46
48
|
|
|
47
49
|
def report_junit_inputs_exist?(inputs)
|
|
@@ -54,11 +56,16 @@ module Polyrun
|
|
|
54
56
|
true
|
|
55
57
|
end
|
|
56
58
|
|
|
57
|
-
def report_junit_resolved_output(inputs, output)
|
|
59
|
+
def report_junit_resolved_output(inputs, output, format)
|
|
58
60
|
if output
|
|
59
61
|
File.expand_path(output)
|
|
60
62
|
else
|
|
61
|
-
|
|
63
|
+
extension = case format.to_s.downcase
|
|
64
|
+
when "csv" then "csv"
|
|
65
|
+
when "markdown", "md" then "md"
|
|
66
|
+
else "xml"
|
|
67
|
+
end
|
|
68
|
+
File.join(File.dirname(inputs.first), "junit.#{extension}")
|
|
62
69
|
end
|
|
63
70
|
end
|
|
64
71
|
|
|
@@ -66,11 +73,13 @@ module Polyrun
|
|
|
66
73
|
input = nil
|
|
67
74
|
out_file = nil
|
|
68
75
|
top = 30
|
|
76
|
+
report_format = "text"
|
|
69
77
|
OptionParser.new do |opts|
|
|
70
|
-
opts.banner = "usage: polyrun report-timing -i FILE [-o PATH] [--top N]"
|
|
78
|
+
opts.banner = "usage: polyrun report-timing -i FILE [-o PATH] [--top N] [--format text|csv|markdown]"
|
|
71
79
|
opts.on("-i", "--input PATH", "Merged polyrun_timing.json (path => seconds)") { |v| input = v }
|
|
72
80
|
opts.on("-o", "--output PATH", "Write summary to file instead of stdout") { |v| out_file = v }
|
|
73
81
|
opts.on("--top N", Integer) { |v| top = v }
|
|
82
|
+
opts.on("--format VAL", "text (default), csv, or markdown") { |v| report_format = v }
|
|
74
83
|
end.parse!(argv)
|
|
75
84
|
input ||= argv.first
|
|
76
85
|
|
|
@@ -80,7 +89,12 @@ module Polyrun
|
|
|
80
89
|
end
|
|
81
90
|
|
|
82
91
|
merged = JSON.parse(File.read(File.expand_path(input)))
|
|
83
|
-
|
|
92
|
+
begin
|
|
93
|
+
text = Polyrun::Timing::Summary.render(merged, format: report_format, top: top)
|
|
94
|
+
rescue Polyrun::Error => e
|
|
95
|
+
Polyrun::Log.warn e.message.to_s
|
|
96
|
+
return 2
|
|
97
|
+
end
|
|
84
98
|
if out_file
|
|
85
99
|
File.write(File.expand_path(out_file), text)
|
|
86
100
|
Polyrun::Log.puts File.expand_path(out_file)
|
|
@@ -23,7 +23,7 @@ module Polyrun
|
|
|
23
23
|
mt = ctx[:matrix_shard_total]
|
|
24
24
|
|
|
25
25
|
pids = []
|
|
26
|
-
Polyrun::WorkerOutput.prepare_log_dir! if Polyrun::WorkerOutput.routing_enabled?
|
|
26
|
+
Polyrun::WorkerOutput.prepare_log_dir! if Polyrun::WorkerOutput.routing_enabled?
|
|
27
27
|
workers.times do |shard|
|
|
28
28
|
paths = plan.shard(shard)
|
|
29
29
|
if paths.empty?
|
|
@@ -63,7 +63,7 @@ module Polyrun
|
|
|
63
63
|
opts.on("--merge-format LIST", String) { |v| st[:merge_format] = v }
|
|
64
64
|
opts.on("--merge-failures", "After all workers exit, merge failure fragments from tmp/polyrun_failures (requires failure fragments in test setup)") { st[:merge_failures] = true }
|
|
65
65
|
opts.on("--merge-failures-output PATH", String) { |v| st[:merge_failures_output] = v }
|
|
66
|
-
opts.on("--merge-failures-format VAL", "jsonl (default) or
|
|
66
|
+
opts.on("--merge-failures-format VAL", "jsonl (default), json, csv, or markdown") { |v| st[:merge_failures_format] = v }
|
|
67
67
|
opts.on("--merge-spec-quality", "After workers exit, merge spec-quality fragments from coverage (enable spec-quality collection in test setup)") { st[:merge_spec_quality] = true }
|
|
68
68
|
opts.on("--merge-spec-quality-output PATH", String) { |v| st[:merge_spec_quality_output] = v }
|
|
69
69
|
opts.on("--no-report-spec-quality", "Skip printing spec-quality report after merge") { st[:report_spec_quality] = false }
|