polyrun 1.4.2 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +38 -0
- data/README.md +2 -2
- data/docs/SETUP_PROFILE.md +2 -0
- data/lib/polyrun/cli/ci_shard_hooks.rb +12 -4
- data/lib/polyrun/cli/ci_shard_run_command.rb +3 -1
- data/lib/polyrun/cli/help.rb +10 -2
- data/lib/polyrun/cli/helpers.rb +38 -0
- data/lib/polyrun/cli/init_command.rb +8 -1
- data/lib/polyrun/cli/partition_diagnostics.rb +22 -0
- data/lib/polyrun/cli/plan_command.rb +47 -18
- data/lib/polyrun/cli/queue_command.rb +25 -2
- data/lib/polyrun/cli/run_queue_command.rb +145 -0
- data/lib/polyrun/cli/run_shards_command.rb +6 -1
- data/lib/polyrun/cli/run_shards_parallel_children.rb +28 -35
- data/lib/polyrun/cli/run_shards_parallel_wait.rb +267 -0
- data/lib/polyrun/cli/run_shards_plan_boot_phases.rb +81 -3
- data/lib/polyrun/cli/run_shards_plan_options.rb +17 -3
- data/lib/polyrun/cli/run_shards_planning.rb +20 -12
- data/lib/polyrun/cli/run_shards_run.rb +28 -37
- data/lib/polyrun/cli/run_shards_worker_interrupt.rb +75 -0
- data/lib/polyrun/cli/spec_quality_commands.rb +140 -0
- data/lib/polyrun/cli.rb +16 -2
- data/lib/polyrun/coverage/example_diff.rb +122 -0
- data/lib/polyrun/coverage/merge/formatters_html.rb +4 -0
- data/lib/polyrun/data/factory_counts.rb +14 -1
- data/lib/polyrun/database/clone_shards.rb +2 -0
- data/lib/polyrun/database/shard.rb +2 -1
- data/lib/polyrun/hooks.rb +9 -1
- data/lib/polyrun/log.rb +16 -0
- data/lib/polyrun/minitest.rb +43 -0
- data/lib/polyrun/partition/hrw.rb +40 -3
- data/lib/polyrun/partition/paths_build.rb +8 -3
- data/lib/polyrun/partition/plan.rb +88 -19
- data/lib/polyrun/partition/plan_lpt.rb +49 -7
- data/lib/polyrun/partition/plan_sharding.rb +8 -0
- data/lib/polyrun/partition/reports.rb +139 -0
- data/lib/polyrun/partition/timing_diagnostics.rb +139 -0
- data/lib/polyrun/partition/timing_keys.rb +2 -1
- data/lib/polyrun/queue/duration.rb +30 -0
- data/lib/polyrun/queue/file_store.rb +107 -3
- data/lib/polyrun/quick/example_runner.rb +13 -0
- data/lib/polyrun/quick/runner.rb +21 -0
- data/lib/polyrun/rspec.rb +26 -0
- data/lib/polyrun/spec_quality/config.rb +134 -0
- data/lib/polyrun/spec_quality/fragment.rb +39 -0
- data/lib/polyrun/spec_quality/merge.rb +78 -0
- data/lib/polyrun/spec_quality/minitest_hook.rb +42 -0
- data/lib/polyrun/spec_quality/plan_loader.rb +47 -0
- data/lib/polyrun/spec_quality/profile.rb +91 -0
- data/lib/polyrun/spec_quality/report.rb +261 -0
- data/lib/polyrun/spec_quality/rspec_hook.rb +55 -0
- data/lib/polyrun/spec_quality/sql_counter.rb +34 -0
- data/lib/polyrun/spec_quality.rb +205 -0
- data/lib/polyrun/templates/POLYRUN.md +6 -0
- data/lib/polyrun/templates/ci_matrix.polyrun.yml +4 -0
- data/lib/polyrun/templates/polyrun_hooks_spec_quality.rb +12 -0
- data/lib/polyrun/templates/polyrun_spec_quality.yml +20 -0
- data/lib/polyrun/templates/rails_prepare.polyrun.yml +5 -0
- data/lib/polyrun/timing/merge.rb +5 -5
- data/lib/polyrun/timing/stats.rb +76 -0
- data/lib/polyrun/timing/summary.rb +5 -2
- data/lib/polyrun/timing/variance_report.rb +51 -0
- data/lib/polyrun/version.rb +1 -1
- data/lib/polyrun/worker_ping.rb +74 -0
- data/sig/polyrun/minitest.rbs +2 -0
- data/sig/polyrun/rspec.rbs +4 -0
- data/sig/polyrun/worker_ping.rbs +10 -0
- metadata +26 -1
|
@@ -0,0 +1,205 @@
|
|
|
1
|
+
# rubocop:disable Polyrun/FileLength -- per-example recorder API
|
|
2
|
+
require "json"
|
|
3
|
+
require "fileutils"
|
|
4
|
+
|
|
5
|
+
require_relative "coverage/example_diff"
|
|
6
|
+
require_relative "partition/timing_keys"
|
|
7
|
+
require_relative "spec_quality/config"
|
|
8
|
+
require_relative "spec_quality/profile"
|
|
9
|
+
require_relative "spec_quality/fragment"
|
|
10
|
+
require_relative "spec_quality/sql_counter"
|
|
11
|
+
require_relative "spec_quality/merge"
|
|
12
|
+
require_relative "spec_quality/plan_loader"
|
|
13
|
+
require_relative "spec_quality/report"
|
|
14
|
+
require_relative "spec_quality/rspec_hook"
|
|
15
|
+
require_relative "spec_quality/minitest_hook"
|
|
16
|
+
|
|
17
|
+
module Polyrun
|
|
18
|
+
# Per-example spec quality: coverage line deltas, resource use, optional SQL counts.
|
|
19
|
+
# Opt-in with +POLYRUN_SPEC_QUALITY=1+ or +Polyrun::SpecQuality.start!+.
|
|
20
|
+
module SpecQuality
|
|
21
|
+
class << self
|
|
22
|
+
def start!(root: Dir.pwd, config_path: nil, output_path: nil, **overrides)
|
|
23
|
+
return if Config.disabled?
|
|
24
|
+
|
|
25
|
+
@config = Config.load(root: root, config_path: config_path, **overrides)
|
|
26
|
+
@output_path = output_path || Fragment.default_fragment_path
|
|
27
|
+
@pause_depth = 0
|
|
28
|
+
@current = nil
|
|
29
|
+
@rng = Random.new(Process.pid ^ Time.now.to_i)
|
|
30
|
+
|
|
31
|
+
Fragment.truncate_fragment!(@output_path) unless fragment_append_mode?
|
|
32
|
+
SqlCounter.install! if @config["sql_counter"]
|
|
33
|
+
self
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def started?
|
|
37
|
+
instance_variable_defined?(:@config) && @config
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def enabled?
|
|
41
|
+
Config.enabled? && !Config.disabled?
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
attr_reader :config
|
|
45
|
+
|
|
46
|
+
attr_reader :output_path
|
|
47
|
+
|
|
48
|
+
def recording?
|
|
49
|
+
started? && @current && @pause_depth.zero?
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
def start_example!(location:, wall_start: nil)
|
|
53
|
+
return unless started?
|
|
54
|
+
return if paused?
|
|
55
|
+
return if Config.ignored_example?(location, @config["ignore_examples"])
|
|
56
|
+
return unless sample_example?
|
|
57
|
+
|
|
58
|
+
@current = {
|
|
59
|
+
location: normalize_location(location),
|
|
60
|
+
wall_start: wall_start || Process.clock_gettime(Process::CLOCK_MONOTONIC),
|
|
61
|
+
coverage_before: Coverage::ExampleDiff.peek_blob,
|
|
62
|
+
profile_before: Profile.snapshot,
|
|
63
|
+
sql_count: 0,
|
|
64
|
+
sql_fingerprints: Hash.new(0),
|
|
65
|
+
factory_counts: {}
|
|
66
|
+
}
|
|
67
|
+
Polyrun::Data::FactoryCounts.reset_example! if defined?(Polyrun::Data::FactoryCounts)
|
|
68
|
+
nil
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
def finish_example!(location: nil, pending: false)
|
|
72
|
+
return unless started?
|
|
73
|
+
cur = @current
|
|
74
|
+
@current = nil
|
|
75
|
+
return if cur.nil? || paused?
|
|
76
|
+
return if pending
|
|
77
|
+
|
|
78
|
+
loc = location || cur[:location]
|
|
79
|
+
return if loc.nil? || loc.to_s.empty?
|
|
80
|
+
|
|
81
|
+
after_cov = Coverage::ExampleDiff.peek_blob
|
|
82
|
+
delta = Coverage::ExampleDiff.diff(cur[:coverage_before], after_cov)
|
|
83
|
+
delta = Coverage::ExampleDiff.apply_track_under(
|
|
84
|
+
delta,
|
|
85
|
+
root: @config["root"],
|
|
86
|
+
track_under: @config["track_under"],
|
|
87
|
+
ignore_paths: @config["ignore_paths"]
|
|
88
|
+
)
|
|
89
|
+
|
|
90
|
+
profile_after = Profile.snapshot
|
|
91
|
+
profile_delta = Profile.diff(cur[:profile_before], profile_after)
|
|
92
|
+
wall = Process.clock_gettime(Process::CLOCK_MONOTONIC) - cur[:wall_start]
|
|
93
|
+
profile_delta["wall"] = wall
|
|
94
|
+
|
|
95
|
+
factory_counts =
|
|
96
|
+
if defined?(Polyrun::Data::FactoryCounts)
|
|
97
|
+
Polyrun::Data::FactoryCounts.example_counts
|
|
98
|
+
else
|
|
99
|
+
{}
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
row = build_row(cur, loc, delta, profile_delta, factory_counts)
|
|
103
|
+
Fragment.append_row!(@output_path, row)
|
|
104
|
+
row
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
def pause
|
|
108
|
+
@pause_depth += 1
|
|
109
|
+
if block_given?
|
|
110
|
+
begin
|
|
111
|
+
yield
|
|
112
|
+
ensure
|
|
113
|
+
resume
|
|
114
|
+
end
|
|
115
|
+
end
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
def resume
|
|
119
|
+
@pause_depth -= 1 if @pause_depth.positive?
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
def paused?
|
|
123
|
+
@pause_depth.positive?
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
def record_sql!(sql)
|
|
127
|
+
return unless recording?
|
|
128
|
+
|
|
129
|
+
@current[:sql_count] += 1
|
|
130
|
+
fp = normalize_sql(sql)
|
|
131
|
+
@current[:sql_fingerprints][fp] += 1
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
def spec_quality_requested_for_quick?(root = Dir.pwd)
|
|
135
|
+
return false if Config.disabled?
|
|
136
|
+
return true if %w[1 true yes].include?(ENV["POLYRUN_SPEC_QUALITY"]&.to_s&.downcase)
|
|
137
|
+
return true if %w[1 true yes].include?(ENV["POLYRUN_SPEC_QUALITY_FRAGMENTS"]&.to_s&.downcase)
|
|
138
|
+
|
|
139
|
+
path = File.join(File.expand_path(root), Config::DEFAULT_CONFIG_RELATIVE)
|
|
140
|
+
return false unless File.file?(path)
|
|
141
|
+
|
|
142
|
+
%w[1 true yes].include?(ENV["POLYRUN_QUICK_SPEC_QUALITY"]&.to_s&.downcase)
|
|
143
|
+
end
|
|
144
|
+
|
|
145
|
+
private
|
|
146
|
+
|
|
147
|
+
def fragment_append_mode?
|
|
148
|
+
truthy?(ENV["POLYRUN_SPEC_QUALITY_FRAGMENT_APPEND"])
|
|
149
|
+
end
|
|
150
|
+
|
|
151
|
+
def sample_example?
|
|
152
|
+
rate = @config["sample"].to_f
|
|
153
|
+
return true if rate >= 1.0
|
|
154
|
+
return false if rate <= 0.0
|
|
155
|
+
|
|
156
|
+
@rng.rand < rate
|
|
157
|
+
end
|
|
158
|
+
|
|
159
|
+
def normalize_location(location)
|
|
160
|
+
s = location.to_s.strip
|
|
161
|
+
return s if s.empty?
|
|
162
|
+
|
|
163
|
+
root = @config["root"]
|
|
164
|
+
if (m = s.match(/\A(.+):(\d+)\z/)) && m[2].match?(/\A\d+\z/)
|
|
165
|
+
fp = Polyrun::Partition::TimingKeys.canonical_file_path(File.expand_path(m[1], root))
|
|
166
|
+
return "#{fp}:#{m[2]}"
|
|
167
|
+
end
|
|
168
|
+
|
|
169
|
+
Polyrun::Partition::TimingKeys.canonical_file_path(File.expand_path(s, root))
|
|
170
|
+
end
|
|
171
|
+
|
|
172
|
+
def build_row(cur, location, delta, profile_delta, factory_counts)
|
|
173
|
+
profile = Profile.slice_profile(profile_delta, @config["profile"])
|
|
174
|
+
repeated_sql = cur[:sql_fingerprints].select { |_sql, n| n >= min_query_count }.transform_keys(&:to_s)
|
|
175
|
+
|
|
176
|
+
{
|
|
177
|
+
"example" => location.to_s,
|
|
178
|
+
"unique_lines" => delta[:unique_lines],
|
|
179
|
+
"line_churn" => delta[:line_churn],
|
|
180
|
+
"max_line_churn" => delta[:max_line_churn],
|
|
181
|
+
"lines" => delta[:lines],
|
|
182
|
+
"profile" => profile,
|
|
183
|
+
"sql_count" => cur[:sql_count],
|
|
184
|
+
"repeated_sql" => repeated_sql,
|
|
185
|
+
"factory_counts" => factory_counts.transform_keys(&:to_s),
|
|
186
|
+
"polyrun_shard_index" => ENV["POLYRUN_SHARD_INDEX"],
|
|
187
|
+
"polyrun_shard_total" => ENV["POLYRUN_SHARD_TOTAL"]
|
|
188
|
+
}.compact
|
|
189
|
+
end
|
|
190
|
+
|
|
191
|
+
def min_query_count
|
|
192
|
+
@config["min_query_count"].to_i
|
|
193
|
+
end
|
|
194
|
+
|
|
195
|
+
def normalize_sql(sql)
|
|
196
|
+
sql.to_s.gsub(/\s+/, " ").strip[0, 500]
|
|
197
|
+
end
|
|
198
|
+
|
|
199
|
+
def truthy?(value)
|
|
200
|
+
Config.send(:truthy?, value)
|
|
201
|
+
end
|
|
202
|
+
end
|
|
203
|
+
end
|
|
204
|
+
end
|
|
205
|
+
# rubocop:enable Polyrun/FileLength
|
|
@@ -40,6 +40,12 @@ Do not combine Model A and Model B in one workflow without a documented reason (
|
|
|
40
40
|
- `spec/spec_helper.rb`: `require "polyrun"` and collector or Rails helper as appropriate.
|
|
41
41
|
- Fragments: `coverage/polyrun-fragment-<shard>.json` → `polyrun merge-coverage` → `polyrun report-coverage` / `report-junit` for CI.
|
|
42
42
|
|
|
43
|
+
## Spec quality (optional)
|
|
44
|
+
|
|
45
|
+
- `POLYRUN_SPEC_QUALITY=1` and `Polyrun::RSpec.install_spec_quality!` in `spec_helper` (requires coverage enabled for line deltas).
|
|
46
|
+
- `config/polyrun_spec_quality.yml` — thresholds and CI gates (`polyrun init --profile spec-quality`).
|
|
47
|
+
- Parallel: `run-shards --merge-spec-quality` → `coverage/polyrun-spec-quality.json` → `report-spec-quality`.
|
|
48
|
+
|
|
43
49
|
## Further reading
|
|
44
50
|
|
|
45
51
|
- Polyrun README: partition, prepare, databases, merge-coverage, `Polyrun::Env::Ci`
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
# Optional Polyrun hooks — copy to config/polyrun_hooks.rb and reference from polyrun.yml:
|
|
2
|
+
# hooks:
|
|
3
|
+
# ruby_file: config/polyrun_hooks.rb
|
|
4
|
+
#
|
|
5
|
+
# Worker-phase hooks run in each parallel test child (around bundle exec rspec).
|
|
6
|
+
|
|
7
|
+
before(:each) do |env|
|
|
8
|
+
next unless env["POLYRUN_HOOK_ORCHESTRATOR"] == "0"
|
|
9
|
+
next unless defined?(Polyrun::SpecQuality) && Polyrun::SpecQuality.enabled?
|
|
10
|
+
|
|
11
|
+
Polyrun::SpecQuality::RspecHook.ensure_started!
|
|
12
|
+
end
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
# Opt-in per-example spec quality (coverage deltas, resource use). See docs/SPEC_QUALITY.md.
|
|
2
|
+
track_under:
|
|
3
|
+
- lib
|
|
4
|
+
- app
|
|
5
|
+
min_line_churn: 50
|
|
6
|
+
min_query_count: 20
|
|
7
|
+
hot_line_example_overlap: 10
|
|
8
|
+
strict: false
|
|
9
|
+
sample: 1.0
|
|
10
|
+
ignore_examples: []
|
|
11
|
+
ignore_paths: []
|
|
12
|
+
ignore_query_patterns: []
|
|
13
|
+
profile:
|
|
14
|
+
- cpu
|
|
15
|
+
- mem
|
|
16
|
+
sql_counter: false
|
|
17
|
+
# CI gates (used with POLYRUN_SPEC_QUALITY_STRICT=1 or strict: true):
|
|
18
|
+
# minimum_unique_lines_per_example: 1
|
|
19
|
+
# max_zero_hit_examples: 0
|
|
20
|
+
# max_hot_line_overlap: 100
|
|
@@ -17,6 +17,11 @@ prepare:
|
|
|
17
17
|
rails_root: .
|
|
18
18
|
command: bundle exec ruby bin/test_prepare.rb
|
|
19
19
|
|
|
20
|
+
# Optional reporting (see docs/SPEC_QUALITY.md):
|
|
21
|
+
# reporting:
|
|
22
|
+
# merge_spec_quality: true
|
|
23
|
+
# merge_spec_quality_output: coverage/polyrun-spec-quality.json
|
|
24
|
+
|
|
20
25
|
# Uncomment and edit when using Postgres template + per-shard DB names from polyrun env:
|
|
21
26
|
# databases:
|
|
22
27
|
# template_db: my_app_template
|
data/lib/polyrun/timing/merge.rb
CHANGED
|
@@ -1,12 +1,11 @@
|
|
|
1
1
|
require "json"
|
|
2
2
|
|
|
3
3
|
require_relative "../debug"
|
|
4
|
+
require_relative "stats"
|
|
5
|
+
require_relative "variance_report"
|
|
4
6
|
|
|
5
7
|
module Polyrun
|
|
6
8
|
module Timing
|
|
7
|
-
# Merges per-shard timing JSON files (spec2 §2.4): path => wall seconds (float), or (experimental)
|
|
8
|
-
# +absolute_path:line+ => seconds for per-example timing.
|
|
9
|
-
# Disjoint suites: values merged by taking the maximum per path when duplicates appear.
|
|
10
9
|
module Merge
|
|
11
10
|
module_function
|
|
12
11
|
|
|
@@ -18,8 +17,8 @@ module Polyrun
|
|
|
18
17
|
|
|
19
18
|
data.each do |file, sec|
|
|
20
19
|
f = file.to_s
|
|
21
|
-
|
|
22
|
-
merged[f] = merged.key?(f) ?
|
|
20
|
+
entry = sec
|
|
21
|
+
merged[f] = merged.key?(f) ? Stats.merge_entries(merged[f], entry) : Stats.normalize_entry(entry)
|
|
23
22
|
end
|
|
24
23
|
end
|
|
25
24
|
merged
|
|
@@ -29,6 +28,7 @@ module Polyrun
|
|
|
29
28
|
Polyrun::Debug.log_kv(merge_timing: "merge_and_write", input_count: paths.size, output_path: output_path)
|
|
30
29
|
merged = Polyrun::Debug.time("Timing::Merge.merge_files") { merge_files(paths) }
|
|
31
30
|
Polyrun::Debug.time("Timing::Merge.write JSON") { File.write(output_path, JSON.pretty_generate(merged)) }
|
|
31
|
+
Timing::VarianceReport.emit_warnings!(merged)
|
|
32
32
|
merged
|
|
33
33
|
end
|
|
34
34
|
end
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
module Polyrun
|
|
2
|
+
module Timing
|
|
3
|
+
# Normalizes scalar or object timing entries for merge and binpack weight lookup.
|
|
4
|
+
module Stats
|
|
5
|
+
STAT_KEYS = %w[last_seconds min max mean p95 runs failures timeouts].freeze
|
|
6
|
+
|
|
7
|
+
module_function
|
|
8
|
+
|
|
9
|
+
def normalize_entry(value)
|
|
10
|
+
case value
|
|
11
|
+
when Hash
|
|
12
|
+
normalize_hash(value)
|
|
13
|
+
else
|
|
14
|
+
sec = value.to_f
|
|
15
|
+
{
|
|
16
|
+
"last_seconds" => sec,
|
|
17
|
+
"min" => sec,
|
|
18
|
+
"max" => sec,
|
|
19
|
+
"mean" => sec,
|
|
20
|
+
"p95" => sec,
|
|
21
|
+
"runs" => 1,
|
|
22
|
+
"failures" => 0,
|
|
23
|
+
"timeouts" => 0
|
|
24
|
+
}
|
|
25
|
+
end
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
# rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity -- timing hash key coercion
|
|
29
|
+
def normalize_hash(h)
|
|
30
|
+
out = {}
|
|
31
|
+
sec = h["last_seconds"] || h[:last_seconds] || h["seconds"] || h[:seconds]
|
|
32
|
+
sec = sec.to_f if sec
|
|
33
|
+
mean = (h["mean"] || h[:mean] || sec)&.to_f
|
|
34
|
+
out["last_seconds"] = (sec || mean || 0.0).to_f
|
|
35
|
+
out["min"] = (h["min"] || h[:min] || out["last_seconds"]).to_f
|
|
36
|
+
out["max"] = (h["max"] || h[:max] || out["last_seconds"]).to_f
|
|
37
|
+
out["mean"] = (mean || out["last_seconds"]).to_f
|
|
38
|
+
out["p95"] = (h["p95"] || h[:p95] || out["max"]).to_f
|
|
39
|
+
out["runs"] = Integer(h["runs"] || h[:runs] || 1)
|
|
40
|
+
out["failures"] = Integer(h["failures"] || h[:failures] || 0)
|
|
41
|
+
out["timeouts"] = Integer(h["timeouts"] || h[:timeouts] || 0)
|
|
42
|
+
out
|
|
43
|
+
end
|
|
44
|
+
# rubocop:enable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
|
|
45
|
+
|
|
46
|
+
def binpack_weight(entry)
|
|
47
|
+
h = normalize_entry(entry)
|
|
48
|
+
h["last_seconds"].positive? ? h["last_seconds"] : h["mean"]
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
# rubocop:disable Metrics/AbcSize -- weighted mean merge
|
|
52
|
+
def merge_entries(a, b)
|
|
53
|
+
ha = normalize_entry(a)
|
|
54
|
+
hb = normalize_entry(b)
|
|
55
|
+
runs = ha["runs"] + hb["runs"]
|
|
56
|
+
mean =
|
|
57
|
+
if runs.positive?
|
|
58
|
+
((ha["mean"] * ha["runs"]) + (hb["mean"] * hb["runs"])) / runs.to_f
|
|
59
|
+
else
|
|
60
|
+
0.0
|
|
61
|
+
end
|
|
62
|
+
{
|
|
63
|
+
"last_seconds" => [ha["last_seconds"], hb["last_seconds"]].max,
|
|
64
|
+
"min" => [ha["min"], hb["min"]].min,
|
|
65
|
+
"max" => [ha["max"], hb["max"]].max,
|
|
66
|
+
"mean" => mean,
|
|
67
|
+
"p95" => [ha["p95"], hb["p95"]].max,
|
|
68
|
+
"runs" => runs,
|
|
69
|
+
"failures" => ha["failures"] + hb["failures"],
|
|
70
|
+
"timeouts" => ha["timeouts"] + hb["timeouts"]
|
|
71
|
+
}
|
|
72
|
+
end
|
|
73
|
+
# rubocop:enable Metrics/AbcSize
|
|
74
|
+
end
|
|
75
|
+
end
|
|
76
|
+
end
|
|
@@ -1,14 +1,17 @@
|
|
|
1
|
+
require_relative "stats"
|
|
2
|
+
|
|
1
3
|
module Polyrun
|
|
2
4
|
module Timing
|
|
3
5
|
# Human-readable slow-file list from merged timing JSON (per-file cost).
|
|
4
6
|
module Summary
|
|
5
7
|
module_function
|
|
6
8
|
|
|
7
|
-
# +merged+ is path (String) => seconds (Float), as produced by +Timing::Merge.merge_files+.
|
|
9
|
+
# +merged+ is path (String) => seconds (Float) or stats Hash, as produced by +Timing::Merge.merge_files+.
|
|
8
10
|
def format_slow_files(merged, top: 30, title: "Polyrun slowest files (by wall time, seconds)")
|
|
9
11
|
return "#{title}\n (no data)\n" if merged.nil? || merged.empty?
|
|
10
12
|
|
|
11
|
-
pairs = merged.
|
|
13
|
+
pairs = merged.map { |path, sec| [path, Stats.binpack_weight(sec)] }
|
|
14
|
+
.sort_by { |(_, sec)| -sec.to_f }.first(Integer(top))
|
|
12
15
|
lines = [title, ""]
|
|
13
16
|
pairs.each_with_index do |(path, sec), i|
|
|
14
17
|
lines << format(" %2d. %s %.4f", i + 1, path, sec.to_f)
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
module Polyrun
|
|
2
|
+
module Timing
|
|
3
|
+
# Flags high-variance, flaky, and regression timing entries.
|
|
4
|
+
module VarianceReport
|
|
5
|
+
module_function
|
|
6
|
+
|
|
7
|
+
# rubocop:disable Metrics/AbcSize -- variance flag scan
|
|
8
|
+
def analyze(merged_stats)
|
|
9
|
+
flags = []
|
|
10
|
+
merged_stats.each do |path, entry|
|
|
11
|
+
h = Stats.normalize_entry(entry)
|
|
12
|
+
next if h["runs"] < 2
|
|
13
|
+
|
|
14
|
+
median = h["mean"]
|
|
15
|
+
if median.positive? && (h["p95"] / median) > 2.0
|
|
16
|
+
flags << {path: path, kind: "high_variance", detail: "p95/mean=#{format("%.2f", h["p95"] / median)}"}
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
if h["runs"] >= 3 && (h["failures"].to_f / h["runs"]) > 0.3
|
|
20
|
+
flags << {path: path, kind: "often_failed", detail: "failures=#{h["failures"]}/#{h["runs"]}"}
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
if h["timeouts"].to_i >= 2
|
|
24
|
+
flags << {path: path, kind: "timeout_cluster", detail: "timeouts=#{h["timeouts"]}"}
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
if h["mean"].positive? && h["last_seconds"] > (2.0 * h["mean"])
|
|
28
|
+
flags << {path: path, kind: "runtime_regression", detail: "last=#{h["last_seconds"]} mean=#{h["mean"]}"}
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
flags
|
|
32
|
+
end
|
|
33
|
+
# rubocop:enable Metrics/AbcSize
|
|
34
|
+
|
|
35
|
+
def emit_warnings!(merged_stats)
|
|
36
|
+
analyze(merged_stats).each do |f|
|
|
37
|
+
Polyrun::Log.warn "polyrun timing #{f[:kind]}: #{f[:path]} (#{f[:detail]})"
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
def format_report(merged_stats)
|
|
42
|
+
lines = ["Polyrun timing variance report", ""]
|
|
43
|
+
analyze(merged_stats).each do |f|
|
|
44
|
+
lines << " [#{f[:kind]}] #{f[:path]} — #{f[:detail]}"
|
|
45
|
+
end
|
|
46
|
+
lines << " (none)" if lines.size == 2
|
|
47
|
+
lines.join("\n") + "\n"
|
|
48
|
+
end
|
|
49
|
+
end
|
|
50
|
+
end
|
|
51
|
+
end
|
data/lib/polyrun/version.rb
CHANGED
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
module Polyrun
|
|
2
|
+
# Writes a monotonic timestamp to +POLYRUN_WORKER_PING_FILE+ when the test process advances
|
|
3
|
+
# (typically once per example). When +location:+ is passed (path:line of the example), the file
|
|
4
|
+
# is two lines: timestamp, then that string. Parents use +--worker-idle-timeout+ to detect a worker with no
|
|
5
|
+
# progress *inside* a single example—unlike a background thread, +ping!+ does not run while Ruby
|
|
6
|
+
# is busy on the main thread, so a tight CPU loop or stuck native code leaves the timestamp stale.
|
|
7
|
+
#
|
|
8
|
+
# Prefer framework installs (call from helpers *after* loading the runner):
|
|
9
|
+
#
|
|
10
|
+
# require "polyrun/rspec"
|
|
11
|
+
# Polyrun::RSpec.install_worker_ping!
|
|
12
|
+
#
|
|
13
|
+
# require "polyrun/minitest"
|
|
14
|
+
# Polyrun::Minitest.install_worker_ping!
|
|
15
|
+
#
|
|
16
|
+
# Polyrun Quick runs +ping!+ automatically when requiring the Quick stack.
|
|
17
|
+
#
|
|
18
|
+
# Optional interval thread (+POLYRUN_WORKER_PING_THREAD=1+, +POLYRUN_WORKER_PING_INTERVAL_SEC+): call
|
|
19
|
+
# {#ensure_interval_ping_thread!} once at worker startup if you rely on periodic pings without per-example {#ping!};
|
|
20
|
+
# installers call this so the env toggle works out of the box.
|
|
21
|
+
module WorkerPing
|
|
22
|
+
class << self
|
|
23
|
+
def ping!(location: nil)
|
|
24
|
+
path = ping_file_path
|
|
25
|
+
return if path.empty?
|
|
26
|
+
|
|
27
|
+
t = Process.clock_gettime(Process::CLOCK_MONOTONIC).to_s
|
|
28
|
+
loc = location.to_s.strip
|
|
29
|
+
payload = loc.empty? ? t : "#{t}\n#{loc}"
|
|
30
|
+
File.binwrite(path, payload)
|
|
31
|
+
rescue SystemCallError
|
|
32
|
+
# best-effort
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def ping_file_path
|
|
36
|
+
ENV["POLYRUN_WORKER_PING_FILE"].to_s.strip
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
# Starts a periodic +ping!+ thread when +POLYRUN_WORKER_PING_THREAD+ is truthy and +POLYRUN_WORKER_PING_FILE+ is set.
|
|
40
|
+
# Prefer per-example {#ping!}; safe to call more than once (idempotent).
|
|
41
|
+
# rubocop:disable ThreadSafety/ClassInstanceVariable -- idempotent once-per-process latch
|
|
42
|
+
def ensure_interval_ping_thread!
|
|
43
|
+
thread_flag = ENV["POLYRUN_WORKER_PING_THREAD"]
|
|
44
|
+
return unless %w[1 true yes].include?(thread_flag&.downcase)
|
|
45
|
+
|
|
46
|
+
path = ping_file_path
|
|
47
|
+
return if path.empty?
|
|
48
|
+
|
|
49
|
+
@interval_ping_mx ||= Mutex.new
|
|
50
|
+
@interval_ping_mx.synchronize do
|
|
51
|
+
return if @interval_ping_started
|
|
52
|
+
|
|
53
|
+
raw = ENV["POLYRUN_WORKER_PING_INTERVAL_SEC"].to_s.strip
|
|
54
|
+
interval = Float(raw.empty? ? "15" : raw, exception: false) || 15.0
|
|
55
|
+
interval = 1.0 if interval < 1.0
|
|
56
|
+
|
|
57
|
+
ping!
|
|
58
|
+
# rubocop:disable ThreadSafety/NewThread -- optional periodic ping alongside per-example ping!
|
|
59
|
+
Thread.new do
|
|
60
|
+
loop do
|
|
61
|
+
sleep(interval)
|
|
62
|
+
ping!
|
|
63
|
+
rescue SystemCallError, Interrupt
|
|
64
|
+
break
|
|
65
|
+
end
|
|
66
|
+
end
|
|
67
|
+
# rubocop:enable ThreadSafety/NewThread
|
|
68
|
+
@interval_ping_started = true
|
|
69
|
+
end
|
|
70
|
+
end
|
|
71
|
+
# rubocop:enable ThreadSafety/ClassInstanceVariable
|
|
72
|
+
end
|
|
73
|
+
end
|
|
74
|
+
end
|
data/sig/polyrun/minitest.rbs
CHANGED
data/sig/polyrun/rspec.rbs
CHANGED
|
@@ -3,5 +3,9 @@ module Polyrun
|
|
|
3
3
|
def self.install_parallel_provisioning!: (untyped rspec_config) -> void
|
|
4
4
|
|
|
5
5
|
def self.install_example_timing!: (?output_path: String? ) -> void
|
|
6
|
+
|
|
7
|
+
def self.install_failure_fragments!: (?only_if: untyped?) -> void
|
|
8
|
+
|
|
9
|
+
def self.install_worker_ping!: () -> void
|
|
6
10
|
end
|
|
7
11
|
end
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
module Polyrun
|
|
2
|
+
module WorkerPing
|
|
3
|
+
def self.ping!: (?location: String?) -> void
|
|
4
|
+
|
|
5
|
+
def self.ping_file_path: () -> String
|
|
6
|
+
|
|
7
|
+
# Idempotent — installers call once; POLYRUN_WORKER_PING_THREAD gates the periodic thread.
|
|
8
|
+
def self.ensure_interval_ping_thread!: () -> void
|
|
9
|
+
end
|
|
10
|
+
end
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: polyrun
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 1.
|
|
4
|
+
version: 2.1.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Andrei Makarov
|
|
@@ -180,18 +180,23 @@ files:
|
|
|
180
180
|
- lib/polyrun/cli/helpers.rb
|
|
181
181
|
- lib/polyrun/cli/hooks_command.rb
|
|
182
182
|
- lib/polyrun/cli/init_command.rb
|
|
183
|
+
- lib/polyrun/cli/partition_diagnostics.rb
|
|
183
184
|
- lib/polyrun/cli/plan_command.rb
|
|
184
185
|
- lib/polyrun/cli/prepare_command.rb
|
|
185
186
|
- lib/polyrun/cli/prepare_recipe.rb
|
|
186
187
|
- lib/polyrun/cli/queue_command.rb
|
|
187
188
|
- lib/polyrun/cli/quick_command.rb
|
|
188
189
|
- lib/polyrun/cli/report_commands.rb
|
|
190
|
+
- lib/polyrun/cli/run_queue_command.rb
|
|
189
191
|
- lib/polyrun/cli/run_shards_command.rb
|
|
190
192
|
- lib/polyrun/cli/run_shards_parallel_children.rb
|
|
193
|
+
- lib/polyrun/cli/run_shards_parallel_wait.rb
|
|
191
194
|
- lib/polyrun/cli/run_shards_plan_boot_phases.rb
|
|
192
195
|
- lib/polyrun/cli/run_shards_plan_options.rb
|
|
193
196
|
- lib/polyrun/cli/run_shards_planning.rb
|
|
194
197
|
- lib/polyrun/cli/run_shards_run.rb
|
|
198
|
+
- lib/polyrun/cli/run_shards_worker_interrupt.rb
|
|
199
|
+
- lib/polyrun/cli/spec_quality_commands.rb
|
|
195
200
|
- lib/polyrun/cli/start_bootstrap.rb
|
|
196
201
|
- lib/polyrun/cli/timing_command.rb
|
|
197
202
|
- lib/polyrun/config.rb
|
|
@@ -202,6 +207,7 @@ files:
|
|
|
202
207
|
- lib/polyrun/coverage/collector.rb
|
|
203
208
|
- lib/polyrun/coverage/collector_finish.rb
|
|
204
209
|
- lib/polyrun/coverage/collector_fragment_meta.rb
|
|
210
|
+
- lib/polyrun/coverage/example_diff.rb
|
|
205
211
|
- lib/polyrun/coverage/filter.rb
|
|
206
212
|
- lib/polyrun/coverage/formatter.rb
|
|
207
213
|
- lib/polyrun/coverage/merge.rb
|
|
@@ -250,11 +256,14 @@ files:
|
|
|
250
256
|
- lib/polyrun/partition/plan.rb
|
|
251
257
|
- lib/polyrun/partition/plan_lpt.rb
|
|
252
258
|
- lib/polyrun/partition/plan_sharding.rb
|
|
259
|
+
- lib/polyrun/partition/reports.rb
|
|
253
260
|
- lib/polyrun/partition/stable_shuffle.rb
|
|
261
|
+
- lib/polyrun/partition/timing_diagnostics.rb
|
|
254
262
|
- lib/polyrun/partition/timing_keys.rb
|
|
255
263
|
- lib/polyrun/prepare/artifacts.rb
|
|
256
264
|
- lib/polyrun/prepare/assets.rb
|
|
257
265
|
- lib/polyrun/process_stdio.rb
|
|
266
|
+
- lib/polyrun/queue/duration.rb
|
|
258
267
|
- lib/polyrun/queue/file_store.rb
|
|
259
268
|
- lib/polyrun/queue/file_store_pending.rb
|
|
260
269
|
- lib/polyrun/quick.rb
|
|
@@ -272,14 +281,29 @@ files:
|
|
|
272
281
|
- lib/polyrun/reporting/rspec_failure_fragment_formatter.rb
|
|
273
282
|
- lib/polyrun/reporting/rspec_junit.rb
|
|
274
283
|
- lib/polyrun/rspec.rb
|
|
284
|
+
- lib/polyrun/spec_quality.rb
|
|
285
|
+
- lib/polyrun/spec_quality/config.rb
|
|
286
|
+
- lib/polyrun/spec_quality/fragment.rb
|
|
287
|
+
- lib/polyrun/spec_quality/merge.rb
|
|
288
|
+
- lib/polyrun/spec_quality/minitest_hook.rb
|
|
289
|
+
- lib/polyrun/spec_quality/plan_loader.rb
|
|
290
|
+
- lib/polyrun/spec_quality/profile.rb
|
|
291
|
+
- lib/polyrun/spec_quality/report.rb
|
|
292
|
+
- lib/polyrun/spec_quality/rspec_hook.rb
|
|
293
|
+
- lib/polyrun/spec_quality/sql_counter.rb
|
|
275
294
|
- lib/polyrun/templates/POLYRUN.md
|
|
276
295
|
- lib/polyrun/templates/ci_matrix.polyrun.yml
|
|
277
296
|
- lib/polyrun/templates/minimal_gem.polyrun.yml
|
|
297
|
+
- lib/polyrun/templates/polyrun_hooks_spec_quality.rb
|
|
298
|
+
- lib/polyrun/templates/polyrun_spec_quality.yml
|
|
278
299
|
- lib/polyrun/templates/rails_prepare.polyrun.yml
|
|
279
300
|
- lib/polyrun/timing/merge.rb
|
|
280
301
|
- lib/polyrun/timing/rspec_example_formatter.rb
|
|
302
|
+
- lib/polyrun/timing/stats.rb
|
|
281
303
|
- lib/polyrun/timing/summary.rb
|
|
304
|
+
- lib/polyrun/timing/variance_report.rb
|
|
282
305
|
- lib/polyrun/version.rb
|
|
306
|
+
- lib/polyrun/worker_ping.rb
|
|
283
307
|
- polyrun.gemspec
|
|
284
308
|
- sig/polyrun.rbs
|
|
285
309
|
- sig/polyrun/cli.rbs
|
|
@@ -289,6 +313,7 @@ files:
|
|
|
289
313
|
- sig/polyrun/minitest.rbs
|
|
290
314
|
- sig/polyrun/quick.rbs
|
|
291
315
|
- sig/polyrun/rspec.rbs
|
|
316
|
+
- sig/polyrun/worker_ping.rbs
|
|
292
317
|
homepage: https://github.com/amkisko/polyrun.rb
|
|
293
318
|
licenses:
|
|
294
319
|
- MIT
|