polyrun 1.5.0 → 2.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +34 -0
- data/README.md +2 -2
- data/docs/SETUP_PROFILE.md +2 -0
- data/lib/polyrun/cli/coverage_commands.rb +1 -1
- data/lib/polyrun/cli/failure_commands.rb +1 -1
- data/lib/polyrun/cli/help.rb +20 -17
- data/lib/polyrun/cli/helpers.rb +16 -0
- data/lib/polyrun/cli/init_command.rb +8 -1
- data/lib/polyrun/cli/partition_diagnostics.rb +22 -0
- data/lib/polyrun/cli/plan_command.rb +47 -18
- data/lib/polyrun/cli/queue_command.rb +25 -2
- data/lib/polyrun/cli/run_queue_command.rb +145 -0
- data/lib/polyrun/cli/run_shards_command.rb +6 -1
- data/lib/polyrun/cli/run_shards_parallel_children.rb +2 -1
- data/lib/polyrun/cli/run_shards_parallel_wait.rb +5 -1
- data/lib/polyrun/cli/run_shards_plan_boot_phases.rb +47 -2
- data/lib/polyrun/cli/run_shards_plan_options.rb +14 -4
- data/lib/polyrun/cli/run_shards_planning.rb +20 -12
- data/lib/polyrun/cli/run_shards_run.rb +22 -5
- data/lib/polyrun/cli/spec_quality_commands.rb +140 -0
- data/lib/polyrun/cli.rb +16 -2
- data/lib/polyrun/coverage/example_diff.rb +122 -0
- data/lib/polyrun/coverage/merge/formatters_html.rb +5 -5
- data/lib/polyrun/data/factory_counts.rb +14 -1
- data/lib/polyrun/database/clone_shards.rb +2 -0
- data/lib/polyrun/database/shard.rb +2 -1
- data/lib/polyrun/minitest.rb +9 -0
- data/lib/polyrun/partition/hrw.rb +40 -3
- data/lib/polyrun/partition/paths_build.rb +8 -3
- data/lib/polyrun/partition/plan.rb +88 -19
- data/lib/polyrun/partition/plan_lpt.rb +49 -7
- data/lib/polyrun/partition/plan_sharding.rb +8 -0
- data/lib/polyrun/partition/reports.rb +139 -0
- data/lib/polyrun/partition/timing_diagnostics.rb +139 -0
- data/lib/polyrun/partition/timing_keys.rb +2 -1
- data/lib/polyrun/queue/duration.rb +30 -0
- data/lib/polyrun/queue/file_store.rb +114 -3
- data/lib/polyrun/quick/example_runner.rb +2 -0
- data/lib/polyrun/quick/runner.rb +21 -0
- data/lib/polyrun/rspec.rb +10 -0
- data/lib/polyrun/spec_quality/config.rb +134 -0
- data/lib/polyrun/spec_quality/fragment.rb +39 -0
- data/lib/polyrun/spec_quality/merge.rb +78 -0
- data/lib/polyrun/spec_quality/minitest_hook.rb +42 -0
- data/lib/polyrun/spec_quality/plan_loader.rb +47 -0
- data/lib/polyrun/spec_quality/profile.rb +91 -0
- data/lib/polyrun/spec_quality/report.rb +261 -0
- data/lib/polyrun/spec_quality/rspec_hook.rb +55 -0
- data/lib/polyrun/spec_quality/sql_counter.rb +34 -0
- data/lib/polyrun/spec_quality.rb +205 -0
- data/lib/polyrun/templates/POLYRUN.md +6 -0
- data/lib/polyrun/templates/ci_matrix.polyrun.yml +4 -0
- data/lib/polyrun/templates/polyrun_hooks_spec_quality.rb +12 -0
- data/lib/polyrun/templates/polyrun_spec_quality.yml +20 -0
- data/lib/polyrun/templates/rails_prepare.polyrun.yml +5 -0
- data/lib/polyrun/timing/merge.rb +5 -5
- data/lib/polyrun/timing/rspec_example_formatter.rb +14 -7
- data/lib/polyrun/timing/stats.rb +76 -0
- data/lib/polyrun/timing/summary.rb +5 -2
- data/lib/polyrun/timing/variance_report.rb +51 -0
- data/lib/polyrun/version.rb +1 -1
- metadata +22 -1
|
@@ -46,7 +46,8 @@ module Polyrun
|
|
|
46
46
|
shard: shard,
|
|
47
47
|
matrix_index: mx,
|
|
48
48
|
matrix_total: mt,
|
|
49
|
-
failure_fragments: ctx[:merge_failures]
|
|
49
|
+
failure_fragments: ctx[:merge_failures],
|
|
50
|
+
spec_quality_fragments: ctx[:merge_spec_quality]
|
|
50
51
|
)
|
|
51
52
|
child_env = child_env.merge("POLYRUN_HOOK_ORCHESTRATOR" => "0")
|
|
52
53
|
child_env = hook_cfg.merge_worker_ruby_env(child_env)
|
|
@@ -2,6 +2,10 @@
|
|
|
2
2
|
module Polyrun
|
|
3
3
|
class CLI
|
|
4
4
|
# Wait, wall/idle timeout, and +after_shard+ hooks for parallel workers (+run-shards+ / +ci-shard-*+).
|
|
5
|
+
#
|
|
6
|
+
# Per-worker wait states: running → normal_exit | wall_timeout (124) | idle_timeout (125).
|
|
7
|
+
# When wall or idle caps are set, the multiplex loop polls every live PID each tick so timeouts
|
|
8
|
+
# apply fairly across shards (not only the worker currently in Process.wait).
|
|
5
9
|
module RunShardsParallelWait
|
|
6
10
|
WORKER_TIMEOUT_EXIT_STATUS = 124
|
|
7
11
|
WORKER_IDLE_TIMEOUT_EXIT_STATUS = 125
|
|
@@ -215,7 +219,7 @@ module Polyrun
|
|
|
215
219
|
ping_suffix = (loc && !loc.to_s.strip.empty?) ? "; last ping #{loc.to_s.strip}" : ""
|
|
216
220
|
Polyrun::Log.orchestration_warn "polyrun run-shards: WORKER IDLE TIMEOUT after #{idle_sec}s since last per-example progress ping — shard #{h[:shard]} pid #{h[:pid]}#{ping_suffix}."
|
|
217
221
|
Polyrun::Log.warn "polyrun run-shards: idle shard file sample: #{sample}#{suffix}"
|
|
218
|
-
Polyrun::Log.warn "polyrun run-shards:
|
|
222
|
+
Polyrun::Log.warn "polyrun run-shards: enable per-example worker progress pings in your test setup so idle timeouts reflect real work; exit #{WORKER_IDLE_TIMEOUT_EXIT_STATUS}."
|
|
219
223
|
end
|
|
220
224
|
|
|
221
225
|
def run_shards_wait_or_force_stop_status(pid)
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
# rubocop:disable Polyrun/FileLength -- run-shards boot phases A/B
|
|
1
2
|
require "shellwords"
|
|
2
3
|
|
|
3
4
|
module Polyrun
|
|
@@ -27,17 +28,35 @@ module Polyrun
|
|
|
27
28
|
[:ok, o, cmd]
|
|
28
29
|
end
|
|
29
30
|
|
|
31
|
+
# rubocop:disable Metrics/AbcSize -- items + costs + plan emit for run-shards
|
|
30
32
|
def run_shards_plan_phase_b(o, cmd, cfg, pc, run_t0, config_path)
|
|
31
33
|
items, paths_source, err = run_shards_resolve_items(o[:paths_file], pc)
|
|
32
34
|
return [err, nil] if err
|
|
33
35
|
|
|
34
|
-
costs, strategy, err = run_shards_resolve_costs(
|
|
36
|
+
costs, strategy, err = run_shards_resolve_costs(
|
|
37
|
+
o[:timing_path],
|
|
38
|
+
o[:strategy],
|
|
39
|
+
o[:timing_granularity],
|
|
40
|
+
strategy_explicit: o[:strategy_explicit]
|
|
41
|
+
)
|
|
35
42
|
return [err, nil] if err
|
|
36
43
|
|
|
37
44
|
run_shards_plan_ready_log(o, cfg, strategy, cmd, paths_source, items.size)
|
|
38
45
|
|
|
39
46
|
constraints = load_partition_constraints(pc, o[:constraints_path])
|
|
40
|
-
|
|
47
|
+
stable = load_stable_assignment(pc)
|
|
48
|
+
plan = run_shards_make_plan(
|
|
49
|
+
items, o[:workers], strategy, o[:seed], costs, constraints, o[:timing_granularity], stable,
|
|
50
|
+
shard_weights: pc["shard_weights"] || pc[:shard_weights]
|
|
51
|
+
)
|
|
52
|
+
|
|
53
|
+
partition_emit_diagnostics!(
|
|
54
|
+
plan: plan,
|
|
55
|
+
items: items,
|
|
56
|
+
costs: costs,
|
|
57
|
+
timing_path: o[:timing_path],
|
|
58
|
+
granularity: o[:timing_granularity]
|
|
59
|
+
)
|
|
41
60
|
|
|
42
61
|
run_shards_debug_shard_sizes(plan, o[:workers])
|
|
43
62
|
Polyrun::Log.warn "polyrun run-shards: #{items.size} paths → #{o[:workers]} workers (#{strategy})" if @verbose
|
|
@@ -47,6 +66,7 @@ module Polyrun
|
|
|
47
66
|
|
|
48
67
|
[nil, run_shards_plan_context_hash(o, cmd, cfg, plan, run_t0, parallel, config_path)]
|
|
49
68
|
end
|
|
69
|
+
# rubocop:enable Metrics/AbcSize
|
|
50
70
|
|
|
51
71
|
def run_shards_plan_boot(argv, config_path)
|
|
52
72
|
run_t0 = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
@@ -69,6 +89,7 @@ module Polyrun
|
|
|
69
89
|
strategy: strategy,
|
|
70
90
|
merge_coverage: o[:merge_coverage],
|
|
71
91
|
merge_failures: run_shards_merge_failures_flag(o, cfg),
|
|
92
|
+
merge_spec_quality: run_shards_merge_spec_quality_flag(o, cfg),
|
|
72
93
|
command: cmd,
|
|
73
94
|
timing_path: o[:timing_path],
|
|
74
95
|
paths_source: paths_source,
|
|
@@ -76,6 +97,26 @@ module Polyrun
|
|
|
76
97
|
)
|
|
77
98
|
end
|
|
78
99
|
|
|
100
|
+
def run_shards_merge_spec_quality_flag(o, cfg)
|
|
101
|
+
return true if o[:merge_spec_quality]
|
|
102
|
+
return true if %w[1 true yes].include?(ENV["POLYRUN_MERGE_SPEC_QUALITY"].to_s.downcase)
|
|
103
|
+
|
|
104
|
+
rep = cfg.reporting
|
|
105
|
+
v = rep["merge_spec_quality"] || rep[:merge_spec_quality]
|
|
106
|
+
v == true || %w[1 true yes].include?(v.to_s.downcase)
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
def run_shards_merge_spec_quality_output_opt(o, cfg)
|
|
110
|
+
x = o[:merge_spec_quality_output]
|
|
111
|
+
return x if x && !x.to_s.strip.empty?
|
|
112
|
+
|
|
113
|
+
x = ENV["POLYRUN_MERGED_SPEC_QUALITY_OUT"]
|
|
114
|
+
return x if x && !x.to_s.strip.empty?
|
|
115
|
+
|
|
116
|
+
rep = cfg.reporting
|
|
117
|
+
rep["merge_spec_quality_output"] || rep[:merge_spec_quality_output]
|
|
118
|
+
end
|
|
119
|
+
|
|
79
120
|
def run_shards_merge_failures_flag(o, cfg)
|
|
80
121
|
return true if o[:merge_failures]
|
|
81
122
|
return true if %w[1 true yes].include?(ENV["POLYRUN_MERGE_FAILURES"].to_s.downcase)
|
|
@@ -121,6 +162,9 @@ module Polyrun
|
|
|
121
162
|
merge_failures: run_shards_merge_failures_flag(o, cfg),
|
|
122
163
|
merge_failures_output: run_shards_merge_failures_output_opt(o, cfg),
|
|
123
164
|
merge_failures_format: run_shards_merge_failures_format_opt(o, cfg),
|
|
165
|
+
merge_spec_quality: run_shards_merge_spec_quality_flag(o, cfg),
|
|
166
|
+
merge_spec_quality_output: run_shards_merge_spec_quality_output_opt(o, cfg),
|
|
167
|
+
report_spec_quality: o[:report_spec_quality] != false,
|
|
124
168
|
config_path: config_path,
|
|
125
169
|
worker_timeout_sec: run_shards_resolved_worker_timeout_sec(o),
|
|
126
170
|
worker_idle_timeout_sec: run_shards_resolved_worker_idle_timeout_sec(o)
|
|
@@ -157,3 +201,4 @@ module Polyrun
|
|
|
157
201
|
end
|
|
158
202
|
end
|
|
159
203
|
end
|
|
204
|
+
# rubocop:enable Polyrun/FileLength
|
|
@@ -18,6 +18,7 @@ module Polyrun
|
|
|
18
18
|
workers: env_int("POLYRUN_WORKERS", Polyrun::Config::DEFAULT_PARALLEL_WORKERS),
|
|
19
19
|
paths_file: nil,
|
|
20
20
|
strategy: (pc["strategy"] || pc[:strategy] || "round_robin").to_s,
|
|
21
|
+
strategy_explicit: !!(pc["strategy"] || pc[:strategy]),
|
|
21
22
|
seed: pc["seed"] || pc[:seed],
|
|
22
23
|
timing_path: nil,
|
|
23
24
|
constraints_path: nil,
|
|
@@ -28,6 +29,9 @@ module Polyrun
|
|
|
28
29
|
merge_failures: false,
|
|
29
30
|
merge_failures_output: nil,
|
|
30
31
|
merge_failures_format: nil,
|
|
32
|
+
merge_spec_quality: false,
|
|
33
|
+
merge_spec_quality_output: nil,
|
|
34
|
+
report_spec_quality: true,
|
|
31
35
|
worker_timeout_sec: nil,
|
|
32
36
|
worker_idle_timeout_sec: nil
|
|
33
37
|
}
|
|
@@ -41,11 +45,14 @@ module Polyrun
|
|
|
41
45
|
|
|
42
46
|
# rubocop:disable Metrics/AbcSize -- one argv block for run-shards
|
|
43
47
|
def run_shards_plan_options_register!(opts, st)
|
|
44
|
-
opts.banner = "usage: polyrun run-shards [--workers N] [--worker-timeout SEC] [--worker-idle-timeout SEC] [--strategy NAME] [--paths-file P] [--timing P] [--timing-granularity VAL] [--constraints P] [--seed S] [--merge-coverage] [--merge-output P] [--merge-format LIST] [--merge-failures] [--merge-failures-output P] [--merge-failures-format jsonl|json] [--] <command> [args...]"
|
|
48
|
+
opts.banner = "usage: polyrun run-shards [--workers N] [--worker-timeout SEC] [--worker-idle-timeout SEC] [--strategy NAME] [--paths-file P] [--timing P] [--timing-granularity VAL] [--constraints P] [--seed S] [--merge-coverage] [--merge-output P] [--merge-format LIST] [--merge-failures] [--merge-failures-output P] [--merge-failures-format jsonl|json] [--merge-spec-quality] [--merge-spec-quality-output P] [--no-report-spec-quality] [--] <command> [args...]"
|
|
45
49
|
opts.on("--workers N", Integer) { |v| st[:workers] = v }
|
|
46
50
|
opts.on("--worker-timeout SEC", Float, "Max seconds per worker since spawn (also POLYRUN_WORKER_TIMEOUT_SEC); kills stuck workers (exit 124)") { |v| st[:worker_timeout_sec] = v }
|
|
47
|
-
opts.on("--worker-idle-timeout SEC", Float, "Max seconds since last
|
|
48
|
-
opts.on("--strategy NAME", String)
|
|
51
|
+
opts.on("--worker-idle-timeout SEC", Float, "Max seconds since last worker progress ping (POLYRUN_WORKER_PING_FILE); enable pings in test setup; exit 125") { |v| st[:worker_idle_timeout_sec] = v }
|
|
52
|
+
opts.on("--strategy NAME", String) do |v|
|
|
53
|
+
st[:strategy] = v
|
|
54
|
+
st[:strategy_explicit] = true
|
|
55
|
+
end
|
|
49
56
|
opts.on("--seed VAL") { |v| st[:seed] = v }
|
|
50
57
|
opts.on("--paths-file PATH", String) { |v| st[:paths_file] = v }
|
|
51
58
|
opts.on("--constraints PATH", String) { |v| st[:constraints_path] = v }
|
|
@@ -54,9 +61,12 @@ module Polyrun
|
|
|
54
61
|
opts.on("--merge-coverage", "After success, merge coverage/polyrun-fragment-*.json (Polyrun coverage must be enabled)") { st[:merge_coverage] = true }
|
|
55
62
|
opts.on("--merge-output PATH", String) { |v| st[:merge_output] = v }
|
|
56
63
|
opts.on("--merge-format LIST", String) { |v| st[:merge_format] = v }
|
|
57
|
-
opts.on("--merge-failures", "After all workers exit, merge tmp/polyrun_failures
|
|
64
|
+
opts.on("--merge-failures", "After all workers exit, merge failure fragments from tmp/polyrun_failures (requires failure fragments in test setup)") { st[:merge_failures] = true }
|
|
58
65
|
opts.on("--merge-failures-output PATH", String) { |v| st[:merge_failures_output] = v }
|
|
59
66
|
opts.on("--merge-failures-format VAL", "jsonl (default) or json") { |v| st[:merge_failures_format] = v }
|
|
67
|
+
opts.on("--merge-spec-quality", "After workers exit, merge spec-quality fragments from coverage (enable spec-quality collection in test setup)") { st[:merge_spec_quality] = true }
|
|
68
|
+
opts.on("--merge-spec-quality-output PATH", String) { |v| st[:merge_spec_quality_output] = v }
|
|
69
|
+
opts.on("--no-report-spec-quality", "Skip printing spec-quality report after merge") { st[:report_spec_quality] = false }
|
|
60
70
|
end
|
|
61
71
|
# rubocop:enable Metrics/AbcSize
|
|
62
72
|
end
|
|
@@ -23,13 +23,10 @@ module Polyrun
|
|
|
23
23
|
run_shards_plan_phase_b(o, cmd, cfg, pc, run_t0, config_path)
|
|
24
24
|
end
|
|
25
25
|
|
|
26
|
-
def run_shards_default_timing_path(pc, timing_path,
|
|
26
|
+
def run_shards_default_timing_path(pc, timing_path, _strategy = nil)
|
|
27
27
|
return timing_path if timing_path
|
|
28
28
|
|
|
29
|
-
|
|
30
|
-
return tf if tf && (Polyrun::Partition::Plan.cost_strategy?(strategy) || Polyrun::Partition::Plan.hrw_strategy?(strategy))
|
|
31
|
-
|
|
32
|
-
nil
|
|
29
|
+
pc["timing_file"] || pc[:timing_file]
|
|
33
30
|
end
|
|
34
31
|
|
|
35
32
|
def run_shards_validate_workers!(o)
|
|
@@ -70,8 +67,13 @@ module Polyrun
|
|
|
70
67
|
[items, paths_source, nil]
|
|
71
68
|
end
|
|
72
69
|
|
|
73
|
-
def run_shards_resolve_costs(timing_path, strategy, timing_granularity)
|
|
70
|
+
def run_shards_resolve_costs(timing_path, strategy, timing_granularity, strategy_explicit: false)
|
|
71
|
+
strategy = strategy.to_s
|
|
74
72
|
if timing_path
|
|
73
|
+
if strategy_explicit && strategy == "round_robin"
|
|
74
|
+
return [nil, strategy, nil]
|
|
75
|
+
end
|
|
76
|
+
|
|
75
77
|
costs = Polyrun::Partition::Plan.load_timing_costs(
|
|
76
78
|
File.expand_path(timing_path.to_s, Dir.pwd),
|
|
77
79
|
granularity: timing_granularity
|
|
@@ -80,12 +82,16 @@ module Polyrun
|
|
|
80
82
|
Polyrun::Log.warn "polyrun run-shards: timing file missing or empty: #{timing_path}"
|
|
81
83
|
return [nil, nil, 2]
|
|
82
84
|
end
|
|
83
|
-
|
|
85
|
+
if Polyrun::Partition::Plan.timing_load_strategy?(strategy)
|
|
86
|
+
return [costs, strategy, nil]
|
|
87
|
+
end
|
|
88
|
+
unless strategy_explicit
|
|
84
89
|
Polyrun::Log.warn "polyrun run-shards: using cost_binpack (timing data present)" if @verbose
|
|
85
|
-
|
|
90
|
+
return [costs, "cost_binpack", nil]
|
|
86
91
|
end
|
|
87
|
-
|
|
88
|
-
|
|
92
|
+
|
|
93
|
+
[nil, strategy, nil]
|
|
94
|
+
elsif Polyrun::Partition::Plan.cost_strategy?(strategy) || Polyrun::Partition::Plan.lazy_robin_strategy?(strategy)
|
|
89
95
|
Polyrun::Log.warn "polyrun run-shards: --timing or partition.timing_file required for strategy #{strategy}"
|
|
90
96
|
[nil, nil, 2]
|
|
91
97
|
else
|
|
@@ -93,7 +99,7 @@ module Polyrun
|
|
|
93
99
|
end
|
|
94
100
|
end
|
|
95
101
|
|
|
96
|
-
def run_shards_make_plan(items, workers, strategy, seed, costs, constraints, timing_granularity)
|
|
102
|
+
def run_shards_make_plan(items, workers, strategy, seed, costs, constraints, timing_granularity, stable_assignment = nil, shard_weights: nil)
|
|
97
103
|
Polyrun::Debug.time("Partition::Plan.new (partition #{items.size} paths → #{workers} shards)") do
|
|
98
104
|
Polyrun::Partition::Plan.new(
|
|
99
105
|
items: items,
|
|
@@ -103,7 +109,9 @@ module Polyrun
|
|
|
103
109
|
costs: costs,
|
|
104
110
|
constraints: constraints,
|
|
105
111
|
root: Dir.pwd,
|
|
106
|
-
timing_granularity: timing_granularity
|
|
112
|
+
timing_granularity: timing_granularity,
|
|
113
|
+
stable_assignment: stable_assignment,
|
|
114
|
+
shard_weights: shard_weights
|
|
107
115
|
)
|
|
108
116
|
end
|
|
109
117
|
end
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
require "shellwords"
|
|
2
2
|
require "rbconfig"
|
|
3
|
+
require "json"
|
|
3
4
|
|
|
4
5
|
require_relative "run_shards_planning"
|
|
5
6
|
require_relative "run_shards_worker_interrupt"
|
|
@@ -22,13 +23,15 @@ module Polyrun
|
|
|
22
23
|
run_shards_workers_and_merge(ctx)
|
|
23
24
|
end
|
|
24
25
|
|
|
25
|
-
# rubocop:disable Metrics/AbcSize -- orchestration: hooks, merge, worker failures
|
|
26
|
+
# rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity -- orchestration: hooks, merge, worker failures
|
|
26
27
|
def run_shards_workers_and_merge(ctx)
|
|
27
28
|
hook_cfg = Polyrun::Hooks.from_config(ctx[:cfg])
|
|
28
29
|
suite_started = false
|
|
29
30
|
exit_code = 1
|
|
30
31
|
merged_failures_path = nil
|
|
31
32
|
merge_failures_errored = false
|
|
33
|
+
merge_spec_quality_errored = false
|
|
34
|
+
spec_quality_strict_failed = false
|
|
32
35
|
|
|
33
36
|
begin
|
|
34
37
|
env_suite = ENV.to_h.merge(
|
|
@@ -64,6 +67,20 @@ module Polyrun
|
|
|
64
67
|
Polyrun::Log.warn "polyrun run-shards: finished #{pids.size} worker(s)" + (failed.any? ? " (some failed)" : " (exit 0)")
|
|
65
68
|
end
|
|
66
69
|
|
|
70
|
+
if ctx[:merge_spec_quality]
|
|
71
|
+
begin
|
|
72
|
+
merged_sq_path = merge_spec_quality_after_shards(ctx)
|
|
73
|
+
if merged_sq_path && ctx[:report_spec_quality]
|
|
74
|
+
cfg = load_spec_quality_config(nil)
|
|
75
|
+
merged = JSON.parse(File.read(merged_sq_path))
|
|
76
|
+
spec_quality_strict_failed = cfg["strict"] && Polyrun::SpecQuality::Report.gate_violations(merged, cfg).any?
|
|
77
|
+
end
|
|
78
|
+
rescue Polyrun::Error, JSON::ParserError => e
|
|
79
|
+
Polyrun::Log.warn e.message.to_s
|
|
80
|
+
merge_spec_quality_errored = true
|
|
81
|
+
end
|
|
82
|
+
end
|
|
83
|
+
|
|
67
84
|
if ctx[:merge_failures]
|
|
68
85
|
begin
|
|
69
86
|
merged_failures_path = merge_failures_after_shards(ctx)
|
|
@@ -79,13 +96,13 @@ module Polyrun
|
|
|
79
96
|
merge_failures: ctx[:merge_failures]
|
|
80
97
|
)
|
|
81
98
|
exit_code = 1
|
|
82
|
-
exit_code = 1 if wait_hook_err != 0
|
|
83
99
|
return exit_code
|
|
84
100
|
end
|
|
85
101
|
|
|
86
102
|
exit_code = run_shards_merge_or_hint_coverage(ctx)
|
|
87
103
|
exit_code = 1 if wait_hook_err != 0 && exit_code == 0
|
|
88
|
-
exit_code = 1 if
|
|
104
|
+
exit_code = 1 if merge_spec_quality_errored && exit_code == 0
|
|
105
|
+
exit_code = 1 if spec_quality_strict_failed && exit_code == 0
|
|
89
106
|
exit_code
|
|
90
107
|
ensure
|
|
91
108
|
if suite_started
|
|
@@ -103,7 +120,7 @@ module Polyrun
|
|
|
103
120
|
end
|
|
104
121
|
end
|
|
105
122
|
end
|
|
106
|
-
# rubocop:enable Metrics/AbcSize
|
|
123
|
+
# rubocop:enable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
|
|
107
124
|
|
|
108
125
|
def run_shards_warn_interleaved(parallel, pid_count)
|
|
109
126
|
return unless parallel && pid_count > 1
|
|
@@ -145,7 +162,7 @@ module Polyrun
|
|
|
145
162
|
Polyrun::Log.warn "polyrun run-shards: shard #{s} re-run (same spec list, no interleave): #{rerun}"
|
|
146
163
|
end
|
|
147
164
|
unless merge_failures
|
|
148
|
-
Polyrun::Log.warn "polyrun run-shards: one
|
|
165
|
+
Polyrun::Log.warn "polyrun run-shards: for one combined failure report, add --merge-failures (requires failure fragments enabled in your test setup)"
|
|
149
166
|
end
|
|
150
167
|
end
|
|
151
168
|
end
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
require "json"
|
|
2
|
+
require "optparse"
|
|
3
|
+
|
|
4
|
+
require_relative "../spec_quality"
|
|
5
|
+
|
|
6
|
+
module Polyrun
|
|
7
|
+
class CLI
|
|
8
|
+
module SpecQualityCommands
|
|
9
|
+
private
|
|
10
|
+
|
|
11
|
+
def cmd_merge_spec_quality(argv)
|
|
12
|
+
inputs = []
|
|
13
|
+
output = "coverage/polyrun-spec-quality.json"
|
|
14
|
+
parser = OptionParser.new do |opts|
|
|
15
|
+
opts.banner = "usage: polyrun merge-spec-quality [-i FILE]... [-o OUT] [FILE...]"
|
|
16
|
+
opts.on("-i", "--input FILE", "Spec quality JSONL fragment (repeatable)") { |f| inputs << f }
|
|
17
|
+
opts.on("-o", "--output PATH", String) { |v| output = v }
|
|
18
|
+
end
|
|
19
|
+
parser.parse!(argv)
|
|
20
|
+
inputs.concat(argv) if inputs.empty?
|
|
21
|
+
|
|
22
|
+
if inputs.empty?
|
|
23
|
+
inputs = Dir.glob(Polyrun::SpecQuality::Fragment.glob_pattern).sort
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
if inputs.empty?
|
|
27
|
+
Polyrun::Log.warn "merge-spec-quality: need -i FILE or coverage/polyrun-spec-quality-fragment-*.jsonl"
|
|
28
|
+
return 2
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
out_abs = File.expand_path(output)
|
|
32
|
+
Polyrun::SpecQuality::Merge.merge_and_write(inputs.map { |p| File.expand_path(p) }, out_abs)
|
|
33
|
+
Polyrun::Log.puts out_abs
|
|
34
|
+
0
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
# rubocop:disable Metrics/AbcSize -- report argv + gate output
|
|
38
|
+
def cmd_report_spec_quality(argv)
|
|
39
|
+
input = nil
|
|
40
|
+
out_file = nil
|
|
41
|
+
top = 30
|
|
42
|
+
profile = nil
|
|
43
|
+
config_path = nil
|
|
44
|
+
strict = false
|
|
45
|
+
json_out = false
|
|
46
|
+
plan_paths = []
|
|
47
|
+
|
|
48
|
+
OptionParser.new do |opts|
|
|
49
|
+
opts.banner = "usage: polyrun report-spec-quality -i FILE [-o PATH] [--top N] [--profile LIST] [--strict] [--json]"
|
|
50
|
+
opts.on("-i", "--input PATH", "Merged polyrun-spec-quality.json") { |v| input = v }
|
|
51
|
+
opts.on("-o", "--output PATH", "Write report to file instead of stdout") { |v| out_file = v }
|
|
52
|
+
opts.on("--top N", Integer) { |v| top = v }
|
|
53
|
+
opts.on("--profile LIST", "cpu,mem,io,wall (comma-separated)") { |v| profile = v }
|
|
54
|
+
opts.on("-c", "--config PATH", "polyrun_spec_quality.yml path") { |v| config_path = v }
|
|
55
|
+
opts.on("--plan PATH", "Partition plan JSON (repeatable; polyrun plan output per shard)") { |v| plan_paths << v }
|
|
56
|
+
opts.on("--strict", "Exit 1 when gate thresholds fail") { strict = true }
|
|
57
|
+
opts.on("--json", "Write analysis JSON instead of text report") { json_out = true }
|
|
58
|
+
end.parse!(argv)
|
|
59
|
+
input ||= argv.first
|
|
60
|
+
|
|
61
|
+
unless input && File.file?(input)
|
|
62
|
+
Polyrun::Log.warn "report-spec-quality: need -i FILE"
|
|
63
|
+
return 2
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
merged = JSON.parse(File.read(File.expand_path(input)))
|
|
67
|
+
cfg = load_spec_quality_config(config_path)
|
|
68
|
+
strict = true if cfg["strict"] || strict
|
|
69
|
+
plan_shards = Polyrun::SpecQuality::PlanLoader.load_shards(plan_paths)
|
|
70
|
+
|
|
71
|
+
text = if json_out
|
|
72
|
+
JSON.pretty_generate(Polyrun::SpecQuality::Report.analyze(merged, cfg, plan_shards: plan_shards))
|
|
73
|
+
else
|
|
74
|
+
Polyrun::SpecQuality::Report.format_report(
|
|
75
|
+
merged, cfg: cfg, top: top, profile: profile, plan_shards: plan_shards
|
|
76
|
+
)
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
if out_file
|
|
80
|
+
File.write(File.expand_path(out_file), text)
|
|
81
|
+
Polyrun::Log.puts File.expand_path(out_file)
|
|
82
|
+
else
|
|
83
|
+
Polyrun::Log.print text
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
violations = Polyrun::SpecQuality::Report.gate_violations(merged, cfg)
|
|
87
|
+
if strict && violations.any?
|
|
88
|
+
violations.each { |v| Polyrun::Log.warn "polyrun spec-quality gate: #{v}" }
|
|
89
|
+
return 1
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
0
|
|
93
|
+
end
|
|
94
|
+
# rubocop:enable Metrics/AbcSize
|
|
95
|
+
|
|
96
|
+
def merge_spec_quality_after_shards(ctx)
|
|
97
|
+
files = merge_spec_quality_fragment_files
|
|
98
|
+
if files.empty?
|
|
99
|
+
Polyrun::Log.warn "polyrun run-shards: --merge-spec-quality: no spec-quality fragments found under coverage (enable spec-quality collection in your test setup)"
|
|
100
|
+
return nil
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
out = ctx[:merge_spec_quality_output] || "coverage/polyrun-spec-quality.json"
|
|
104
|
+
Polyrun::Log.warn "polyrun run-shards: merging #{files.size} spec-quality fragment(s) → #{out}"
|
|
105
|
+
Polyrun::SpecQuality::Merge.merge_and_write(files, File.expand_path(out))
|
|
106
|
+
report_spec_quality_after_merge(out, ctx)
|
|
107
|
+
File.expand_path(out)
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
def report_spec_quality_after_merge(merged_path, ctx)
|
|
111
|
+
return unless ctx[:report_spec_quality]
|
|
112
|
+
|
|
113
|
+
cfg = load_spec_quality_config(ctx[:config_path])
|
|
114
|
+
merged = JSON.parse(File.read(File.expand_path(merged_path)))
|
|
115
|
+
text = Polyrun::SpecQuality::Report.format_report(merged, cfg: cfg)
|
|
116
|
+
Polyrun::Log.print text
|
|
117
|
+
|
|
118
|
+
violations = Polyrun::SpecQuality::Report.gate_violations(merged, cfg)
|
|
119
|
+
return if violations.empty?
|
|
120
|
+
|
|
121
|
+
violations.each { |v| Polyrun::Log.warn "polyrun spec-quality gate: #{v}" }
|
|
122
|
+
end
|
|
123
|
+
|
|
124
|
+
def merge_spec_quality_fragment_files
|
|
125
|
+
Dir.glob(Polyrun::SpecQuality::Fragment.glob_pattern).sort
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
def load_spec_quality_config(config_path)
|
|
129
|
+
root = Dir.pwd
|
|
130
|
+
path = config_path
|
|
131
|
+
if path && !path.to_s.empty?
|
|
132
|
+
path = File.expand_path(path, root)
|
|
133
|
+
end
|
|
134
|
+
Polyrun::SpecQuality::Config.load(root: root, config_path: path)
|
|
135
|
+
rescue
|
|
136
|
+
Polyrun::SpecQuality::Config::DEFAULTS.dup
|
|
137
|
+
end
|
|
138
|
+
end
|
|
139
|
+
end
|
|
140
|
+
end
|
data/lib/polyrun/cli.rb
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
|
+
# rubocop:disable Polyrun/FileLength -- CLI dispatch + require wiring
|
|
1
2
|
require "optparse"
|
|
2
3
|
|
|
3
4
|
require_relative "cli/helpers"
|
|
5
|
+
require_relative "cli/partition_diagnostics"
|
|
4
6
|
require_relative "cli/plan_command"
|
|
5
7
|
require_relative "cli/prepare_command"
|
|
6
8
|
require_relative "cli/coverage_commands"
|
|
@@ -8,8 +10,10 @@ require_relative "cli/report_commands"
|
|
|
8
10
|
require_relative "cli/env_commands"
|
|
9
11
|
require_relative "cli/database_commands"
|
|
10
12
|
require_relative "cli/run_shards_command"
|
|
13
|
+
require_relative "cli/run_queue_command"
|
|
11
14
|
require_relative "cli/queue_command"
|
|
12
15
|
require_relative "cli/timing_command"
|
|
16
|
+
require_relative "cli/spec_quality_commands"
|
|
13
17
|
require_relative "cli/init_command"
|
|
14
18
|
require_relative "cli/quick_command"
|
|
15
19
|
require_relative "cli/ci_shard_run_parse"
|
|
@@ -28,9 +32,9 @@ module Polyrun
|
|
|
28
32
|
|
|
29
33
|
# Keep in sync with +dispatch_cli_command_subcommands+ (+when+ branches). Used for implicit path routing.
|
|
30
34
|
DISPATCH_SUBCOMMAND_NAMES = %w[
|
|
31
|
-
plan prepare merge-coverage merge-failures report-coverage report-junit report-timing
|
|
35
|
+
plan prepare merge-coverage merge-failures merge-spec-quality report-coverage report-junit report-timing report-spec-quality
|
|
32
36
|
env config merge-timing db:setup-template db:setup-shard db:clone-shards
|
|
33
|
-
run-shards parallel-rspec start build-paths init queue quick hook
|
|
37
|
+
run-shards parallel-rspec start build-paths init queue run-queue quick hook
|
|
34
38
|
].freeze
|
|
35
39
|
|
|
36
40
|
# First argv token that is a normal subcommand (not a path); if argv[0] is not here but looks like paths, run implicit parallel.
|
|
@@ -39,6 +43,7 @@ module Polyrun
|
|
|
39
43
|
).freeze
|
|
40
44
|
|
|
41
45
|
include Helpers
|
|
46
|
+
include PartitionDiagnostics
|
|
42
47
|
include PlanCommand
|
|
43
48
|
include PrepareCommand
|
|
44
49
|
include CoverageCommands
|
|
@@ -46,8 +51,10 @@ module Polyrun
|
|
|
46
51
|
include EnvCommands
|
|
47
52
|
include DatabaseCommands
|
|
48
53
|
include RunShardsCommand
|
|
54
|
+
include RunQueueCommand
|
|
49
55
|
include QueueCommand
|
|
50
56
|
include TimingCommand
|
|
57
|
+
include SpecQualityCommands
|
|
51
58
|
include InitCommand
|
|
52
59
|
include QuickCommand
|
|
53
60
|
include CiShardRunParse
|
|
@@ -159,6 +166,10 @@ module Polyrun
|
|
|
159
166
|
cmd_config(argv, config_path)
|
|
160
167
|
when "merge-timing"
|
|
161
168
|
cmd_merge_timing(argv)
|
|
169
|
+
when "merge-spec-quality"
|
|
170
|
+
cmd_merge_spec_quality(argv)
|
|
171
|
+
when "report-spec-quality"
|
|
172
|
+
cmd_report_spec_quality(argv)
|
|
162
173
|
when "db:setup-template"
|
|
163
174
|
cmd_db_setup_template(argv, config_path)
|
|
164
175
|
when "db:setup-shard"
|
|
@@ -179,6 +190,8 @@ module Polyrun
|
|
|
179
190
|
cmd_init(argv, config_path)
|
|
180
191
|
when "queue"
|
|
181
192
|
cmd_queue(argv)
|
|
193
|
+
when "run-queue"
|
|
194
|
+
cmd_run_queue(argv, config_path)
|
|
182
195
|
when "quick"
|
|
183
196
|
cmd_quick(argv)
|
|
184
197
|
when "hook"
|
|
@@ -196,3 +209,4 @@ module Polyrun
|
|
|
196
209
|
end
|
|
197
210
|
end
|
|
198
211
|
end
|
|
212
|
+
# rubocop:enable Polyrun/FileLength
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
module Polyrun
|
|
2
|
+
module Coverage
|
|
3
|
+
# Per-example line hit deltas from stdlib +Coverage.peek_result+ snapshots.
|
|
4
|
+
module ExampleDiff
|
|
5
|
+
module_function
|
|
6
|
+
|
|
7
|
+
# @return [Hash{String=>Hash}] path => +{"lines"=>[...]}+
|
|
8
|
+
def peek_blob
|
|
9
|
+
return {} unless coverage_active?
|
|
10
|
+
|
|
11
|
+
normalize_peek(::Coverage.peek_result)
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
def coverage_active?
|
|
15
|
+
defined?(::Coverage) && ::Coverage.respond_to?(:running?) && ::Coverage.running?
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
def normalize_peek(raw)
|
|
19
|
+
out = {}
|
|
20
|
+
raw.each do |path, cov|
|
|
21
|
+
next unless cov.is_a?(Hash)
|
|
22
|
+
|
|
23
|
+
lines = cov[:lines] || cov["lines"]
|
|
24
|
+
next unless lines.is_a?(Array)
|
|
25
|
+
|
|
26
|
+
out[path.to_s] = {"lines" => lines.map { |x| x }}
|
|
27
|
+
end
|
|
28
|
+
out
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
# @return [Hash] +:unique_lines+, +:line_churn+, +:max_line_churn+, +:lines+ (compact triples)
|
|
32
|
+
# rubocop:disable Metrics/AbcSize -- per-file coverage delta walk
|
|
33
|
+
def diff(before_blob, after_blob)
|
|
34
|
+
before_blob ||= {}
|
|
35
|
+
after_blob ||= {}
|
|
36
|
+
files = before_blob.keys | after_blob.keys
|
|
37
|
+
|
|
38
|
+
line_entries = []
|
|
39
|
+
unique = 0
|
|
40
|
+
churn = 0
|
|
41
|
+
max_churn = 0
|
|
42
|
+
|
|
43
|
+
files.each do |path|
|
|
44
|
+
b_lines = line_array(before_blob[path])
|
|
45
|
+
a_lines = line_array(after_blob[path])
|
|
46
|
+
max_len = [b_lines.size, a_lines.size].max
|
|
47
|
+
|
|
48
|
+
(0...max_len).each do |i|
|
|
49
|
+
b = b_lines[i]
|
|
50
|
+
a = a_lines[i]
|
|
51
|
+
next if a.nil? && b.nil?
|
|
52
|
+
|
|
53
|
+
delta = (integer_hit(a) - integer_hit(b))
|
|
54
|
+
next unless delta.positive?
|
|
55
|
+
|
|
56
|
+
line_no = i + 1
|
|
57
|
+
line_entries << [path, line_no, delta]
|
|
58
|
+
unique += 1
|
|
59
|
+
churn += delta
|
|
60
|
+
max_churn = delta if delta > max_churn
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
{
|
|
65
|
+
unique_lines: unique,
|
|
66
|
+
line_churn: churn,
|
|
67
|
+
max_line_churn: max_churn,
|
|
68
|
+
lines: line_entries
|
|
69
|
+
}
|
|
70
|
+
end
|
|
71
|
+
# rubocop:enable Metrics/AbcSize
|
|
72
|
+
|
|
73
|
+
def filter_lines(lines, root:, track_under:, ignore_paths: [])
|
|
74
|
+
root = File.expand_path(root)
|
|
75
|
+
prefixes = Array(track_under).map { |d| File.join(root, d.to_s) }
|
|
76
|
+
ignore = Array(ignore_paths).map(&:to_s).reject(&:empty?)
|
|
77
|
+
|
|
78
|
+
lines.select do |path, _line, _delta|
|
|
79
|
+
p = File.expand_path(path.to_s, root)
|
|
80
|
+
next false if ignore.any? { |pat| path_matches_ignore?(p, pat) }
|
|
81
|
+
|
|
82
|
+
prefixes.any? { |pre| p == pre || p.start_with?(pre + "/") }
|
|
83
|
+
end
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
def apply_track_under(delta, root:, track_under:, ignore_paths: [])
|
|
87
|
+
filtered = filter_lines(delta[:lines] || [], root: root, track_under: track_under, ignore_paths: ignore_paths)
|
|
88
|
+
unique = filtered.size
|
|
89
|
+
churn = filtered.sum { |(_p, _l, d)| d }
|
|
90
|
+
max_churn = filtered.map { |(_p, _l, d)| d }.max || 0
|
|
91
|
+
{
|
|
92
|
+
unique_lines: unique,
|
|
93
|
+
line_churn: churn,
|
|
94
|
+
max_line_churn: max_churn,
|
|
95
|
+
lines: filtered
|
|
96
|
+
}
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
def path_matches_ignore?(path, pattern)
|
|
100
|
+
return path.include?(pattern) if pattern.is_a?(String) && !pattern.start_with?("/")
|
|
101
|
+
|
|
102
|
+
path.match?(Regexp.new(pattern))
|
|
103
|
+
rescue RegexpError
|
|
104
|
+
path.include?(pattern.to_s)
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
def line_array(entry)
|
|
108
|
+
return [] unless entry.is_a?(Hash)
|
|
109
|
+
|
|
110
|
+
arr = entry["lines"] || entry[:lines]
|
|
111
|
+
arr.is_a?(Array) ? arr : []
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
def integer_hit(value)
|
|
115
|
+
return 0 if value.nil?
|
|
116
|
+
|
|
117
|
+
value.is_a?(Integer) ? value : value.to_i
|
|
118
|
+
end
|
|
119
|
+
private_class_method :line_array, :integer_hit
|
|
120
|
+
end
|
|
121
|
+
end
|
|
122
|
+
end
|