polyrun 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CODE_OF_CONDUCT.md +31 -0
- data/CONTRIBUTING.md +84 -0
- data/LICENSE +21 -0
- data/README.md +140 -0
- data/SECURITY.md +27 -0
- data/bin/polyrun +6 -0
- data/docs/SETUP_PROFILE.md +106 -0
- data/lib/polyrun/cli/coverage_commands.rb +150 -0
- data/lib/polyrun/cli/coverage_merge_io.rb +124 -0
- data/lib/polyrun/cli/database_commands.rb +149 -0
- data/lib/polyrun/cli/env_commands.rb +43 -0
- data/lib/polyrun/cli/helpers.rb +113 -0
- data/lib/polyrun/cli/init_command.rb +99 -0
- data/lib/polyrun/cli/plan_command.rb +134 -0
- data/lib/polyrun/cli/prepare_command.rb +71 -0
- data/lib/polyrun/cli/prepare_recipe.rb +77 -0
- data/lib/polyrun/cli/queue_command.rb +101 -0
- data/lib/polyrun/cli/quick_command.rb +13 -0
- data/lib/polyrun/cli/report_commands.rb +94 -0
- data/lib/polyrun/cli/run_shards_command.rb +88 -0
- data/lib/polyrun/cli/run_shards_plan_boot_phases.rb +91 -0
- data/lib/polyrun/cli/run_shards_plan_options.rb +45 -0
- data/lib/polyrun/cli/run_shards_planning.rb +124 -0
- data/lib/polyrun/cli/run_shards_run.rb +168 -0
- data/lib/polyrun/cli/start_bootstrap.rb +99 -0
- data/lib/polyrun/cli/timing_command.rb +31 -0
- data/lib/polyrun/cli.rb +184 -0
- data/lib/polyrun/config.rb +61 -0
- data/lib/polyrun/coverage/cobertura_zero_lines.rb +32 -0
- data/lib/polyrun/coverage/collector.rb +184 -0
- data/lib/polyrun/coverage/collector_finish.rb +95 -0
- data/lib/polyrun/coverage/filter.rb +22 -0
- data/lib/polyrun/coverage/formatter.rb +115 -0
- data/lib/polyrun/coverage/merge/formatters.rb +181 -0
- data/lib/polyrun/coverage/merge/formatters_html.rb +55 -0
- data/lib/polyrun/coverage/merge.rb +127 -0
- data/lib/polyrun/coverage/merge_fragment_meta.rb +47 -0
- data/lib/polyrun/coverage/merge_merge_two.rb +117 -0
- data/lib/polyrun/coverage/rails.rb +128 -0
- data/lib/polyrun/coverage/reporting.rb +41 -0
- data/lib/polyrun/coverage/result.rb +18 -0
- data/lib/polyrun/coverage/track_files.rb +141 -0
- data/lib/polyrun/data/cached_fixtures.rb +122 -0
- data/lib/polyrun/data/factory_counts.rb +35 -0
- data/lib/polyrun/data/factory_instrumentation.rb +50 -0
- data/lib/polyrun/data/fixtures.rb +68 -0
- data/lib/polyrun/data/parallel_provisioning.rb +93 -0
- data/lib/polyrun/data/snapshot.rb +84 -0
- data/lib/polyrun/database/clone_shards.rb +81 -0
- data/lib/polyrun/database/provision.rb +72 -0
- data/lib/polyrun/database/shard.rb +63 -0
- data/lib/polyrun/database/url_builder/connection/infer.rb +49 -0
- data/lib/polyrun/database/url_builder/connection/url_builders.rb +43 -0
- data/lib/polyrun/database/url_builder/connection.rb +191 -0
- data/lib/polyrun/database/url_builder/template_prepare.rb +21 -0
- data/lib/polyrun/database/url_builder.rb +160 -0
- data/lib/polyrun/debug.rb +81 -0
- data/lib/polyrun/env/ci.rb +65 -0
- data/lib/polyrun/log.rb +70 -0
- data/lib/polyrun/minitest.rb +17 -0
- data/lib/polyrun/partition/constraints.rb +69 -0
- data/lib/polyrun/partition/hrw.rb +33 -0
- data/lib/polyrun/partition/min_heap.rb +64 -0
- data/lib/polyrun/partition/paths.rb +28 -0
- data/lib/polyrun/partition/paths_build.rb +128 -0
- data/lib/polyrun/partition/plan.rb +189 -0
- data/lib/polyrun/partition/plan_lpt.rb +49 -0
- data/lib/polyrun/partition/plan_sharding.rb +48 -0
- data/lib/polyrun/partition/stable_shuffle.rb +18 -0
- data/lib/polyrun/prepare/artifacts.rb +40 -0
- data/lib/polyrun/prepare/assets.rb +57 -0
- data/lib/polyrun/queue/file_store.rb +199 -0
- data/lib/polyrun/queue/file_store_pending.rb +48 -0
- data/lib/polyrun/quick/assertions.rb +32 -0
- data/lib/polyrun/quick/errors.rb +6 -0
- data/lib/polyrun/quick/example_group.rb +66 -0
- data/lib/polyrun/quick/example_runner.rb +93 -0
- data/lib/polyrun/quick/matchers.rb +156 -0
- data/lib/polyrun/quick/reporter.rb +42 -0
- data/lib/polyrun/quick/runner.rb +180 -0
- data/lib/polyrun/quick.rb +1 -0
- data/lib/polyrun/railtie.rb +7 -0
- data/lib/polyrun/reporting/junit.rb +125 -0
- data/lib/polyrun/reporting/junit_emit.rb +58 -0
- data/lib/polyrun/reporting/rspec_junit.rb +39 -0
- data/lib/polyrun/rspec.rb +15 -0
- data/lib/polyrun/templates/POLYRUN.md +45 -0
- data/lib/polyrun/templates/ci_matrix.polyrun.yml +14 -0
- data/lib/polyrun/templates/minimal_gem.polyrun.yml +13 -0
- data/lib/polyrun/templates/rails_prepare.polyrun.yml +31 -0
- data/lib/polyrun/timing/merge.rb +35 -0
- data/lib/polyrun/timing/summary.rb +25 -0
- data/lib/polyrun/version.rb +3 -0
- data/lib/polyrun.rb +58 -0
- data/polyrun.gemspec +37 -0
- data/sig/polyrun/cli.rbs +6 -0
- data/sig/polyrun/config.rbs +20 -0
- data/sig/polyrun/debug.rbs +12 -0
- data/sig/polyrun/log.rbs +12 -0
- data/sig/polyrun/minitest.rbs +5 -0
- data/sig/polyrun/quick.rbs +19 -0
- data/sig/polyrun/rspec.rbs +5 -0
- data/sig/polyrun.rbs +11 -0
- metadata +288 -0
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
require "json"
|
|
2
|
+
require "fileutils"
|
|
3
|
+
require "optparse"
|
|
4
|
+
|
|
5
|
+
module Polyrun
|
|
6
|
+
class CLI
|
|
7
|
+
# Writes merged coverage blob to requested formats (merge-coverage / run-shards).
|
|
8
|
+
module CoverageMergeIo
|
|
9
|
+
private
|
|
10
|
+
|
|
11
|
+
def merge_coverage_parse_argv(argv)
|
|
12
|
+
inputs = []
|
|
13
|
+
output = "coverage/polyrun-merged.json"
|
|
14
|
+
formats = ["json"]
|
|
15
|
+
parser = OptionParser.new do |opts|
|
|
16
|
+
opts.banner = "usage: polyrun merge-coverage -i FILE [-i FILE] [-o PATH] [--format json,lcov,cobertura,console,html]"
|
|
17
|
+
opts.on("-i", "--input FILE", "Coverage JSON (repeatable; globs ok)") do |f|
|
|
18
|
+
expand_merge_input_pattern(f).each { |x| inputs << x }
|
|
19
|
+
end
|
|
20
|
+
opts.on("-o", "--output PATH", String) { |v| output = v }
|
|
21
|
+
opts.on("--format LIST", String) { |v| formats = v.split(",").map(&:strip) }
|
|
22
|
+
end
|
|
23
|
+
parser.parse!(argv)
|
|
24
|
+
inputs.uniq!
|
|
25
|
+
inputs.select! { |p| File.file?(p) }
|
|
26
|
+
[inputs, output, formats]
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def merge_coverage_merge_fragments(inputs)
|
|
30
|
+
Polyrun::Debug.time("Coverage::Merge.merge_fragments") do
|
|
31
|
+
Polyrun::Coverage::Merge.merge_fragments(inputs)
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def merge_coverage_write_json_payload(out_abs, payload)
|
|
36
|
+
Polyrun::Debug.time("write merged JSON") do
|
|
37
|
+
FileUtils.mkdir_p(File.dirname(out_abs))
|
|
38
|
+
File.write(out_abs, JSON.generate(payload))
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
def merge_coverage_write_format_outputs(merged, r, out_abs, formats)
|
|
43
|
+
write_merge_lcov(merged, out_abs) if formats.include?("lcov")
|
|
44
|
+
write_merge_cobertura(merged, r, out_abs) if formats.include?("cobertura")
|
|
45
|
+
write_merge_console(merged, out_abs) if formats.include?("console")
|
|
46
|
+
write_merge_html(merged, out_abs) if formats.include?("html")
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
def write_merge_lcov(merged, out_abs)
|
|
50
|
+
lcov_path = out_abs.sub(/\.json\z/, ".lcov")
|
|
51
|
+
lcov_path = "#{out_abs}.lcov" if lcov_path == out_abs
|
|
52
|
+
Polyrun::Debug.time("write lcov") { File.write(lcov_path, Polyrun::Coverage::Merge.emit_lcov(merged)) }
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
def write_merge_cobertura(merged, r, out_abs)
|
|
56
|
+
cob_path = out_abs.sub(/\.json\z/, ".xml")
|
|
57
|
+
cob_path = "#{out_abs}.cobertura.xml" if cob_path == out_abs
|
|
58
|
+
root = merge_cobertura_root(r)
|
|
59
|
+
Polyrun::Debug.time("write cobertura XML") do
|
|
60
|
+
File.write(cob_path, Polyrun::Coverage::Merge.emit_cobertura(merged, root: root))
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
def merge_cobertura_root(r)
|
|
65
|
+
return nil unless r[:meta].is_a?(Hash)
|
|
66
|
+
|
|
67
|
+
r[:meta]["polyrun_coverage_root"] || r[:meta][:polyrun_coverage_root]
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
def write_merge_console(merged, out_abs)
|
|
71
|
+
sum_path = out_abs.sub(/\.json\z/, "-summary.txt")
|
|
72
|
+
sum_path = "#{out_abs}-summary.txt" if sum_path == out_abs
|
|
73
|
+
summary = Polyrun::Coverage::Merge.console_summary(merged)
|
|
74
|
+
summary_text = Polyrun::Coverage::Merge.format_console_summary(summary)
|
|
75
|
+
Polyrun::Debug.time("write console summary") { File.write(sum_path, summary_text) }
|
|
76
|
+
Polyrun::Log.print summary_text
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
def write_merge_html(merged, out_abs)
|
|
80
|
+
html_path = out_abs.sub(/\.json\z/, ".html")
|
|
81
|
+
html_path = "#{out_abs}.html" if html_path == out_abs
|
|
82
|
+
Polyrun::Debug.time("write HTML report") { File.write(html_path, Polyrun::Coverage::Merge.emit_html(merged)) }
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
def merge_coverage_log_finish(elapsed, inputs)
|
|
86
|
+
thr = merge_slow_warn_threshold_seconds
|
|
87
|
+
merge_coverage_warn_if_slow(elapsed, thr, inputs) if thr && elapsed > thr
|
|
88
|
+
return unless @verbose || ENV["POLYRUN_PROFILE_MERGE"] == "1" || Polyrun::Debug.enabled?
|
|
89
|
+
|
|
90
|
+
Polyrun::Log.warn format("merge-coverage: finished in %.2fs (%d input fragment(s))", elapsed, inputs.size)
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
def merge_coverage_warn_if_slow(elapsed, thr, inputs)
|
|
94
|
+
Polyrun::Log.warn format(
|
|
95
|
+
"merge-coverage: slow merge took %.2fs (warn above %.0fs; typical suites are JSON fragments, not TB-scale data; disable: POLYRUN_MERGE_SLOW_WARN_SECONDS=0)",
|
|
96
|
+
elapsed,
|
|
97
|
+
thr
|
|
98
|
+
)
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
def merge_coverage_min_line_gate_below?(merged_path, gate)
|
|
102
|
+
Polyrun::Debug.time("minimum_line_percent gate (merged JSON)") do
|
|
103
|
+
data = JSON.parse(File.read(merged_path))
|
|
104
|
+
blob = Polyrun::Coverage::Merge.extract_coverage_blob(data)
|
|
105
|
+
summary = Polyrun::Coverage::Merge.console_summary(blob)
|
|
106
|
+
min = gate[:minimum]
|
|
107
|
+
below = summary[:line_percent].round < min.round
|
|
108
|
+
Polyrun::Debug.log_kv(
|
|
109
|
+
merged_line_percent: summary[:line_percent],
|
|
110
|
+
gate_minimum: min,
|
|
111
|
+
below_gate: below
|
|
112
|
+
)
|
|
113
|
+
log_gate_below_minimum(summary, min) if below
|
|
114
|
+
below
|
|
115
|
+
end
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
def log_gate_below_minimum(summary, min)
|
|
119
|
+
Polyrun::Log.warn Polyrun::Coverage::Merge.format_console_summary(summary)
|
|
120
|
+
Polyrun::Log.warn "Polyrun coverage: #{summary[:line_percent].round(2)}% rounds below minimum #{min}% (merged)."
|
|
121
|
+
end
|
|
122
|
+
end
|
|
123
|
+
end
|
|
124
|
+
end
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
require "optparse"
|
|
2
|
+
|
|
3
|
+
module Polyrun
|
|
4
|
+
class CLI
|
|
5
|
+
module DatabaseCommands
|
|
6
|
+
private
|
|
7
|
+
|
|
8
|
+
def cmd_db_setup_template(argv, config_path)
|
|
9
|
+
dry = false
|
|
10
|
+
rails_root = Dir.pwd
|
|
11
|
+
parser = OptionParser.new do |opts|
|
|
12
|
+
opts.on("--dry-run", "Print only") { dry = true }
|
|
13
|
+
opts.on("--rails-root PATH", String) { |v| rails_root = v }
|
|
14
|
+
end
|
|
15
|
+
parser.parse!(argv)
|
|
16
|
+
|
|
17
|
+
cfg = Polyrun::Config.load(path: config_path || ENV["POLYRUN_CONFIG"])
|
|
18
|
+
dh = cfg.databases
|
|
19
|
+
if !dh.is_a?(Hash) || dh.empty?
|
|
20
|
+
Polyrun::Log.warn "db:setup-template: configure databases: in polyrun.yml"
|
|
21
|
+
return 2
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
begin
|
|
25
|
+
te = Polyrun::Database::UrlBuilder.template_prepare_env(dh)
|
|
26
|
+
rescue Polyrun::Error => e
|
|
27
|
+
Polyrun::Log.warn "db:setup-template: #{e.message}"
|
|
28
|
+
return 2
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
if dry
|
|
32
|
+
log = Polyrun::Database::UrlBuilder.template_prepare_env_shell_log(dh)
|
|
33
|
+
Polyrun::Log.warn "would: RAILS_ENV=test #{log} bin/rails db:prepare"
|
|
34
|
+
return 0
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
child_env = ENV.to_h.merge(te)
|
|
38
|
+
child_env["RAILS_ENV"] ||= ENV["RAILS_ENV"] || "test"
|
|
39
|
+
Polyrun::Database::Provision.prepare_template!(
|
|
40
|
+
rails_root: File.expand_path(rails_root),
|
|
41
|
+
env: child_env,
|
|
42
|
+
silent: !@verbose
|
|
43
|
+
)
|
|
44
|
+
0
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def cmd_db_setup_shard(argv, config_path)
|
|
48
|
+
dry = db_setup_shard_parse_options!(argv)
|
|
49
|
+
cfg = Polyrun::Config.load(path: config_path || ENV["POLYRUN_CONFIG"])
|
|
50
|
+
dh = cfg.databases
|
|
51
|
+
if !dh.is_a?(Hash) || dh.empty?
|
|
52
|
+
Polyrun::Log.warn "db:setup-shard: configure databases: in polyrun.yml"
|
|
53
|
+
return 2
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
shard = resolve_shard_index(cfg.partition)
|
|
57
|
+
template = dh["template_db"] || dh[:template_db]
|
|
58
|
+
if !template
|
|
59
|
+
Polyrun::Log.warn "db:setup-shard: set databases.template_db"
|
|
60
|
+
return 2
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
plan = Polyrun::Database::UrlBuilder.shard_database_plan(dh, shard_index: shard)
|
|
64
|
+
if plan.empty?
|
|
65
|
+
Polyrun::Log.warn "db:setup-shard: could not derive shard database names from polyrun.yml"
|
|
66
|
+
return 2
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
db_setup_shard_run_plan(plan, dry: dry)
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
def db_setup_shard_parse_options!(argv)
|
|
73
|
+
dry = false
|
|
74
|
+
OptionParser.new do |opts|
|
|
75
|
+
opts.on("--dry-run", "Print only") { dry = true }
|
|
76
|
+
end.parse!(argv)
|
|
77
|
+
dry
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
def db_setup_shard_run_plan(plan, dry:)
|
|
81
|
+
if dry
|
|
82
|
+
plan.each do |row|
|
|
83
|
+
Polyrun::Log.warn "would: CREATE DATABASE #{row[:new_db]} TEMPLATE #{row[:template_db]}"
|
|
84
|
+
end
|
|
85
|
+
return 0
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
threads = plan.map do |row|
|
|
89
|
+
Thread.new do
|
|
90
|
+
Polyrun::Database::Provision.create_database_from_template!(
|
|
91
|
+
new_db: row[:new_db],
|
|
92
|
+
template_db: row[:template_db].to_s
|
|
93
|
+
)
|
|
94
|
+
end
|
|
95
|
+
end
|
|
96
|
+
threads.each(&:join)
|
|
97
|
+
0
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
# Migrate all template DBs + create every shard database (primary + +connections+).
|
|
101
|
+
def cmd_db_clone_shards(argv, config_path)
|
|
102
|
+
opts = db_clone_shards_parse_options!(argv)
|
|
103
|
+
cfg = Polyrun::Config.load(path: config_path || ENV["POLYRUN_CONFIG"])
|
|
104
|
+
dh = cfg.databases
|
|
105
|
+
if !dh.is_a?(Hash) || dh.empty?
|
|
106
|
+
Polyrun::Log.warn "db:clone-shards: configure databases: in polyrun.yml"
|
|
107
|
+
return 2
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
workers = opts[:workers].clamp(1, 10)
|
|
111
|
+
Polyrun::Database::CloneShards.provision!(
|
|
112
|
+
dh,
|
|
113
|
+
workers: workers,
|
|
114
|
+
rails_root: File.expand_path(opts[:rails_root]),
|
|
115
|
+
migrate: opts[:migrate],
|
|
116
|
+
replace: opts[:replace],
|
|
117
|
+
force_drop: opts[:force_drop],
|
|
118
|
+
dry_run: opts[:dry],
|
|
119
|
+
silent: !@verbose
|
|
120
|
+
)
|
|
121
|
+
0
|
|
122
|
+
rescue Polyrun::Error => e
|
|
123
|
+
Polyrun::Log.warn "db:clone-shards: #{e.message}"
|
|
124
|
+
1
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
def db_clone_shards_parse_options!(argv)
|
|
128
|
+
dry = false
|
|
129
|
+
migrate = true
|
|
130
|
+
replace = true
|
|
131
|
+
force_drop = false
|
|
132
|
+
rails_root = Dir.pwd
|
|
133
|
+
workers = env_int("POLYRUN_WORKERS", 5)
|
|
134
|
+
|
|
135
|
+
OptionParser.new do |opts|
|
|
136
|
+
opts.banner = "usage: polyrun db:clone-shards [--workers N] [--rails-root PATH] [--dry-run] [--no-migrate] [--no-replace] [--force-drop]"
|
|
137
|
+
opts.on("--workers N", Integer) { |v| workers = v }
|
|
138
|
+
opts.on("--dry-run", "Print only") { dry = true }
|
|
139
|
+
opts.on("--rails-root PATH", String) { |v| rails_root = v }
|
|
140
|
+
opts.on("--no-migrate", "Skip db:prepare on template databases") { migrate = false }
|
|
141
|
+
opts.on("--no-replace", "Skip DROP DATABASE before CREATE (fail if shard DB exists)") { replace = false }
|
|
142
|
+
opts.on("--force-drop", "DROP DATABASE … WITH (FORCE) (PostgreSQL 13+)") { force_drop = true }
|
|
143
|
+
end.parse!(argv)
|
|
144
|
+
|
|
145
|
+
{dry: dry, migrate: migrate, replace: replace, force_drop: force_drop, rails_root: rails_root, workers: workers}
|
|
146
|
+
end
|
|
147
|
+
end
|
|
148
|
+
end
|
|
149
|
+
end
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
require "optparse"
|
|
2
|
+
|
|
3
|
+
module Polyrun
|
|
4
|
+
class CLI
|
|
5
|
+
module EnvCommands
|
|
6
|
+
private
|
|
7
|
+
|
|
8
|
+
def cmd_env(argv, config_path)
|
|
9
|
+
shard, total, base_database = env_parse_options!(argv)
|
|
10
|
+
cfg = Polyrun::Config.load(path: config_path || ENV["POLYRUN_CONFIG"])
|
|
11
|
+
pc = cfg.partition
|
|
12
|
+
shard = shard.nil? ? resolve_shard_index(pc) : shard
|
|
13
|
+
total = total.nil? ? resolve_shard_total(pc) : total
|
|
14
|
+
|
|
15
|
+
env_print_database_exports(cfg.databases, shard)
|
|
16
|
+
Polyrun::Database::Shard.env_map(shard_index: shard, shard_total: total, base_database: base_database).each do |k, v|
|
|
17
|
+
Polyrun::Log.puts %(export #{k}=#{v})
|
|
18
|
+
end
|
|
19
|
+
0
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def env_parse_options!(argv)
|
|
23
|
+
shard = nil
|
|
24
|
+
total = nil
|
|
25
|
+
base_database = nil
|
|
26
|
+
OptionParser.new do |opts|
|
|
27
|
+
opts.on("--shard INDEX", Integer) { |v| shard = v }
|
|
28
|
+
opts.on("--total N", Integer) { |v| total = v }
|
|
29
|
+
opts.on("--database TEMPLATE", String) { |v| base_database = v }
|
|
30
|
+
end.parse!(argv)
|
|
31
|
+
[shard, total, base_database]
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def env_print_database_exports(dh, shard)
|
|
35
|
+
return unless dh.is_a?(Hash) && !dh.empty?
|
|
36
|
+
|
|
37
|
+
Polyrun::Database::UrlBuilder.env_exports_for_databases(dh, shard_index: shard).each do |k, v|
|
|
38
|
+
Polyrun::Log.puts %(export #{k}=#{v})
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
end
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
require "yaml"
|
|
2
|
+
|
|
3
|
+
module Polyrun
|
|
4
|
+
class CLI
|
|
5
|
+
module Helpers
|
|
6
|
+
private
|
|
7
|
+
|
|
8
|
+
def partition_int(pc, keys, default)
|
|
9
|
+
keys.each do |k|
|
|
10
|
+
v = pc[k] || pc[k.to_sym]
|
|
11
|
+
next if v.nil? || v.to_s.empty?
|
|
12
|
+
|
|
13
|
+
i = Integer(v, exception: false)
|
|
14
|
+
return i unless i.nil?
|
|
15
|
+
end
|
|
16
|
+
default
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
def env_int(name, fallback)
|
|
20
|
+
s = ENV[name]
|
|
21
|
+
return fallback if s.nil? || s.empty?
|
|
22
|
+
|
|
23
|
+
Integer(s, exception: false) || fallback
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def resolve_shard_index(pc)
|
|
27
|
+
return Integer(ENV["POLYRUN_SHARD_INDEX"]) if ENV["POLYRUN_SHARD_INDEX"] && !ENV["POLYRUN_SHARD_INDEX"].empty?
|
|
28
|
+
|
|
29
|
+
ci = Polyrun::Env::Ci.detect_shard_index
|
|
30
|
+
return ci unless ci.nil?
|
|
31
|
+
|
|
32
|
+
partition_int(pc, %w[shard_index shard], 0)
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def resolve_shard_total(pc)
|
|
36
|
+
return Integer(ENV["POLYRUN_SHARD_TOTAL"]) if ENV["POLYRUN_SHARD_TOTAL"] && !ENV["POLYRUN_SHARD_TOTAL"].empty?
|
|
37
|
+
|
|
38
|
+
ci = Polyrun::Env::Ci.detect_shard_total
|
|
39
|
+
return ci unless ci.nil?
|
|
40
|
+
|
|
41
|
+
partition_int(pc, %w[shard_total total], 1)
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
def expand_merge_input_pattern(path)
|
|
45
|
+
p = path.to_s
|
|
46
|
+
abs = File.expand_path(p, Dir.pwd)
|
|
47
|
+
return Dir.glob(abs).sort if p.include?("*") || p.include?("?")
|
|
48
|
+
|
|
49
|
+
[abs]
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
# Same rounding/strict semantics as {Polyrun::Coverage::Collector} for +config/polyrun_coverage.yml+.
|
|
53
|
+
def coverage_minimum_line_gate_from_polyrun_coverage_yml
|
|
54
|
+
path = File.join(Dir.pwd, "config", "polyrun_coverage.yml")
|
|
55
|
+
return nil unless File.file?(path)
|
|
56
|
+
|
|
57
|
+
data = YAML.safe_load_file(path, permitted_classes: [Symbol], aliases: true)
|
|
58
|
+
return nil unless data.is_a?(Hash)
|
|
59
|
+
|
|
60
|
+
min = data["minimum_line_percent"] || data[:minimum_line_percent]
|
|
61
|
+
return nil if min.nil?
|
|
62
|
+
|
|
63
|
+
sv = data["strict"] if data.key?("strict")
|
|
64
|
+
sv = data[:strict] if !data.key?("strict") && data.key?(:strict)
|
|
65
|
+
strict = sv.nil? || sv
|
|
66
|
+
|
|
67
|
+
{minimum: min.to_f, strict: strict != false}
|
|
68
|
+
rescue Psych::SyntaxError, ArgumentError, TypeError
|
|
69
|
+
nil
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
def load_partition_constraints(pc, constraints_path)
|
|
73
|
+
if constraints_path
|
|
74
|
+
path = File.expand_path(constraints_path.to_s, Dir.pwd)
|
|
75
|
+
unless File.file?(path)
|
|
76
|
+
Polyrun::Log.warn "polyrun: constraints file not found: #{path}"
|
|
77
|
+
return nil
|
|
78
|
+
end
|
|
79
|
+
h = YAML.safe_load_file(path, permitted_classes: [Symbol], aliases: true)
|
|
80
|
+
return Polyrun::Partition::Constraints.from_hash(h, root: Dir.pwd)
|
|
81
|
+
end
|
|
82
|
+
if pc["constraints"].is_a?(Hash)
|
|
83
|
+
return Polyrun::Partition::Constraints.from_hash(pc["constraints"], root: Dir.pwd)
|
|
84
|
+
end
|
|
85
|
+
cf = pc["constraints_file"] || pc[:constraints_file]
|
|
86
|
+
if cf
|
|
87
|
+
path = File.expand_path(cf.to_s, Dir.pwd)
|
|
88
|
+
return nil unless File.file?(path)
|
|
89
|
+
|
|
90
|
+
h = YAML.safe_load_file(path, permitted_classes: [Symbol], aliases: true)
|
|
91
|
+
return Polyrun::Partition::Constraints.from_hash(h, root: Dir.pwd)
|
|
92
|
+
end
|
|
93
|
+
nil
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
# +default_weight+ should be precomputed when sorting many paths (e.g. +queue init+), matching
|
|
97
|
+
# {Partition::Plan#default_weight} semantics: mean of known timing costs for missing paths.
|
|
98
|
+
def queue_weight_for(path, costs, default_weight = nil)
|
|
99
|
+
abs = File.expand_path(path.to_s, Dir.pwd)
|
|
100
|
+
return costs[abs] if costs.key?(abs)
|
|
101
|
+
|
|
102
|
+
unless default_weight.nil?
|
|
103
|
+
return default_weight
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
vals = costs.values
|
|
107
|
+
return 1.0 if vals.empty?
|
|
108
|
+
|
|
109
|
+
vals.sum / vals.size.to_f
|
|
110
|
+
end
|
|
111
|
+
end
|
|
112
|
+
end
|
|
113
|
+
end
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
require "optparse"
|
|
2
|
+
|
|
3
|
+
module Polyrun
|
|
4
|
+
class CLI
|
|
5
|
+
module InitCommand
|
|
6
|
+
INIT_PROFILES = {
|
|
7
|
+
"gem" => "minimal_gem.polyrun.yml",
|
|
8
|
+
"rails" => "rails_prepare.polyrun.yml",
|
|
9
|
+
"ci-matrix" => "ci_matrix.polyrun.yml",
|
|
10
|
+
"doc" => "POLYRUN.md"
|
|
11
|
+
}.freeze
|
|
12
|
+
|
|
13
|
+
private
|
|
14
|
+
|
|
15
|
+
def templates_dir
|
|
16
|
+
File.expand_path("../templates", __dir__)
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
def cmd_init(argv, _config_path)
|
|
20
|
+
profile, output, dry_run, force, list = init_parse_options!(argv)
|
|
21
|
+
return init_list_profiles if list
|
|
22
|
+
return init_missing_profile if profile.nil?
|
|
23
|
+
|
|
24
|
+
filename = INIT_PROFILES[profile]
|
|
25
|
+
return init_unknown_profile(profile) unless filename
|
|
26
|
+
|
|
27
|
+
src = File.join(templates_dir, filename)
|
|
28
|
+
return init_missing_template(src) unless File.file?(src)
|
|
29
|
+
|
|
30
|
+
body = File.read(src, encoding: Encoding::UTF_8)
|
|
31
|
+
dest = output || default_init_output(profile)
|
|
32
|
+
return init_dry_run_print(body) if dry_run
|
|
33
|
+
|
|
34
|
+
path = File.expand_path(dest)
|
|
35
|
+
return init_refuses_overwrite(path) if File.file?(path) && !force
|
|
36
|
+
|
|
37
|
+
File.write(path, body)
|
|
38
|
+
Polyrun::Log.warn "polyrun init: wrote #{path}"
|
|
39
|
+
0
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
def init_parse_options!(argv)
|
|
43
|
+
profile = nil
|
|
44
|
+
output = nil
|
|
45
|
+
dry_run = false
|
|
46
|
+
force = false
|
|
47
|
+
list = false
|
|
48
|
+
|
|
49
|
+
OptionParser.new do |opts|
|
|
50
|
+
opts.banner = "usage: polyrun init [--profile NAME] [--output PATH] [--dry-run] [--force]\n polyrun init --list"
|
|
51
|
+
opts.on("--profile NAME", INIT_PROFILES.keys.join(", ")) { |p| profile = p }
|
|
52
|
+
opts.on("-o", "--output PATH", "destination file (default: polyrun.yml or POLYRUN.md for --profile doc)") { |p| output = p }
|
|
53
|
+
opts.on("--dry-run", "print template to stdout; do not write") { dry_run = true }
|
|
54
|
+
opts.on("--force", "overwrite existing output file") { force = true }
|
|
55
|
+
opts.on("--list", "print available profiles") { list = true }
|
|
56
|
+
end.parse!(argv)
|
|
57
|
+
|
|
58
|
+
[profile, output, dry_run, force, list]
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
def init_list_profiles
|
|
62
|
+
Polyrun::Log.puts "polyrun init profiles:"
|
|
63
|
+
INIT_PROFILES.each do |name, file|
|
|
64
|
+
Polyrun::Log.puts " #{name.ljust(12)} #{file}"
|
|
65
|
+
end
|
|
66
|
+
0
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
def init_missing_profile
|
|
70
|
+
Polyrun::Log.warn "polyrun init: specify --profile (#{INIT_PROFILES.keys.join(", ")}) or --list"
|
|
71
|
+
2
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
def init_unknown_profile(profile)
|
|
75
|
+
Polyrun::Log.warn "polyrun init: unknown profile #{profile.inspect}"
|
|
76
|
+
2
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
def init_missing_template(src)
|
|
80
|
+
Polyrun::Log.warn "polyrun init: template missing: #{src}"
|
|
81
|
+
1
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
def init_dry_run_print(body)
|
|
85
|
+
Polyrun::Log.print body
|
|
86
|
+
0
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
def init_refuses_overwrite(path)
|
|
90
|
+
Polyrun::Log.warn "polyrun init: #{path} exists (use --force to overwrite)"
|
|
91
|
+
1
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
def default_init_output(profile)
|
|
95
|
+
(profile == "doc") ? "POLYRUN.md" : "polyrun.yml"
|
|
96
|
+
end
|
|
97
|
+
end
|
|
98
|
+
end
|
|
99
|
+
end
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
require "json"
|
|
2
|
+
require "optparse"
|
|
3
|
+
|
|
4
|
+
module Polyrun
|
|
5
|
+
class CLI
|
|
6
|
+
module PlanCommand
|
|
7
|
+
private
|
|
8
|
+
|
|
9
|
+
def cmd_plan(argv, config_path)
|
|
10
|
+
cfg = Polyrun::Config.load(path: config_path || ENV["POLYRUN_CONFIG"])
|
|
11
|
+
pc = cfg.partition
|
|
12
|
+
ctx = plan_command_initial_context(pc)
|
|
13
|
+
plan_command_parse_argv!(argv, ctx)
|
|
14
|
+
|
|
15
|
+
paths_file = ctx[:paths_file] || (pc["paths_file"] || pc[:paths_file])
|
|
16
|
+
code = Polyrun::Partition::PathsBuild.apply!(partition: pc, cwd: Dir.pwd)
|
|
17
|
+
return code if code != 0
|
|
18
|
+
|
|
19
|
+
timing_path = plan_resolve_timing_path(pc, ctx[:timing_path], ctx[:strategy])
|
|
20
|
+
Polyrun::Log.warn "polyrun plan: using #{cfg.path}" if @verbose && cfg.path
|
|
21
|
+
|
|
22
|
+
items = plan_plan_items(paths_file, argv)
|
|
23
|
+
return 2 if items.nil?
|
|
24
|
+
|
|
25
|
+
loaded = plan_load_costs_and_strategy(timing_path, ctx[:strategy])
|
|
26
|
+
return 2 if loaded.nil?
|
|
27
|
+
|
|
28
|
+
costs, strategy = loaded
|
|
29
|
+
|
|
30
|
+
constraints = load_partition_constraints(pc, ctx[:constraints_path])
|
|
31
|
+
|
|
32
|
+
plan_command_emit_manifest(
|
|
33
|
+
items: items,
|
|
34
|
+
total: ctx[:total],
|
|
35
|
+
strategy: strategy,
|
|
36
|
+
seed: ctx[:seed],
|
|
37
|
+
costs: costs,
|
|
38
|
+
constraints: constraints,
|
|
39
|
+
shard: ctx[:shard]
|
|
40
|
+
)
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def plan_command_initial_context(pc)
|
|
44
|
+
{
|
|
45
|
+
shard: resolve_shard_index(pc),
|
|
46
|
+
total: resolve_shard_total(pc),
|
|
47
|
+
strategy: (pc["strategy"] || pc[:strategy] || "round_robin").to_s,
|
|
48
|
+
seed: pc["seed"] || pc[:seed],
|
|
49
|
+
paths_file: nil,
|
|
50
|
+
timing_path: nil,
|
|
51
|
+
constraints_path: nil
|
|
52
|
+
}
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
def plan_command_parse_argv!(argv, ctx)
|
|
56
|
+
OptionParser.new do |opts|
|
|
57
|
+
opts.banner = "usage: polyrun plan [options] [--] [paths...]"
|
|
58
|
+
opts.on("--shard INDEX", Integer) { |v| ctx[:shard] = v }
|
|
59
|
+
opts.on("--total N", Integer) { |v| ctx[:total] = v }
|
|
60
|
+
opts.on("--strategy NAME", String) { |v| ctx[:strategy] = v }
|
|
61
|
+
opts.on("--seed VAL") { |v| ctx[:seed] = v }
|
|
62
|
+
opts.on("--paths-file PATH", String) { |v| ctx[:paths_file] = v }
|
|
63
|
+
opts.on("--constraints PATH", "YAML: pin / serial_glob (see spec_queue.md)") { |v| ctx[:constraints_path] = v }
|
|
64
|
+
opts.on("--timing PATH", "path => seconds JSON; implies cost_binpack unless strategy is cost-based or hrw") do |v|
|
|
65
|
+
ctx[:timing_path] = v
|
|
66
|
+
end
|
|
67
|
+
end.parse!(argv)
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
def plan_command_emit_manifest(items:, total:, strategy:, seed:, costs:, constraints:, shard:)
|
|
71
|
+
plan = Polyrun::Debug.time("Partition::Plan.new (plan command)") do
|
|
72
|
+
Polyrun::Partition::Plan.new(
|
|
73
|
+
items: items,
|
|
74
|
+
total_shards: total,
|
|
75
|
+
strategy: strategy,
|
|
76
|
+
seed: seed,
|
|
77
|
+
costs: costs,
|
|
78
|
+
constraints: constraints,
|
|
79
|
+
root: Dir.pwd
|
|
80
|
+
)
|
|
81
|
+
end
|
|
82
|
+
Polyrun::Debug.log_kv(
|
|
83
|
+
plan: "emit manifest JSON",
|
|
84
|
+
shard: shard,
|
|
85
|
+
total: total,
|
|
86
|
+
strategy: strategy,
|
|
87
|
+
path_count: items.size
|
|
88
|
+
)
|
|
89
|
+
Polyrun::Log.puts JSON.generate(plan.manifest(shard))
|
|
90
|
+
0
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
def plan_resolve_timing_path(pc, timing_path, strategy)
|
|
94
|
+
return timing_path if timing_path
|
|
95
|
+
|
|
96
|
+
tf = pc["timing_file"] || pc[:timing_file]
|
|
97
|
+
return tf if tf && (Polyrun::Partition::Plan.cost_strategy?(strategy) || Polyrun::Partition::Plan.hrw_strategy?(strategy))
|
|
98
|
+
|
|
99
|
+
nil
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
def plan_plan_items(paths_file, argv)
|
|
103
|
+
if paths_file
|
|
104
|
+
Polyrun::Partition::Paths.read_lines(paths_file)
|
|
105
|
+
elsif argv.empty?
|
|
106
|
+
Polyrun::Log.warn "polyrun plan: provide spec paths, --paths-file, or partition.paths_file in polyrun.yml"
|
|
107
|
+
nil
|
|
108
|
+
else
|
|
109
|
+
argv
|
|
110
|
+
end
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
def plan_load_costs_and_strategy(timing_path, strategy)
|
|
114
|
+
if timing_path
|
|
115
|
+
costs = Polyrun::Partition::Plan.load_timing_costs(File.expand_path(timing_path.to_s, Dir.pwd))
|
|
116
|
+
if costs.empty?
|
|
117
|
+
Polyrun::Log.warn "polyrun plan: timing file missing or has no entries: #{timing_path}"
|
|
118
|
+
return nil
|
|
119
|
+
end
|
|
120
|
+
unless Polyrun::Partition::Plan.cost_strategy?(strategy) || Polyrun::Partition::Plan.hrw_strategy?(strategy)
|
|
121
|
+
Polyrun::Log.warn "polyrun plan: using cost_binpack (timing data present)" if @verbose
|
|
122
|
+
strategy = "cost_binpack"
|
|
123
|
+
end
|
|
124
|
+
[costs, strategy]
|
|
125
|
+
elsif Polyrun::Partition::Plan.cost_strategy?(strategy)
|
|
126
|
+
Polyrun::Log.warn "polyrun plan: --timing or partition.timing_file required for strategy #{strategy}"
|
|
127
|
+
nil
|
|
128
|
+
else
|
|
129
|
+
[nil, strategy]
|
|
130
|
+
end
|
|
131
|
+
end
|
|
132
|
+
end
|
|
133
|
+
end
|
|
134
|
+
end
|