polyrun 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (105) hide show
  1. checksums.yaml +7 -0
  2. data/CODE_OF_CONDUCT.md +31 -0
  3. data/CONTRIBUTING.md +84 -0
  4. data/LICENSE +21 -0
  5. data/README.md +140 -0
  6. data/SECURITY.md +27 -0
  7. data/bin/polyrun +6 -0
  8. data/docs/SETUP_PROFILE.md +106 -0
  9. data/lib/polyrun/cli/coverage_commands.rb +150 -0
  10. data/lib/polyrun/cli/coverage_merge_io.rb +124 -0
  11. data/lib/polyrun/cli/database_commands.rb +149 -0
  12. data/lib/polyrun/cli/env_commands.rb +43 -0
  13. data/lib/polyrun/cli/helpers.rb +113 -0
  14. data/lib/polyrun/cli/init_command.rb +99 -0
  15. data/lib/polyrun/cli/plan_command.rb +134 -0
  16. data/lib/polyrun/cli/prepare_command.rb +71 -0
  17. data/lib/polyrun/cli/prepare_recipe.rb +77 -0
  18. data/lib/polyrun/cli/queue_command.rb +101 -0
  19. data/lib/polyrun/cli/quick_command.rb +13 -0
  20. data/lib/polyrun/cli/report_commands.rb +94 -0
  21. data/lib/polyrun/cli/run_shards_command.rb +88 -0
  22. data/lib/polyrun/cli/run_shards_plan_boot_phases.rb +91 -0
  23. data/lib/polyrun/cli/run_shards_plan_options.rb +45 -0
  24. data/lib/polyrun/cli/run_shards_planning.rb +124 -0
  25. data/lib/polyrun/cli/run_shards_run.rb +168 -0
  26. data/lib/polyrun/cli/start_bootstrap.rb +99 -0
  27. data/lib/polyrun/cli/timing_command.rb +31 -0
  28. data/lib/polyrun/cli.rb +184 -0
  29. data/lib/polyrun/config.rb +61 -0
  30. data/lib/polyrun/coverage/cobertura_zero_lines.rb +32 -0
  31. data/lib/polyrun/coverage/collector.rb +184 -0
  32. data/lib/polyrun/coverage/collector_finish.rb +95 -0
  33. data/lib/polyrun/coverage/filter.rb +22 -0
  34. data/lib/polyrun/coverage/formatter.rb +115 -0
  35. data/lib/polyrun/coverage/merge/formatters.rb +181 -0
  36. data/lib/polyrun/coverage/merge/formatters_html.rb +55 -0
  37. data/lib/polyrun/coverage/merge.rb +127 -0
  38. data/lib/polyrun/coverage/merge_fragment_meta.rb +47 -0
  39. data/lib/polyrun/coverage/merge_merge_two.rb +117 -0
  40. data/lib/polyrun/coverage/rails.rb +128 -0
  41. data/lib/polyrun/coverage/reporting.rb +41 -0
  42. data/lib/polyrun/coverage/result.rb +18 -0
  43. data/lib/polyrun/coverage/track_files.rb +141 -0
  44. data/lib/polyrun/data/cached_fixtures.rb +122 -0
  45. data/lib/polyrun/data/factory_counts.rb +35 -0
  46. data/lib/polyrun/data/factory_instrumentation.rb +50 -0
  47. data/lib/polyrun/data/fixtures.rb +68 -0
  48. data/lib/polyrun/data/parallel_provisioning.rb +93 -0
  49. data/lib/polyrun/data/snapshot.rb +84 -0
  50. data/lib/polyrun/database/clone_shards.rb +81 -0
  51. data/lib/polyrun/database/provision.rb +72 -0
  52. data/lib/polyrun/database/shard.rb +63 -0
  53. data/lib/polyrun/database/url_builder/connection/infer.rb +49 -0
  54. data/lib/polyrun/database/url_builder/connection/url_builders.rb +43 -0
  55. data/lib/polyrun/database/url_builder/connection.rb +191 -0
  56. data/lib/polyrun/database/url_builder/template_prepare.rb +21 -0
  57. data/lib/polyrun/database/url_builder.rb +160 -0
  58. data/lib/polyrun/debug.rb +81 -0
  59. data/lib/polyrun/env/ci.rb +65 -0
  60. data/lib/polyrun/log.rb +70 -0
  61. data/lib/polyrun/minitest.rb +17 -0
  62. data/lib/polyrun/partition/constraints.rb +69 -0
  63. data/lib/polyrun/partition/hrw.rb +33 -0
  64. data/lib/polyrun/partition/min_heap.rb +64 -0
  65. data/lib/polyrun/partition/paths.rb +28 -0
  66. data/lib/polyrun/partition/paths_build.rb +128 -0
  67. data/lib/polyrun/partition/plan.rb +189 -0
  68. data/lib/polyrun/partition/plan_lpt.rb +49 -0
  69. data/lib/polyrun/partition/plan_sharding.rb +48 -0
  70. data/lib/polyrun/partition/stable_shuffle.rb +18 -0
  71. data/lib/polyrun/prepare/artifacts.rb +40 -0
  72. data/lib/polyrun/prepare/assets.rb +57 -0
  73. data/lib/polyrun/queue/file_store.rb +199 -0
  74. data/lib/polyrun/queue/file_store_pending.rb +48 -0
  75. data/lib/polyrun/quick/assertions.rb +32 -0
  76. data/lib/polyrun/quick/errors.rb +6 -0
  77. data/lib/polyrun/quick/example_group.rb +66 -0
  78. data/lib/polyrun/quick/example_runner.rb +93 -0
  79. data/lib/polyrun/quick/matchers.rb +156 -0
  80. data/lib/polyrun/quick/reporter.rb +42 -0
  81. data/lib/polyrun/quick/runner.rb +180 -0
  82. data/lib/polyrun/quick.rb +1 -0
  83. data/lib/polyrun/railtie.rb +7 -0
  84. data/lib/polyrun/reporting/junit.rb +125 -0
  85. data/lib/polyrun/reporting/junit_emit.rb +58 -0
  86. data/lib/polyrun/reporting/rspec_junit.rb +39 -0
  87. data/lib/polyrun/rspec.rb +15 -0
  88. data/lib/polyrun/templates/POLYRUN.md +45 -0
  89. data/lib/polyrun/templates/ci_matrix.polyrun.yml +14 -0
  90. data/lib/polyrun/templates/minimal_gem.polyrun.yml +13 -0
  91. data/lib/polyrun/templates/rails_prepare.polyrun.yml +31 -0
  92. data/lib/polyrun/timing/merge.rb +35 -0
  93. data/lib/polyrun/timing/summary.rb +25 -0
  94. data/lib/polyrun/version.rb +3 -0
  95. data/lib/polyrun.rb +58 -0
  96. data/polyrun.gemspec +37 -0
  97. data/sig/polyrun/cli.rbs +6 -0
  98. data/sig/polyrun/config.rbs +20 -0
  99. data/sig/polyrun/debug.rbs +12 -0
  100. data/sig/polyrun/log.rbs +12 -0
  101. data/sig/polyrun/minitest.rbs +5 -0
  102. data/sig/polyrun/quick.rbs +19 -0
  103. data/sig/polyrun/rspec.rbs +5 -0
  104. data/sig/polyrun.rbs +11 -0
  105. metadata +288 -0
@@ -0,0 +1,124 @@
1
+ require "json"
2
+ require "fileutils"
3
+ require "optparse"
4
+
5
+ module Polyrun
6
+ class CLI
7
+ # Writes merged coverage blob to requested formats (merge-coverage / run-shards).
8
+ module CoverageMergeIo
9
+ private
10
+
11
+ def merge_coverage_parse_argv(argv)
12
+ inputs = []
13
+ output = "coverage/polyrun-merged.json"
14
+ formats = ["json"]
15
+ parser = OptionParser.new do |opts|
16
+ opts.banner = "usage: polyrun merge-coverage -i FILE [-i FILE] [-o PATH] [--format json,lcov,cobertura,console,html]"
17
+ opts.on("-i", "--input FILE", "Coverage JSON (repeatable; globs ok)") do |f|
18
+ expand_merge_input_pattern(f).each { |x| inputs << x }
19
+ end
20
+ opts.on("-o", "--output PATH", String) { |v| output = v }
21
+ opts.on("--format LIST", String) { |v| formats = v.split(",").map(&:strip) }
22
+ end
23
+ parser.parse!(argv)
24
+ inputs.uniq!
25
+ inputs.select! { |p| File.file?(p) }
26
+ [inputs, output, formats]
27
+ end
28
+
29
+ def merge_coverage_merge_fragments(inputs)
30
+ Polyrun::Debug.time("Coverage::Merge.merge_fragments") do
31
+ Polyrun::Coverage::Merge.merge_fragments(inputs)
32
+ end
33
+ end
34
+
35
+ def merge_coverage_write_json_payload(out_abs, payload)
36
+ Polyrun::Debug.time("write merged JSON") do
37
+ FileUtils.mkdir_p(File.dirname(out_abs))
38
+ File.write(out_abs, JSON.generate(payload))
39
+ end
40
+ end
41
+
42
+ def merge_coverage_write_format_outputs(merged, r, out_abs, formats)
43
+ write_merge_lcov(merged, out_abs) if formats.include?("lcov")
44
+ write_merge_cobertura(merged, r, out_abs) if formats.include?("cobertura")
45
+ write_merge_console(merged, out_abs) if formats.include?("console")
46
+ write_merge_html(merged, out_abs) if formats.include?("html")
47
+ end
48
+
49
+ def write_merge_lcov(merged, out_abs)
50
+ lcov_path = out_abs.sub(/\.json\z/, ".lcov")
51
+ lcov_path = "#{out_abs}.lcov" if lcov_path == out_abs
52
+ Polyrun::Debug.time("write lcov") { File.write(lcov_path, Polyrun::Coverage::Merge.emit_lcov(merged)) }
53
+ end
54
+
55
+ def write_merge_cobertura(merged, r, out_abs)
56
+ cob_path = out_abs.sub(/\.json\z/, ".xml")
57
+ cob_path = "#{out_abs}.cobertura.xml" if cob_path == out_abs
58
+ root = merge_cobertura_root(r)
59
+ Polyrun::Debug.time("write cobertura XML") do
60
+ File.write(cob_path, Polyrun::Coverage::Merge.emit_cobertura(merged, root: root))
61
+ end
62
+ end
63
+
64
+ def merge_cobertura_root(r)
65
+ return nil unless r[:meta].is_a?(Hash)
66
+
67
+ r[:meta]["polyrun_coverage_root"] || r[:meta][:polyrun_coverage_root]
68
+ end
69
+
70
+ def write_merge_console(merged, out_abs)
71
+ sum_path = out_abs.sub(/\.json\z/, "-summary.txt")
72
+ sum_path = "#{out_abs}-summary.txt" if sum_path == out_abs
73
+ summary = Polyrun::Coverage::Merge.console_summary(merged)
74
+ summary_text = Polyrun::Coverage::Merge.format_console_summary(summary)
75
+ Polyrun::Debug.time("write console summary") { File.write(sum_path, summary_text) }
76
+ Polyrun::Log.print summary_text
77
+ end
78
+
79
+ def write_merge_html(merged, out_abs)
80
+ html_path = out_abs.sub(/\.json\z/, ".html")
81
+ html_path = "#{out_abs}.html" if html_path == out_abs
82
+ Polyrun::Debug.time("write HTML report") { File.write(html_path, Polyrun::Coverage::Merge.emit_html(merged)) }
83
+ end
84
+
85
+ def merge_coverage_log_finish(elapsed, inputs)
86
+ thr = merge_slow_warn_threshold_seconds
87
+ merge_coverage_warn_if_slow(elapsed, thr, inputs) if thr && elapsed > thr
88
+ return unless @verbose || ENV["POLYRUN_PROFILE_MERGE"] == "1" || Polyrun::Debug.enabled?
89
+
90
+ Polyrun::Log.warn format("merge-coverage: finished in %.2fs (%d input fragment(s))", elapsed, inputs.size)
91
+ end
92
+
93
+ def merge_coverage_warn_if_slow(elapsed, thr, inputs)
94
+ Polyrun::Log.warn format(
95
+ "merge-coverage: slow merge took %.2fs (warn above %.0fs; typical suites are JSON fragments, not TB-scale data; disable: POLYRUN_MERGE_SLOW_WARN_SECONDS=0)",
96
+ elapsed,
97
+ thr
98
+ )
99
+ end
100
+
101
+ def merge_coverage_min_line_gate_below?(merged_path, gate)
102
+ Polyrun::Debug.time("minimum_line_percent gate (merged JSON)") do
103
+ data = JSON.parse(File.read(merged_path))
104
+ blob = Polyrun::Coverage::Merge.extract_coverage_blob(data)
105
+ summary = Polyrun::Coverage::Merge.console_summary(blob)
106
+ min = gate[:minimum]
107
+ below = summary[:line_percent].round < min.round
108
+ Polyrun::Debug.log_kv(
109
+ merged_line_percent: summary[:line_percent],
110
+ gate_minimum: min,
111
+ below_gate: below
112
+ )
113
+ log_gate_below_minimum(summary, min) if below
114
+ below
115
+ end
116
+ end
117
+
118
+ def log_gate_below_minimum(summary, min)
119
+ Polyrun::Log.warn Polyrun::Coverage::Merge.format_console_summary(summary)
120
+ Polyrun::Log.warn "Polyrun coverage: #{summary[:line_percent].round(2)}% rounds below minimum #{min}% (merged)."
121
+ end
122
+ end
123
+ end
124
+ end
@@ -0,0 +1,149 @@
1
+ require "optparse"
2
+
3
+ module Polyrun
4
+ class CLI
5
+ module DatabaseCommands
6
+ private
7
+
8
+ def cmd_db_setup_template(argv, config_path)
9
+ dry = false
10
+ rails_root = Dir.pwd
11
+ parser = OptionParser.new do |opts|
12
+ opts.on("--dry-run", "Print only") { dry = true }
13
+ opts.on("--rails-root PATH", String) { |v| rails_root = v }
14
+ end
15
+ parser.parse!(argv)
16
+
17
+ cfg = Polyrun::Config.load(path: config_path || ENV["POLYRUN_CONFIG"])
18
+ dh = cfg.databases
19
+ if !dh.is_a?(Hash) || dh.empty?
20
+ Polyrun::Log.warn "db:setup-template: configure databases: in polyrun.yml"
21
+ return 2
22
+ end
23
+
24
+ begin
25
+ te = Polyrun::Database::UrlBuilder.template_prepare_env(dh)
26
+ rescue Polyrun::Error => e
27
+ Polyrun::Log.warn "db:setup-template: #{e.message}"
28
+ return 2
29
+ end
30
+
31
+ if dry
32
+ log = Polyrun::Database::UrlBuilder.template_prepare_env_shell_log(dh)
33
+ Polyrun::Log.warn "would: RAILS_ENV=test #{log} bin/rails db:prepare"
34
+ return 0
35
+ end
36
+
37
+ child_env = ENV.to_h.merge(te)
38
+ child_env["RAILS_ENV"] ||= ENV["RAILS_ENV"] || "test"
39
+ Polyrun::Database::Provision.prepare_template!(
40
+ rails_root: File.expand_path(rails_root),
41
+ env: child_env,
42
+ silent: !@verbose
43
+ )
44
+ 0
45
+ end
46
+
47
+ def cmd_db_setup_shard(argv, config_path)
48
+ dry = db_setup_shard_parse_options!(argv)
49
+ cfg = Polyrun::Config.load(path: config_path || ENV["POLYRUN_CONFIG"])
50
+ dh = cfg.databases
51
+ if !dh.is_a?(Hash) || dh.empty?
52
+ Polyrun::Log.warn "db:setup-shard: configure databases: in polyrun.yml"
53
+ return 2
54
+ end
55
+
56
+ shard = resolve_shard_index(cfg.partition)
57
+ template = dh["template_db"] || dh[:template_db]
58
+ if !template
59
+ Polyrun::Log.warn "db:setup-shard: set databases.template_db"
60
+ return 2
61
+ end
62
+
63
+ plan = Polyrun::Database::UrlBuilder.shard_database_plan(dh, shard_index: shard)
64
+ if plan.empty?
65
+ Polyrun::Log.warn "db:setup-shard: could not derive shard database names from polyrun.yml"
66
+ return 2
67
+ end
68
+
69
+ db_setup_shard_run_plan(plan, dry: dry)
70
+ end
71
+
72
+ def db_setup_shard_parse_options!(argv)
73
+ dry = false
74
+ OptionParser.new do |opts|
75
+ opts.on("--dry-run", "Print only") { dry = true }
76
+ end.parse!(argv)
77
+ dry
78
+ end
79
+
80
+ def db_setup_shard_run_plan(plan, dry:)
81
+ if dry
82
+ plan.each do |row|
83
+ Polyrun::Log.warn "would: CREATE DATABASE #{row[:new_db]} TEMPLATE #{row[:template_db]}"
84
+ end
85
+ return 0
86
+ end
87
+
88
+ threads = plan.map do |row|
89
+ Thread.new do
90
+ Polyrun::Database::Provision.create_database_from_template!(
91
+ new_db: row[:new_db],
92
+ template_db: row[:template_db].to_s
93
+ )
94
+ end
95
+ end
96
+ threads.each(&:join)
97
+ 0
98
+ end
99
+
100
+ # Migrate all template DBs + create every shard database (primary + +connections+).
101
+ def cmd_db_clone_shards(argv, config_path)
102
+ opts = db_clone_shards_parse_options!(argv)
103
+ cfg = Polyrun::Config.load(path: config_path || ENV["POLYRUN_CONFIG"])
104
+ dh = cfg.databases
105
+ if !dh.is_a?(Hash) || dh.empty?
106
+ Polyrun::Log.warn "db:clone-shards: configure databases: in polyrun.yml"
107
+ return 2
108
+ end
109
+
110
+ workers = opts[:workers].clamp(1, 10)
111
+ Polyrun::Database::CloneShards.provision!(
112
+ dh,
113
+ workers: workers,
114
+ rails_root: File.expand_path(opts[:rails_root]),
115
+ migrate: opts[:migrate],
116
+ replace: opts[:replace],
117
+ force_drop: opts[:force_drop],
118
+ dry_run: opts[:dry],
119
+ silent: !@verbose
120
+ )
121
+ 0
122
+ rescue Polyrun::Error => e
123
+ Polyrun::Log.warn "db:clone-shards: #{e.message}"
124
+ 1
125
+ end
126
+
127
+ def db_clone_shards_parse_options!(argv)
128
+ dry = false
129
+ migrate = true
130
+ replace = true
131
+ force_drop = false
132
+ rails_root = Dir.pwd
133
+ workers = env_int("POLYRUN_WORKERS", 5)
134
+
135
+ OptionParser.new do |opts|
136
+ opts.banner = "usage: polyrun db:clone-shards [--workers N] [--rails-root PATH] [--dry-run] [--no-migrate] [--no-replace] [--force-drop]"
137
+ opts.on("--workers N", Integer) { |v| workers = v }
138
+ opts.on("--dry-run", "Print only") { dry = true }
139
+ opts.on("--rails-root PATH", String) { |v| rails_root = v }
140
+ opts.on("--no-migrate", "Skip db:prepare on template databases") { migrate = false }
141
+ opts.on("--no-replace", "Skip DROP DATABASE before CREATE (fail if shard DB exists)") { replace = false }
142
+ opts.on("--force-drop", "DROP DATABASE … WITH (FORCE) (PostgreSQL 13+)") { force_drop = true }
143
+ end.parse!(argv)
144
+
145
+ {dry: dry, migrate: migrate, replace: replace, force_drop: force_drop, rails_root: rails_root, workers: workers}
146
+ end
147
+ end
148
+ end
149
+ end
@@ -0,0 +1,43 @@
1
+ require "optparse"
2
+
3
+ module Polyrun
4
+ class CLI
5
+ module EnvCommands
6
+ private
7
+
8
+ def cmd_env(argv, config_path)
9
+ shard, total, base_database = env_parse_options!(argv)
10
+ cfg = Polyrun::Config.load(path: config_path || ENV["POLYRUN_CONFIG"])
11
+ pc = cfg.partition
12
+ shard = shard.nil? ? resolve_shard_index(pc) : shard
13
+ total = total.nil? ? resolve_shard_total(pc) : total
14
+
15
+ env_print_database_exports(cfg.databases, shard)
16
+ Polyrun::Database::Shard.env_map(shard_index: shard, shard_total: total, base_database: base_database).each do |k, v|
17
+ Polyrun::Log.puts %(export #{k}=#{v})
18
+ end
19
+ 0
20
+ end
21
+
22
+ def env_parse_options!(argv)
23
+ shard = nil
24
+ total = nil
25
+ base_database = nil
26
+ OptionParser.new do |opts|
27
+ opts.on("--shard INDEX", Integer) { |v| shard = v }
28
+ opts.on("--total N", Integer) { |v| total = v }
29
+ opts.on("--database TEMPLATE", String) { |v| base_database = v }
30
+ end.parse!(argv)
31
+ [shard, total, base_database]
32
+ end
33
+
34
+ def env_print_database_exports(dh, shard)
35
+ return unless dh.is_a?(Hash) && !dh.empty?
36
+
37
+ Polyrun::Database::UrlBuilder.env_exports_for_databases(dh, shard_index: shard).each do |k, v|
38
+ Polyrun::Log.puts %(export #{k}=#{v})
39
+ end
40
+ end
41
+ end
42
+ end
43
+ end
@@ -0,0 +1,113 @@
1
+ require "yaml"
2
+
3
+ module Polyrun
4
+ class CLI
5
+ module Helpers
6
+ private
7
+
8
+ def partition_int(pc, keys, default)
9
+ keys.each do |k|
10
+ v = pc[k] || pc[k.to_sym]
11
+ next if v.nil? || v.to_s.empty?
12
+
13
+ i = Integer(v, exception: false)
14
+ return i unless i.nil?
15
+ end
16
+ default
17
+ end
18
+
19
+ def env_int(name, fallback)
20
+ s = ENV[name]
21
+ return fallback if s.nil? || s.empty?
22
+
23
+ Integer(s, exception: false) || fallback
24
+ end
25
+
26
+ def resolve_shard_index(pc)
27
+ return Integer(ENV["POLYRUN_SHARD_INDEX"]) if ENV["POLYRUN_SHARD_INDEX"] && !ENV["POLYRUN_SHARD_INDEX"].empty?
28
+
29
+ ci = Polyrun::Env::Ci.detect_shard_index
30
+ return ci unless ci.nil?
31
+
32
+ partition_int(pc, %w[shard_index shard], 0)
33
+ end
34
+
35
+ def resolve_shard_total(pc)
36
+ return Integer(ENV["POLYRUN_SHARD_TOTAL"]) if ENV["POLYRUN_SHARD_TOTAL"] && !ENV["POLYRUN_SHARD_TOTAL"].empty?
37
+
38
+ ci = Polyrun::Env::Ci.detect_shard_total
39
+ return ci unless ci.nil?
40
+
41
+ partition_int(pc, %w[shard_total total], 1)
42
+ end
43
+
44
+ def expand_merge_input_pattern(path)
45
+ p = path.to_s
46
+ abs = File.expand_path(p, Dir.pwd)
47
+ return Dir.glob(abs).sort if p.include?("*") || p.include?("?")
48
+
49
+ [abs]
50
+ end
51
+
52
+ # Same rounding/strict semantics as {Polyrun::Coverage::Collector} for +config/polyrun_coverage.yml+.
53
+ def coverage_minimum_line_gate_from_polyrun_coverage_yml
54
+ path = File.join(Dir.pwd, "config", "polyrun_coverage.yml")
55
+ return nil unless File.file?(path)
56
+
57
+ data = YAML.safe_load_file(path, permitted_classes: [Symbol], aliases: true)
58
+ return nil unless data.is_a?(Hash)
59
+
60
+ min = data["minimum_line_percent"] || data[:minimum_line_percent]
61
+ return nil if min.nil?
62
+
63
+ sv = data["strict"] if data.key?("strict")
64
+ sv = data[:strict] if !data.key?("strict") && data.key?(:strict)
65
+ strict = sv.nil? || sv
66
+
67
+ {minimum: min.to_f, strict: strict != false}
68
+ rescue Psych::SyntaxError, ArgumentError, TypeError
69
+ nil
70
+ end
71
+
72
+ def load_partition_constraints(pc, constraints_path)
73
+ if constraints_path
74
+ path = File.expand_path(constraints_path.to_s, Dir.pwd)
75
+ unless File.file?(path)
76
+ Polyrun::Log.warn "polyrun: constraints file not found: #{path}"
77
+ return nil
78
+ end
79
+ h = YAML.safe_load_file(path, permitted_classes: [Symbol], aliases: true)
80
+ return Polyrun::Partition::Constraints.from_hash(h, root: Dir.pwd)
81
+ end
82
+ if pc["constraints"].is_a?(Hash)
83
+ return Polyrun::Partition::Constraints.from_hash(pc["constraints"], root: Dir.pwd)
84
+ end
85
+ cf = pc["constraints_file"] || pc[:constraints_file]
86
+ if cf
87
+ path = File.expand_path(cf.to_s, Dir.pwd)
88
+ return nil unless File.file?(path)
89
+
90
+ h = YAML.safe_load_file(path, permitted_classes: [Symbol], aliases: true)
91
+ return Polyrun::Partition::Constraints.from_hash(h, root: Dir.pwd)
92
+ end
93
+ nil
94
+ end
95
+
96
+ # +default_weight+ should be precomputed when sorting many paths (e.g. +queue init+), matching
97
+ # {Partition::Plan#default_weight} semantics: mean of known timing costs for missing paths.
98
+ def queue_weight_for(path, costs, default_weight = nil)
99
+ abs = File.expand_path(path.to_s, Dir.pwd)
100
+ return costs[abs] if costs.key?(abs)
101
+
102
+ unless default_weight.nil?
103
+ return default_weight
104
+ end
105
+
106
+ vals = costs.values
107
+ return 1.0 if vals.empty?
108
+
109
+ vals.sum / vals.size.to_f
110
+ end
111
+ end
112
+ end
113
+ end
@@ -0,0 +1,99 @@
1
+ require "optparse"
2
+
3
+ module Polyrun
4
+ class CLI
5
+ module InitCommand
6
+ INIT_PROFILES = {
7
+ "gem" => "minimal_gem.polyrun.yml",
8
+ "rails" => "rails_prepare.polyrun.yml",
9
+ "ci-matrix" => "ci_matrix.polyrun.yml",
10
+ "doc" => "POLYRUN.md"
11
+ }.freeze
12
+
13
+ private
14
+
15
+ def templates_dir
16
+ File.expand_path("../templates", __dir__)
17
+ end
18
+
19
+ def cmd_init(argv, _config_path)
20
+ profile, output, dry_run, force, list = init_parse_options!(argv)
21
+ return init_list_profiles if list
22
+ return init_missing_profile if profile.nil?
23
+
24
+ filename = INIT_PROFILES[profile]
25
+ return init_unknown_profile(profile) unless filename
26
+
27
+ src = File.join(templates_dir, filename)
28
+ return init_missing_template(src) unless File.file?(src)
29
+
30
+ body = File.read(src, encoding: Encoding::UTF_8)
31
+ dest = output || default_init_output(profile)
32
+ return init_dry_run_print(body) if dry_run
33
+
34
+ path = File.expand_path(dest)
35
+ return init_refuses_overwrite(path) if File.file?(path) && !force
36
+
37
+ File.write(path, body)
38
+ Polyrun::Log.warn "polyrun init: wrote #{path}"
39
+ 0
40
+ end
41
+
42
+ def init_parse_options!(argv)
43
+ profile = nil
44
+ output = nil
45
+ dry_run = false
46
+ force = false
47
+ list = false
48
+
49
+ OptionParser.new do |opts|
50
+ opts.banner = "usage: polyrun init [--profile NAME] [--output PATH] [--dry-run] [--force]\n polyrun init --list"
51
+ opts.on("--profile NAME", INIT_PROFILES.keys.join(", ")) { |p| profile = p }
52
+ opts.on("-o", "--output PATH", "destination file (default: polyrun.yml or POLYRUN.md for --profile doc)") { |p| output = p }
53
+ opts.on("--dry-run", "print template to stdout; do not write") { dry_run = true }
54
+ opts.on("--force", "overwrite existing output file") { force = true }
55
+ opts.on("--list", "print available profiles") { list = true }
56
+ end.parse!(argv)
57
+
58
+ [profile, output, dry_run, force, list]
59
+ end
60
+
61
+ def init_list_profiles
62
+ Polyrun::Log.puts "polyrun init profiles:"
63
+ INIT_PROFILES.each do |name, file|
64
+ Polyrun::Log.puts " #{name.ljust(12)} #{file}"
65
+ end
66
+ 0
67
+ end
68
+
69
+ def init_missing_profile
70
+ Polyrun::Log.warn "polyrun init: specify --profile (#{INIT_PROFILES.keys.join(", ")}) or --list"
71
+ 2
72
+ end
73
+
74
+ def init_unknown_profile(profile)
75
+ Polyrun::Log.warn "polyrun init: unknown profile #{profile.inspect}"
76
+ 2
77
+ end
78
+
79
+ def init_missing_template(src)
80
+ Polyrun::Log.warn "polyrun init: template missing: #{src}"
81
+ 1
82
+ end
83
+
84
+ def init_dry_run_print(body)
85
+ Polyrun::Log.print body
86
+ 0
87
+ end
88
+
89
+ def init_refuses_overwrite(path)
90
+ Polyrun::Log.warn "polyrun init: #{path} exists (use --force to overwrite)"
91
+ 1
92
+ end
93
+
94
+ def default_init_output(profile)
95
+ (profile == "doc") ? "POLYRUN.md" : "polyrun.yml"
96
+ end
97
+ end
98
+ end
99
+ end
@@ -0,0 +1,134 @@
1
+ require "json"
2
+ require "optparse"
3
+
4
+ module Polyrun
5
+ class CLI
6
+ module PlanCommand
7
+ private
8
+
9
+ def cmd_plan(argv, config_path)
10
+ cfg = Polyrun::Config.load(path: config_path || ENV["POLYRUN_CONFIG"])
11
+ pc = cfg.partition
12
+ ctx = plan_command_initial_context(pc)
13
+ plan_command_parse_argv!(argv, ctx)
14
+
15
+ paths_file = ctx[:paths_file] || (pc["paths_file"] || pc[:paths_file])
16
+ code = Polyrun::Partition::PathsBuild.apply!(partition: pc, cwd: Dir.pwd)
17
+ return code if code != 0
18
+
19
+ timing_path = plan_resolve_timing_path(pc, ctx[:timing_path], ctx[:strategy])
20
+ Polyrun::Log.warn "polyrun plan: using #{cfg.path}" if @verbose && cfg.path
21
+
22
+ items = plan_plan_items(paths_file, argv)
23
+ return 2 if items.nil?
24
+
25
+ loaded = plan_load_costs_and_strategy(timing_path, ctx[:strategy])
26
+ return 2 if loaded.nil?
27
+
28
+ costs, strategy = loaded
29
+
30
+ constraints = load_partition_constraints(pc, ctx[:constraints_path])
31
+
32
+ plan_command_emit_manifest(
33
+ items: items,
34
+ total: ctx[:total],
35
+ strategy: strategy,
36
+ seed: ctx[:seed],
37
+ costs: costs,
38
+ constraints: constraints,
39
+ shard: ctx[:shard]
40
+ )
41
+ end
42
+
43
+ def plan_command_initial_context(pc)
44
+ {
45
+ shard: resolve_shard_index(pc),
46
+ total: resolve_shard_total(pc),
47
+ strategy: (pc["strategy"] || pc[:strategy] || "round_robin").to_s,
48
+ seed: pc["seed"] || pc[:seed],
49
+ paths_file: nil,
50
+ timing_path: nil,
51
+ constraints_path: nil
52
+ }
53
+ end
54
+
55
+ def plan_command_parse_argv!(argv, ctx)
56
+ OptionParser.new do |opts|
57
+ opts.banner = "usage: polyrun plan [options] [--] [paths...]"
58
+ opts.on("--shard INDEX", Integer) { |v| ctx[:shard] = v }
59
+ opts.on("--total N", Integer) { |v| ctx[:total] = v }
60
+ opts.on("--strategy NAME", String) { |v| ctx[:strategy] = v }
61
+ opts.on("--seed VAL") { |v| ctx[:seed] = v }
62
+ opts.on("--paths-file PATH", String) { |v| ctx[:paths_file] = v }
63
+ opts.on("--constraints PATH", "YAML: pin / serial_glob (see spec_queue.md)") { |v| ctx[:constraints_path] = v }
64
+ opts.on("--timing PATH", "path => seconds JSON; implies cost_binpack unless strategy is cost-based or hrw") do |v|
65
+ ctx[:timing_path] = v
66
+ end
67
+ end.parse!(argv)
68
+ end
69
+
70
+ def plan_command_emit_manifest(items:, total:, strategy:, seed:, costs:, constraints:, shard:)
71
+ plan = Polyrun::Debug.time("Partition::Plan.new (plan command)") do
72
+ Polyrun::Partition::Plan.new(
73
+ items: items,
74
+ total_shards: total,
75
+ strategy: strategy,
76
+ seed: seed,
77
+ costs: costs,
78
+ constraints: constraints,
79
+ root: Dir.pwd
80
+ )
81
+ end
82
+ Polyrun::Debug.log_kv(
83
+ plan: "emit manifest JSON",
84
+ shard: shard,
85
+ total: total,
86
+ strategy: strategy,
87
+ path_count: items.size
88
+ )
89
+ Polyrun::Log.puts JSON.generate(plan.manifest(shard))
90
+ 0
91
+ end
92
+
93
+ def plan_resolve_timing_path(pc, timing_path, strategy)
94
+ return timing_path if timing_path
95
+
96
+ tf = pc["timing_file"] || pc[:timing_file]
97
+ return tf if tf && (Polyrun::Partition::Plan.cost_strategy?(strategy) || Polyrun::Partition::Plan.hrw_strategy?(strategy))
98
+
99
+ nil
100
+ end
101
+
102
+ def plan_plan_items(paths_file, argv)
103
+ if paths_file
104
+ Polyrun::Partition::Paths.read_lines(paths_file)
105
+ elsif argv.empty?
106
+ Polyrun::Log.warn "polyrun plan: provide spec paths, --paths-file, or partition.paths_file in polyrun.yml"
107
+ nil
108
+ else
109
+ argv
110
+ end
111
+ end
112
+
113
+ def plan_load_costs_and_strategy(timing_path, strategy)
114
+ if timing_path
115
+ costs = Polyrun::Partition::Plan.load_timing_costs(File.expand_path(timing_path.to_s, Dir.pwd))
116
+ if costs.empty?
117
+ Polyrun::Log.warn "polyrun plan: timing file missing or has no entries: #{timing_path}"
118
+ return nil
119
+ end
120
+ unless Polyrun::Partition::Plan.cost_strategy?(strategy) || Polyrun::Partition::Plan.hrw_strategy?(strategy)
121
+ Polyrun::Log.warn "polyrun plan: using cost_binpack (timing data present)" if @verbose
122
+ strategy = "cost_binpack"
123
+ end
124
+ [costs, strategy]
125
+ elsif Polyrun::Partition::Plan.cost_strategy?(strategy)
126
+ Polyrun::Log.warn "polyrun plan: --timing or partition.timing_file required for strategy #{strategy}"
127
+ nil
128
+ else
129
+ [nil, strategy]
130
+ end
131
+ end
132
+ end
133
+ end
134
+ end