polyrun 2.2.5 → 2.2.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +22 -0
  3. data/README.md +1 -0
  4. data/docs/SETUP_PROFILE.md +1 -0
  5. data/lib/polyrun/cli/database_commands.rb +21 -7
  6. data/lib/polyrun/cli/default_run.rb +13 -3
  7. data/lib/polyrun/cli/helpers.rb +41 -6
  8. data/lib/polyrun/cli/plan_command.rb +10 -29
  9. data/lib/polyrun/cli/run_queue_command.rb +10 -1
  10. data/lib/polyrun/cli/run_shards_parallel_children.rb +52 -39
  11. data/lib/polyrun/cli/run_shards_parallel_wait.rb +2 -2
  12. data/lib/polyrun/cli/run_shards_planning.rb +7 -29
  13. data/lib/polyrun/cli/run_shards_run.rb +1 -1
  14. data/lib/polyrun/cli/run_shards_worker_interrupt.rb +16 -14
  15. data/lib/polyrun/cli/start_bootstrap.rb +51 -22
  16. data/lib/polyrun/cli.rb +3 -0
  17. data/lib/polyrun/config.rb +9 -4
  18. data/lib/polyrun/coverage/example_diff.rb +6 -7
  19. data/lib/polyrun/coverage/merge/formatters_html.rb +20 -2
  20. data/lib/polyrun/coverage/merge.rb +9 -3
  21. data/lib/polyrun/coverage/merge_fragment_meta.rb +6 -1
  22. data/lib/polyrun/coverage/rails.rb +1 -1
  23. data/lib/polyrun/database/clone_shards.rb +28 -5
  24. data/lib/polyrun/database/provision.rb +15 -9
  25. data/lib/polyrun/database/url_builder/connection/infer.rb +4 -0
  26. data/lib/polyrun/database/url_builder.rb +31 -3
  27. data/lib/polyrun/partition/plan.rb +6 -24
  28. data/lib/polyrun/partition/timing_diagnostics.rb +2 -3
  29. data/lib/polyrun/queue/file_store.rb +133 -30
  30. data/lib/polyrun/queue/file_store_intent.rb +188 -0
  31. data/lib/polyrun/queue/worker_loop.rb +41 -7
  32. data/lib/polyrun/spec_quality/config.rb +1 -1
  33. data/lib/polyrun/spec_quality/profile.rb +0 -1
  34. data/lib/polyrun/spec_quality/report.rb +16 -7
  35. data/lib/polyrun/templates/rails_prepare.polyrun.yml +3 -1
  36. data/lib/polyrun/version.rb +1 -1
  37. data/lib/polyrun/worker_output.rb +30 -13
  38. metadata +2 -1
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 6364a918386209ff30b2c12280f3a37a0cb70de54a83f1f07606c2c526ac773c
4
- data.tar.gz: 8d75d162af33d721f864cf8a4002d98f57e53d24db55dc607dad82e94f5416f7
3
+ metadata.gz: 69dd2432a149cc9790756d2d54b6481e898604a7ae84219b8b5cfbd65997e231
4
+ data.tar.gz: ab1ea9013d606dd53e717cc421e353c668b47382871e3494f03fa689333e422c
5
5
  SHA512:
6
- metadata.gz: e92439850018abacae0f1041315b213f4624abc78e49b3d4ca53d53129428790961b1d77b8a923d882e0fe0c9157c14c125ab13bbdf34db52c86c82082917104
7
- data.tar.gz: 6d64c292b054ffd8cf95a47bda1fa3253284af76e98d8d9fc32631b389e51e9acbf2b3ad6c0e64e19f471847fba17b00b706ef1d4d8c549414abea9be4a9806a
6
+ metadata.gz: 70c45810383e893e6d4ed8a9b2288f3540bb85e37345149283cd9992e2f23c5fa1d475b9e43766d20ddc6f93eaae562c258ff99769284bf606aa2858bbaae515
7
+ data.tar.gz: 17a14844465eade2840dcc968d1c432e4462991ac099831cd03cd19f262aec9b42d5bd129efcc8e0b843534357ae281380507efb7c40f530e772a7bcb78d9f40
data/CHANGELOG.md CHANGED
@@ -1,5 +1,27 @@
1
1
  # CHANGELOG
2
2
 
3
+ ## Unreleased
4
+
5
+ ## 2.2.6 (2026-10-29)
6
+
7
+ - Fix `polyrun start` auto database provision when `databases:` uses a non-Postgres adapter (`mysql2`, trilogy, …): skip CloneShards with a clear warning (or exit 1 if `start.databases: true`); keep `template_db` / `shard_db_pattern` for URL shaping.
8
+ - Fail `db:clone-shards` / `db:setup-shard` before `psql` when the inferred adapter is not PostgreSQL.
9
+ - Pass `postgresql:` host, port, username, and password from `polyrun.yml` into `psql` during shard provision (password via child `PGPASSWORD`).
10
+ - Fix implicit path runs so tokens after `--` stay the worker command, and `--paths-file` is passed before `--`.
11
+ - Accept `polyrun start --workers=N` (equals form) so shard DB provision matches worker count.
12
+ - Reject `databases.shard_db_pattern` without `%{shard}` / `%<shard>d`, and refuse patterns that resolve to `template_db`.
13
+ - Redact database passwords in `db:*` / CloneShards dry-run shell logs.
14
+ - Raise when an explicit `-c` / `POLYRUN_CONFIG` path is missing; map `Polyrun::Error` to exit 2.
15
+ - Validate `run-queue --workers` (1..max) and exit forked queue workers with `exit!`.
16
+ - Fix SpecQuality `minimum_unique_lines_per_example` gate for thresholds above 1; keep only positive or CPU profile deltas.
17
+ - Align file-granularity timing cost keys with realpath normalization so cost binpack hits under symlink roots.
18
+ - TERM shard workers before draining routed worker output on SIGINT/SIGTERM; skip empty queue leases.
19
+ - Recover interrupted queue claim/ack/reclaim via `intent.json`; age leases by `heartbeat_at`; quarantine poison paths after `POLYRUN_QUEUE_MAX_ATTEMPTS` (default 3).
20
+ - On a failed `run-queue` batch, re-run paths one-by-one and ack successes so a poison path does not requeue neighbors.
21
+ - Spawn shard workers in their own process group and signal the group on timeout/interrupt; abort already-started workers if a later spawn fails.
22
+ - Cache coverage HTML ERB templates; fold merge fragments without retaining full JSON documents; share plan/run-shards timing cost resolution.
23
+ - Clarify `run-shards` interleaved-output notice: RSpec summary counts apply to individual shards (no false per-worker total).
24
+
3
25
  ## 2.2.5 (2026-09-24)
4
26
 
5
27
  - Fix bare `polyrun` suite selection when both `spec/` and `test/` exist: honor `partition.suite`, else infer from `partition.paths_file`, else filesystem auto-detect (RSpec first).
data/README.md CHANGED
@@ -147,6 +147,7 @@ That single require loads the CLI and core library **without** loading RSpec or
147
147
  | `Polyrun::Prepare::Assets` | Digest trees, marker file, `assets:precompile`. |
148
148
  | `Polyrun::Database::Shard` | Shard env map, `%{shard}` DB names, URL path suffix for `postgres://`, `mysql2://`, `mongodb://`, etc. |
149
149
  | `Polyrun::Database::UrlBuilder` | URLs from `polyrun.yml` `databases:` — nested blocks or `adapter:` for common Rails stacks (`postgresql`, `mysql`/`mysql2`, `trilogy`, `sqlserver`/`mssql`, `sqlite3`/`sqlite`, `mongodb`/`mongo`). |
150
+ | `Polyrun::Database::CloneShards` / `Provision` | PostgreSQL-only template + shard `CREATE DATABASE … TEMPLATE` via `psql`. Non-Postgres configs still use UrlBuilder for env; set `start.databases: false` and provision shards in prepare. |
150
151
  | `Polyrun::Hooks` | Load from `Config#hooks`; `run_phase` / `run_phase_if_enabled`; `build_worker_shell_script` wraps the worker command. |
151
152
  | `Polyrun::Hooks::Dsl` | Ruby hook file (`hooks.ruby`); `before(:suite)` / `after(:each)` etc. in `config/polyrun_hooks.rb` (see README). |
152
153
 
@@ -25,6 +25,7 @@ Do not mix “fan out N workers inside one job” with “matrix shard index”
25
25
  |--------|---------|
26
26
  | Single test DB | Omitting `databases:` in `polyrun.yml` may be enough; ensure parallel workers do not share one DB if they mutate data |
27
27
  | Multi-DB or shard suffixes | `databases:` in `polyrun.yml` (`shard_db_pattern`, `template_db`, optional `connections` and `env_key`). `database.yml` or `DATABASE_URL` may use `%{shard}` or `POLYRUN_SHARD_INDEX` suffixes—the same convention as `polyrun env` |
28
+ | Non-Postgres adapters (mysql2, trilogy, …) | `UrlBuilder` still emits URLs from `template_db` / `shard_db_pattern`. Auto provision on `polyrun start` and `db:clone-shards` / `db:setup-shard` are PostgreSQL-only (`CREATE DATABASE … TEMPLATE`). For MySQL/MariaDB, set `start.databases: false` (or rely on the skip warning) and provision shards in `prepare` |
28
29
  | External provisioning | Sometimes `db:prepare` plus shell clone scripts (multi-DB apps)—document ordering: prepare databases before `run-shards` |
29
30
 
30
31
  ## 4. Prepare (run once before workers)
@@ -47,26 +47,32 @@ module Polyrun
47
47
  def cmd_db_setup_shard(argv, config_path)
48
48
  dry = db_setup_shard_parse_options!(argv)
49
49
  cfg = Polyrun::Config.load(path: config_path || ENV["POLYRUN_CONFIG"])
50
- dh = cfg.databases
51
- if !dh.is_a?(Hash) || dh.empty?
50
+ databases = cfg.databases
51
+ if !databases.is_a?(Hash) || databases.empty?
52
52
  Polyrun::Log.warn "db:setup-shard: configure databases: in polyrun.yml"
53
53
  return 2
54
54
  end
55
55
 
56
+ unless Polyrun::Database::UrlBuilder::ConnectionInfer.postgresql?(databases)
57
+ adapter = Polyrun::Database::UrlBuilder::ConnectionInfer.infer_adapter_name(databases)
58
+ Polyrun::Log.warn "db:setup-shard: PostgreSQL-only (databases adapter is #{adapter})"
59
+ return 2
60
+ end
61
+
56
62
  shard = resolve_shard_index(cfg.partition)
57
- template = dh["template_db"] || dh[:template_db]
63
+ template = databases["template_db"] || databases[:template_db]
58
64
  if !template
59
65
  Polyrun::Log.warn "db:setup-shard: set databases.template_db"
60
66
  return 2
61
67
  end
62
68
 
63
- plan = Polyrun::Database::UrlBuilder.shard_database_plan(dh, shard_index: shard)
69
+ plan = Polyrun::Database::UrlBuilder.shard_database_plan(databases, shard_index: shard)
64
70
  if plan.empty?
65
71
  Polyrun::Log.warn "db:setup-shard: could not derive shard database names from polyrun.yml"
66
72
  return 2
67
73
  end
68
74
 
69
- db_setup_shard_run_plan(plan, dry: dry)
75
+ db_setup_shard_run_plan(plan, databases, dry: dry)
70
76
  end
71
77
 
72
78
  def db_setup_shard_parse_options!(argv)
@@ -77,7 +83,7 @@ module Polyrun
77
83
  dry
78
84
  end
79
85
 
80
- def db_setup_shard_run_plan(plan, dry:)
86
+ def db_setup_shard_run_plan(plan, databases, dry:)
81
87
  if dry
82
88
  plan.each do |row|
83
89
  Polyrun::Log.warn "would: CREATE DATABASE #{row[:new_db]} TEMPLATE #{row[:template_db]}"
@@ -85,11 +91,19 @@ module Polyrun
85
91
  return 0
86
92
  end
87
93
 
94
+ conn = Polyrun::Database::UrlBuilder::Connection.resolve_connection(databases)
95
+ opts = {
96
+ host: conn[:host],
97
+ port: conn[:port],
98
+ username: conn[:user],
99
+ password: conn[:password]
100
+ }
88
101
  threads = plan.map do |row|
89
102
  Thread.new do
90
103
  Polyrun::Database::Provision.create_database_from_template!(
91
104
  new_db: row[:new_db],
92
- template_db: row[:template_db].to_s
105
+ template_db: row[:template_db].to_s,
106
+ **opts
93
107
  )
94
108
  end
95
109
  end
@@ -60,8 +60,7 @@ module Polyrun
60
60
  end
61
61
 
62
62
  def dispatch_implicit_parallel_targets!(argv, config_path)
63
- path_tokens = argv.select { |a| cli_implicit_path_token?(a) }
64
- head = argv.reject { |a| cli_implicit_path_token?(a) }
63
+ path_tokens, head, after = split_implicit_arguments(argv)
65
64
  expanded = expand_implicit_target_paths(path_tokens)
66
65
  if expanded.empty?
67
66
  Polyrun::Log.warn "polyrun: no files matched path arguments"
@@ -82,7 +81,8 @@ module Polyrun
82
81
  begin
83
82
  tmp.write(expanded.join("\n") + "\n")
84
83
  tmp.close
85
- combined = head + ["--paths-file", tmp.path]
84
+ # Keep --paths-file before `--` so OptionParser sees it; after `--` it becomes a worker argv token.
85
+ combined = head + ["--paths-file", tmp.path] + after
86
86
  case suite
87
87
  when :rspec
88
88
  cmd_start(combined, config_path)
@@ -103,6 +103,16 @@ module Polyrun
103
103
  end
104
104
  end
105
105
 
106
+ def split_implicit_arguments(argv)
107
+ # Only tokens before `--` are suite paths; everything from `--` onward is the worker command.
108
+ sep = argv.index("--")
109
+ before = sep ? argv[0...sep] : argv
110
+ after = sep ? argv[sep..] : []
111
+ path_tokens = before.select { |a| cli_implicit_path_token?(a) }
112
+ head = before.reject { |a| cli_implicit_path_token?(a) }
113
+ [path_tokens, head, after]
114
+ end
115
+
106
116
  def expand_implicit_target_paths(path_tokens)
107
117
  path_tokens.flat_map do |p|
108
118
  abs = File.expand_path(p)
@@ -11,6 +11,46 @@ module Polyrun
11
11
  Polyrun::Config::Resolver.env_int(name, fallback)
12
12
  end
13
13
 
14
+ # Shared timing → costs/strategy resolution for +plan+ and +run-shards+.
15
+ # @return [Array] +[costs, strategy, error_code]+ where +error_code+ is nil on success
16
+ def partition_resolve_costs_and_strategy(
17
+ timing_path,
18
+ strategy,
19
+ timing_granularity,
20
+ strategy_explicit: false,
21
+ log_prefix: "polyrun"
22
+ )
23
+ strategy = strategy.to_s
24
+ if timing_path
25
+ if strategy_explicit && strategy == "round_robin"
26
+ return [nil, strategy, nil]
27
+ end
28
+
29
+ costs = Polyrun::Partition::Plan.load_timing_costs(
30
+ File.expand_path(timing_path.to_s, Dir.pwd),
31
+ granularity: timing_granularity
32
+ )
33
+ if costs.empty?
34
+ Polyrun::Log.warn "#{log_prefix}: timing file missing or empty: #{timing_path}"
35
+ return [nil, nil, 2]
36
+ end
37
+ if Polyrun::Partition::Plan.timing_load_strategy?(strategy)
38
+ return [costs, strategy, nil]
39
+ end
40
+ unless strategy_explicit
41
+ Polyrun::Log.warn "#{log_prefix}: using cost_binpack (timing data present)" if @verbose
42
+ return [costs, "cost_binpack", nil]
43
+ end
44
+
45
+ [nil, strategy, nil]
46
+ elsif Polyrun::Partition::Plan.cost_strategy?(strategy) || Polyrun::Partition::Plan.lazy_robin_strategy?(strategy)
47
+ Polyrun::Log.warn "#{log_prefix}: --timing or partition.timing_file required for strategy #{strategy}"
48
+ [nil, nil, 2]
49
+ else
50
+ [nil, strategy, nil]
51
+ end
52
+ end
53
+
14
54
  # Per-worker wall clock (from spawn) for run-shards / ci-shard fan-out; unset or invalid means no limit.
15
55
  def env_worker_timeout_sec
16
56
  s = ENV["POLYRUN_WORKER_TIMEOUT_SEC"].to_s.strip
@@ -97,12 +137,7 @@ module Polyrun
97
137
  # {Partition::Plan#default_weight} semantics: mean of known timing costs for missing paths.
98
138
  def queue_weight_for(path, costs, default_weight = nil, granularity: :file)
99
139
  g = Polyrun::Partition::TimingKeys.normalize_granularity(granularity)
100
- key =
101
- if g == :example
102
- Polyrun::Partition::TimingKeys.normalize_locator(path.to_s, Dir.pwd, :example)
103
- else
104
- File.expand_path(path.to_s, Dir.pwd)
105
- end
140
+ key = Polyrun::Partition::TimingKeys.normalize_locator(path.to_s, Dir.pwd, g)
106
141
  return costs[key] if costs.key?(key)
107
142
 
108
143
  unless default_weight.nil?
@@ -167,35 +167,16 @@ module Polyrun
167
167
  end
168
168
 
169
169
  def plan_load_costs_and_strategy(timing_path, strategy, timing_granularity, strategy_explicit: false)
170
- strategy = strategy.to_s
171
- if timing_path
172
- if strategy_explicit && strategy == "round_robin"
173
- return [nil, strategy]
174
- end
175
-
176
- costs = Polyrun::Partition::Plan.load_timing_costs(
177
- File.expand_path(timing_path.to_s, Dir.pwd),
178
- granularity: timing_granularity
179
- )
180
- if costs.empty?
181
- Polyrun::Log.warn "polyrun plan: timing file missing or has no entries: #{timing_path}"
182
- return nil
183
- end
184
- if Polyrun::Partition::Plan.timing_load_strategy?(strategy)
185
- return [costs, strategy]
186
- end
187
- unless strategy_explicit
188
- Polyrun::Log.warn "polyrun plan: using cost_binpack (timing data present)" if @verbose
189
- return [costs, "cost_binpack"]
190
- end
191
-
192
- [nil, strategy]
193
- elsif Polyrun::Partition::Plan.cost_strategy?(strategy) || Polyrun::Partition::Plan.lazy_robin_strategy?(strategy)
194
- Polyrun::Log.warn "polyrun plan: --timing or partition.timing_file required for strategy #{strategy}"
195
- nil
196
- else
197
- [nil, strategy]
198
- end
170
+ costs, resolved, err = partition_resolve_costs_and_strategy(
171
+ timing_path,
172
+ strategy,
173
+ timing_granularity,
174
+ strategy_explicit: strategy_explicit,
175
+ log_prefix: "polyrun plan"
176
+ )
177
+ return nil if err
178
+
179
+ [costs, resolved]
199
180
  end
200
181
  end
201
182
  end
@@ -43,6 +43,14 @@ module Polyrun
43
43
  end.parse!(head)
44
44
 
45
45
  workers ||= env_int("POLYRUN_WORKERS", Polyrun::Config::DEFAULT_PARALLEL_WORKERS)
46
+ unless workers.is_a?(Integer) && workers.between?(1, Polyrun::Config::MAX_PARALLEL_WORKERS)
47
+ Polyrun::Log.warn "polyrun run-queue: --workers must be between 1 and #{Polyrun::Config::MAX_PARALLEL_WORKERS}"
48
+ return 2
49
+ end
50
+ unless batch.is_a?(Integer) && batch >= 1
51
+ Polyrun::Log.warn "polyrun run-queue: --batch must be >= 1"
52
+ return 2
53
+ end
46
54
  paths_file ||= pc["paths_file"] || pc[:paths_file]
47
55
  unless paths_file
48
56
  Polyrun::Log.warn "polyrun run-queue: need --paths-file or partition.paths_file"
@@ -87,7 +95,8 @@ module Polyrun
87
95
  result = Polyrun::Queue::WorkerLoop.run(
88
96
  store: store, worker_id: wid, batch: batch, cmd: cmd, on_failure: on_failure
89
97
  )
90
- exit result[:exit_code]
98
+ # exit! skips parent at_exit hooks (Coverage finish, RSpec autorun) loaded in this process.
99
+ exit! result[:exit_code]
91
100
  end
92
101
  pids << {pid: pid, worker_id: wid}
93
102
  end
@@ -24,49 +24,55 @@ module Polyrun
24
24
 
25
25
  pids = []
26
26
  Polyrun::WorkerOutput.prepare_log_dir! if Polyrun::WorkerOutput.routing_enabled?
27
- workers.times do |shard|
28
- paths = plan.shard(shard)
29
- if paths.empty?
30
- Polyrun::Log.warn "polyrun run-shards: shard #{shard} skipped (no paths)" if @verbose || parallel
31
- next
32
- end
27
+ begin
28
+ workers.times do |shard|
29
+ paths = plan.shard(shard)
30
+ if paths.empty?
31
+ Polyrun::Log.warn "polyrun run-shards: shard #{shard} skipped (no paths)" if @verbose || parallel
32
+ next
33
+ end
33
34
 
34
- env_shard = ENV.to_h.merge(
35
- "POLYRUN_HOOK_ORCHESTRATOR" => "1",
36
- "POLYRUN_SHARD_INDEX" => shard.to_s,
37
- "POLYRUN_SHARD_TOTAL" => workers.to_s
38
- )
39
- code = hook_cfg.run_phase_if_enabled(:before_shard, env_shard)
40
- if code != 0
41
- run_shards_terminate_children!(pids)
42
- return [pids, code]
43
- end
35
+ env_shard = ENV.to_h.merge(
36
+ "POLYRUN_HOOK_ORCHESTRATOR" => "1",
37
+ "POLYRUN_SHARD_INDEX" => shard.to_s,
38
+ "POLYRUN_SHARD_TOTAL" => workers.to_s
39
+ )
40
+ code = hook_cfg.run_phase_if_enabled(:before_shard, env_shard)
41
+ if code != 0
42
+ run_shards_terminate_children!(pids)
43
+ return [pids, code]
44
+ end
44
45
 
45
- child_env = shard_child_env(
46
- cfg: cfg,
47
- workers: workers,
48
- shard: shard,
49
- matrix_index: mx,
50
- matrix_total: mt,
51
- failure_fragments: ctx[:merge_failures],
52
- spec_quality_fragments: ctx[:merge_spec_quality]
53
- )
54
- child_env = child_env.merge("POLYRUN_HOOK_ORCHESTRATOR" => "0")
55
- child_env = hook_cfg.merge_worker_ruby_env(child_env)
46
+ child_env = shard_child_env(
47
+ cfg: cfg,
48
+ workers: workers,
49
+ shard: shard,
50
+ matrix_index: mx,
51
+ matrix_total: mt,
52
+ failure_fragments: ctx[:merge_failures],
53
+ spec_quality_fragments: ctx[:merge_spec_quality]
54
+ )
55
+ child_env = child_env.merge("POLYRUN_HOOK_ORCHESTRATOR" => "0")
56
+ child_env = hook_cfg.merge_worker_ruby_env(child_env)
56
57
 
57
- ping_path = run_shards_prepare_worker_ping!(ctx, child_env, shard)
58
+ ping_path = run_shards_prepare_worker_ping!(ctx, child_env, shard)
58
59
 
59
- Polyrun::Log.warn "polyrun run-shards: shard #{shard} → #{paths.size} file(s)" if @verbose
60
- spawned_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
61
- pid = run_shards_spawn_one_worker(child_env, cmd, paths, hook_cfg)
62
- pids << {pid: pid, shard: shard, spawned_at: spawned_at, ping_path: ping_path}
63
- Polyrun::Debug.log("[parent pid=#{$$}] run-shards: Process.spawn shard=#{shard} child_pid=#{pid} spec_files=#{paths.size}")
64
- Polyrun::Log.warn "polyrun run-shards: started shard #{shard} pid=#{pid} (#{paths.size} file(s))" if parallel
65
- if Polyrun::WorkerOutput.routing_enabled? && parallel
66
- Polyrun::Log.warn "polyrun run-shards: shard #{shard} log → #{Polyrun::WorkerOutput.log_path_for(shard)}"
60
+ Polyrun::Log.warn "polyrun run-shards: shard #{shard} → #{paths.size} file(s)" if @verbose
61
+ spawned_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
62
+ pid = run_shards_spawn_one_worker(child_env, cmd, paths, hook_cfg)
63
+ pids << {pid: pid, shard: shard, spawned_at: spawned_at, ping_path: ping_path}
64
+ Polyrun::Debug.log("[parent pid=#{$$}] run-shards: Process.spawn shard=#{shard} child_pid=#{pid} spec_files=#{paths.size}")
65
+ Polyrun::Log.warn "polyrun run-shards: started shard #{shard} pid=#{pid} (#{paths.size} file(s))" if parallel
66
+ if Polyrun::WorkerOutput.routing_enabled? && parallel
67
+ Polyrun::Log.warn "polyrun run-shards: shard #{shard} log → #{Polyrun::WorkerOutput.log_path_for(shard)}"
68
+ end
67
69
  end
70
+ [pids, nil]
71
+ rescue StandardError, ScriptError => e
72
+ Polyrun::Log.warn "polyrun run-shards: spawn failed (#{e.class}: #{e.message}); stopping #{pids.size} worker(s)"
73
+ run_shards_terminate_children!(pids)
74
+ [pids, 1]
68
75
  end
69
- [pids, nil]
70
76
  end
71
77
  # rubocop:enable Metrics/AbcSize
72
78
 
@@ -85,13 +91,20 @@ module Polyrun
85
91
  path
86
92
  end
87
93
 
94
+ # Workers start in their own process group so TERM/KILL can reach hook shell grandchildren.
88
95
  def run_shards_spawn_one_worker(child_env, cmd, paths, hook_cfg)
89
96
  return Polyrun::WorkerOutput.spawn_worker(child_env, cmd, paths, hook_cfg) if Polyrun::WorkerOutput.routing_enabled?
90
97
 
91
98
  if hook_cfg.worker_hooks? && !Polyrun::Hooks.disabled?
92
- Process.spawn(child_env, "sh", "-c", hook_cfg.build_worker_shell_script(cmd, paths))
99
+ Process.spawn(
100
+ child_env,
101
+ "sh",
102
+ "-c",
103
+ hook_cfg.build_worker_shell_script(cmd, paths),
104
+ pgroup: true
105
+ )
93
106
  else
94
- Process.spawn(child_env, *cmd, *paths)
107
+ Process.spawn(child_env, *cmd, *paths, pgroup: true)
95
108
  end
96
109
  end
97
110
  end
@@ -234,7 +234,7 @@ module Polyrun
234
234
  end
235
235
 
236
236
  def run_shards_force_stop_pid_status(pid)
237
- Process.kill(:KILL, pid)
237
+ run_shards_kill_process_group!(:KILL, pid)
238
238
  st = nil
239
239
  begin
240
240
  Process.wait(pid)
@@ -243,7 +243,7 @@ module Polyrun
243
243
  # child already reaped
244
244
  end
245
245
  st
246
- rescue Errno::ESRCH
246
+ rescue Errno::ESRCH, Errno::EPERM
247
247
  begin
248
248
  Process.wait(pid)
249
249
  $?
@@ -77,35 +77,13 @@ module Polyrun
77
77
  end
78
78
 
79
79
  def run_shards_resolve_costs(timing_path, strategy, timing_granularity, strategy_explicit: false)
80
- strategy = strategy.to_s
81
- if timing_path
82
- if strategy_explicit && strategy == "round_robin"
83
- return [nil, strategy, nil]
84
- end
85
-
86
- costs = Polyrun::Partition::Plan.load_timing_costs(
87
- File.expand_path(timing_path.to_s, Dir.pwd),
88
- granularity: timing_granularity
89
- )
90
- if costs.empty?
91
- Polyrun::Log.warn "polyrun run-shards: timing file missing or empty: #{timing_path}"
92
- return [nil, nil, 2]
93
- end
94
- if Polyrun::Partition::Plan.timing_load_strategy?(strategy)
95
- return [costs, strategy, nil]
96
- end
97
- unless strategy_explicit
98
- Polyrun::Log.warn "polyrun run-shards: using cost_binpack (timing data present)" if @verbose
99
- return [costs, "cost_binpack", nil]
100
- end
101
-
102
- [nil, strategy, nil]
103
- elsif Polyrun::Partition::Plan.cost_strategy?(strategy) || Polyrun::Partition::Plan.lazy_robin_strategy?(strategy)
104
- Polyrun::Log.warn "polyrun run-shards: --timing or partition.timing_file required for strategy #{strategy}"
105
- [nil, nil, 2]
106
- else
107
- [nil, strategy, nil]
108
- end
80
+ partition_resolve_costs_and_strategy(
81
+ timing_path,
82
+ strategy,
83
+ timing_granularity,
84
+ strategy_explicit: strategy_explicit,
85
+ log_prefix: "polyrun run-shards"
86
+ )
109
87
  end
110
88
 
111
89
  def run_shards_make_plan(items, workers, strategy, seed, costs, constraints, timing_granularity, stable_assignment = nil, shard_weights: nil)
@@ -127,7 +127,7 @@ module Polyrun
127
127
  return if Polyrun::WorkerOutput.routing_enabled?
128
128
 
129
129
  Polyrun::Log.warn "polyrun run-shards: #{pid_count} children running; RSpec output below may be interleaved."
130
- Polyrun::Log.warn "polyrun run-shards: each worker prints its own summary line; the last \"N examples\" line is not a total across shards."
130
+ Polyrun::Log.warn "polyrun run-shards: RSpec summary counts apply to individual shards."
131
131
  end
132
132
 
133
133
  def run_shards_merge_or_hint_coverage(ctx)
@@ -19,33 +19,38 @@ module Polyrun
19
19
  # Best-effort worker teardown then exit. Does not return.
20
20
  def run_shards_shutdown_on_signal!(pids, code)
21
21
  run_shards_log_interrupt_workers(pids, nil)
22
- Polyrun::WorkerOutput.shutdown_all!
23
22
  run_shards_terminate_children!(pids)
24
23
  exit(code)
25
24
  rescue Interrupt
26
25
  run_shards_signal_workers_kill(pids)
27
26
  run_shards_reap_worker_pids_interruptible(pids.map { |h| h[:pid] })
27
+ Polyrun::WorkerOutput.shutdown_all!
28
28
  exit(code)
29
29
  end
30
30
 
31
- # Send SIGTERM to each worker PID and wait so Ctrl+C / SIGTERM does not leave orphans.
31
+ # Signal workers first so pipe readers see EOF; join forwarders only after children exit.
32
32
  def run_shards_terminate_children!(pids)
33
- Polyrun::WorkerOutput.shutdown_all!
34
33
  run_shards_signal_workers_term(pids)
35
34
  run_shards_reap_worker_pids_interruptible(pids.map { |h| h[:pid] })
35
+ Polyrun::WorkerOutput.shutdown_all!
36
36
  end
37
37
 
38
38
  def run_shards_signal_workers_term(pids)
39
- pids.each do |h|
40
- Process.kill(:TERM, h[:pid])
41
- rescue Errno::ESRCH
42
- end
39
+ pids.each { |h| run_shards_kill_process_group!(:TERM, h[:pid]) }
43
40
  end
44
41
 
45
42
  def run_shards_signal_workers_kill(pids)
46
- pids.each do |h|
47
- Process.kill(:KILL, h[:pid])
48
- rescue Errno::ESRCH
43
+ pids.each { |h| run_shards_kill_process_group!(:KILL, h[:pid]) }
44
+ end
45
+
46
+ # Negative PID signals the process group (workers spawn with +pgroup: true+).
47
+ def run_shards_kill_process_group!(signal, pid)
48
+ Process.kill(signal, -Integer(pid))
49
+ rescue Errno::ESRCH, Errno::EPERM
50
+ begin
51
+ Process.kill(signal, Integer(pid))
52
+ rescue Errno::ESRCH, Errno::EPERM
53
+ nil
49
54
  end
50
55
  end
51
56
 
@@ -71,10 +76,7 @@ module Polyrun
71
76
  force_note = true
72
77
  Polyrun::Log.orchestration_warn "polyrun run-shards: repeated SIGINT during worker cleanup — SIGKILL to #{pending.size} process(es)"
73
78
  end
74
- pending.each do |pid|
75
- Process.kill(:KILL, pid)
76
- rescue Errno::ESRCH
77
- end
79
+ pending.each { |pid| run_shards_kill_process_group!(:KILL, pid) }
78
80
  end
79
81
  end
80
82
  end
@@ -12,24 +12,30 @@ module Polyrun
12
12
  return code if code != 0
13
13
  end
14
14
 
15
- if start_run_database_provision?(cfg) && !truthy_env?("POLYRUN_START_SKIP_DATABASES")
16
- workers = parse_workers_from_start_argv(argv)
17
- Polyrun::Log.warn "polyrun start: provisioning test DBs (template + shards 0..#{workers - 1})" if @verbose
18
- begin
19
- Polyrun::Database::CloneShards.provision!(
20
- cfg.databases,
21
- workers: workers,
22
- rails_root: Dir.pwd,
23
- migrate: true,
24
- replace: true,
25
- force_drop: truthy_env?("POLYRUN_PG_DROP_FORCE"),
26
- dry_run: false,
27
- silent: !@verbose
28
- )
29
- rescue Polyrun::Error => e
30
- Polyrun::Log.warn "polyrun start: #{e.message}"
31
- return 1
32
- end
15
+ return 0 unless start_databases_provision_wanted?(cfg) && !truthy_env?("POLYRUN_START_SKIP_DATABASES")
16
+
17
+ dh = cfg.databases
18
+ adapter = Polyrun::Database::UrlBuilder::ConnectionInfer.infer_adapter_name(dh)
19
+ unless adapter == "postgresql"
20
+ return start_non_postgresql_databases_outcome(cfg, adapter)
21
+ end
22
+
23
+ workers = parse_workers_from_start_argv(argv)
24
+ Polyrun::Log.warn "polyrun start: provisioning test DBs (template + shards 0..#{workers - 1})" if @verbose
25
+ begin
26
+ Polyrun::Database::CloneShards.provision!(
27
+ dh,
28
+ workers: workers,
29
+ rails_root: Dir.pwd,
30
+ migrate: true,
31
+ replace: true,
32
+ force_drop: truthy_env?("POLYRUN_PG_DROP_FORCE"),
33
+ dry_run: false,
34
+ silent: !@verbose
35
+ )
36
+ rescue Polyrun::Error => e
37
+ Polyrun::Log.warn "polyrun start: #{e.message}"
38
+ return 1
33
39
  end
34
40
  0
35
41
  end
@@ -53,7 +59,7 @@ module Polyrun
53
59
  false
54
60
  end
55
61
 
56
- def start_run_database_provision?(cfg)
62
+ def start_databases_provision_wanted?(cfg)
57
63
  st = cfg.start_config
58
64
  dh = cfg.databases
59
65
  return false unless dh.is_a?(Hash)
@@ -61,14 +67,33 @@ module Polyrun
61
67
  template = (dh["template_db"] || dh[:template_db]).to_s
62
68
  return false if template.empty?
63
69
 
64
- if st["databases"] == true || st[:databases] == true
65
- return true
66
- end
67
70
  return false if st["databases"] == false || st[:databases] == false
68
71
 
69
72
  true
70
73
  end
71
74
 
75
+ def start_databases_explicitly_enabled?(cfg)
76
+ st = cfg.start_config
77
+ st["databases"] == true || st[:databases] == true
78
+ end
79
+
80
+ def start_non_postgresql_databases_outcome(cfg, adapter)
81
+ if start_databases_explicitly_enabled?(cfg)
82
+ Polyrun::Log.warn(
83
+ "polyrun start: start.databases is true but databases adapter is #{adapter}; " \
84
+ "CloneShards auto provision is PostgreSQL-only. Set start.databases: false and provision " \
85
+ "shards yourself (for example in prepare), or use a postgresql: connection block."
86
+ )
87
+ return 1
88
+ end
89
+
90
+ Polyrun::Log.warn(
91
+ "polyrun start: skipping database provision (adapter=#{adapter}; CloneShards is PostgreSQL-only). " \
92
+ "Provision shards in prepare, or set start.databases: false to silence this message."
93
+ )
94
+ 0
95
+ end
96
+
72
97
  def parse_workers_from_start_argv(argv)
73
98
  sep = argv.index("--")
74
99
  head = sep ? argv[0...sep] : argv
@@ -79,6 +104,10 @@ module Polyrun
79
104
  w = Integer(head[i + 1], exception: false)
80
105
  workers = w if w && w >= 1
81
106
  i += 2
107
+ elsif head[i].start_with?("--workers=")
108
+ w = Integer(head[i].split("=", 2).last, exception: false)
109
+ workers = w if w && w >= 1
110
+ i += 1
82
111
  else
83
112
  i += 1
84
113
  end