polyrun 2.2.4 → 2.2.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +28 -0
  3. data/README.md +4 -1
  4. data/docs/SETUP_PROFILE.md +1 -0
  5. data/lib/polyrun/cli/database_commands.rb +21 -7
  6. data/lib/polyrun/cli/default_run.rb +28 -6
  7. data/lib/polyrun/cli/help.rb +4 -2
  8. data/lib/polyrun/cli/helpers.rb +41 -6
  9. data/lib/polyrun/cli/plan_command.rb +10 -29
  10. data/lib/polyrun/cli/run_queue_command.rb +10 -1
  11. data/lib/polyrun/cli/run_shards_command.rb +2 -8
  12. data/lib/polyrun/cli/run_shards_parallel_children.rb +52 -39
  13. data/lib/polyrun/cli/run_shards_parallel_wait.rb +2 -2
  14. data/lib/polyrun/cli/run_shards_plan_boot_phases.rb +3 -0
  15. data/lib/polyrun/cli/run_shards_planning.rb +16 -29
  16. data/lib/polyrun/cli/run_shards_run.rb +1 -1
  17. data/lib/polyrun/cli/run_shards_worker_interrupt.rb +16 -14
  18. data/lib/polyrun/cli/start_bootstrap.rb +51 -22
  19. data/lib/polyrun/cli.rb +8 -1
  20. data/lib/polyrun/config.rb +9 -4
  21. data/lib/polyrun/coverage/example_diff.rb +6 -7
  22. data/lib/polyrun/coverage/merge/formatters_html.rb +20 -2
  23. data/lib/polyrun/coverage/merge.rb +9 -3
  24. data/lib/polyrun/coverage/merge_fragment_meta.rb +6 -1
  25. data/lib/polyrun/coverage/rails.rb +1 -1
  26. data/lib/polyrun/database/clone_shards.rb +28 -5
  27. data/lib/polyrun/database/provision.rb +15 -9
  28. data/lib/polyrun/database/url_builder/connection/infer.rb +4 -0
  29. data/lib/polyrun/database/url_builder.rb +31 -3
  30. data/lib/polyrun/partition/paths.rb +23 -4
  31. data/lib/polyrun/partition/plan.rb +6 -24
  32. data/lib/polyrun/partition/suite.rb +116 -0
  33. data/lib/polyrun/partition/timing_diagnostics.rb +2 -3
  34. data/lib/polyrun/queue/file_store.rb +133 -30
  35. data/lib/polyrun/queue/file_store_intent.rb +188 -0
  36. data/lib/polyrun/queue/worker_loop.rb +41 -7
  37. data/lib/polyrun/spec_quality/config.rb +1 -1
  38. data/lib/polyrun/spec_quality/profile.rb +0 -1
  39. data/lib/polyrun/spec_quality/report.rb +16 -7
  40. data/lib/polyrun/templates/rails_prepare.polyrun.yml +3 -1
  41. data/lib/polyrun/version.rb +1 -1
  42. data/lib/polyrun/worker_output.rb +30 -13
  43. data/lib/polyrun.rb +1 -0
  44. metadata +3 -1
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: de79eaae07044fa991b839a44011f7648df5ba16d0586a1ea2708c1d7160cf5e
4
- data.tar.gz: f3cbab3bf642164ffa507aa75be1acaaf7a232057b1b1c5f3fcdb0f804c2a0a7
3
+ metadata.gz: 69dd2432a149cc9790756d2d54b6481e898604a7ae84219b8b5cfbd65997e231
4
+ data.tar.gz: ab1ea9013d606dd53e717cc421e353c668b47382871e3494f03fa689333e422c
5
5
  SHA512:
6
- metadata.gz: f62f0cbc1760e3fc42a4711b7e55eace3b8b5bb038836abcae91eba876cfabf9b72832e2f04a284744fdf09247288bff6b60c7a3b1a38d00ee140d37634702df
7
- data.tar.gz: 4f9b6d94992745419d35a5bbb40041094569f725470904f546566bfdba7a3e7eeca899f5b80844d9908ca36e744870fef3f8ecef6e20d589fa090d0363859147
6
+ metadata.gz: 70c45810383e893e6d4ed8a9b2288f3540bb85e37345149283cd9992e2f23c5fa1d475b9e43766d20ddc6f93eaae562c258ff99769284bf606aa2858bbaae515
7
+ data.tar.gz: 17a14844465eade2840dcc968d1c432e4462991ac099831cd03cd19f262aec9b42d5bd129efcc8e0b843534357ae281380507efb7c40f530e772a7bcb78d9f40
data/CHANGELOG.md CHANGED
@@ -2,6 +2,34 @@
2
2
 
3
3
  ## Unreleased
4
4
 
5
+ ## 2.2.6 (2026-10-29)
6
+
7
+ - Fix `polyrun start` auto database provision when `databases:` uses a non-Postgres adapter (`mysql2`, trilogy, …): skip CloneShards with a clear warning (or exit 1 if `start.databases: true`); keep `template_db` / `shard_db_pattern` for URL shaping.
8
+ - Fail `db:clone-shards` / `db:setup-shard` before `psql` when the inferred adapter is not PostgreSQL.
9
+ - Pass `postgresql:` host, port, username, and password from `polyrun.yml` into `psql` during shard provision (password via child `PGPASSWORD`).
10
+ - Fix implicit path runs so tokens after `--` stay the worker command, and `--paths-file` is passed before `--`.
11
+ - Accept `polyrun start --workers=N` (equals form) so shard DB provision matches worker count.
12
+ - Reject `databases.shard_db_pattern` without `%{shard}` / `%<shard>d`, and refuse patterns that resolve to `template_db`.
13
+ - Redact database passwords in `db:*` / CloneShards dry-run shell logs.
14
+ - Raise when an explicit `-c` / `POLYRUN_CONFIG` path is missing; map `Polyrun::Error` to exit 2.
15
+ - Validate `run-queue --workers` (1..max) and exit forked queue workers with `exit!`.
16
+ - Fix SpecQuality `minimum_unique_lines_per_example` gate for thresholds above 1; keep only positive or CPU profile deltas.
17
+ - Align file-granularity timing cost keys with realpath normalization so cost binpack hits under symlink roots.
18
+ - TERM shard workers before draining routed worker output on SIGINT/SIGTERM; skip empty queue leases.
19
+ - Recover interrupted queue claim/ack/reclaim via `intent.json`; age leases by `heartbeat_at`; quarantine poison paths after `POLYRUN_QUEUE_MAX_ATTEMPTS` (default 3).
20
+ - On a failed `run-queue` batch, re-run paths one-by-one and ack successes so a poison path does not requeue neighbors.
21
+ - Spawn shard workers in their own process group and signal the group on timeout/interrupt; abort already-started workers if a later spawn fails.
22
+ - Cache coverage HTML ERB templates; fold merge fragments without retaining full JSON documents; share plan/run-shards timing cost resolution.
23
+ - Clarify `run-shards` interleaved-output notice: RSpec summary counts apply to individual shards (no false per-worker total).
24
+
25
+ ## 2.2.5 (2026-09-24)
26
+
27
+ - Fix bare `polyrun` suite selection when both `spec/` and `test/` exist: honor `partition.suite`, else infer from `partition.paths_file`, else filesystem auto-detect (RSpec first).
28
+ - Exit 2 when `partition.suite` disagrees with `paths_file`, or when `run-shards` paths infer a different suite than a recognizable command after `--`.
29
+ - Add `parallel-minitest` and `parallel-quick` CLI subcommands (same shape as `parallel-rspec`); they do not run `start` prepare or database bootstrap.
30
+ - Recognize `bin/rails test` and compact `ruby -Itest` in the suite/command mismatch guard; strip `path:line` locators before classifying paths.
31
+ - Skip legacy `spec/spec_paths.txt` when `partition.suite` is `minitest` or `quick`; allow mixed path lists only with suite-agnostic custom runners.
32
+
5
33
  ## 2.2.4 (2026-09-04)
6
34
 
7
35
  - Fix `DEBUG_SQL` / `POLYRUN_DEBUG_SQL` raising `NoMethodError` for `loggable_sql?` when the SQL subscriber runs on an RSpec example group.
data/README.md CHANGED
@@ -10,7 +10,7 @@ Running tests in parallel across processes still requires a single merged covera
10
10
 
11
11
  Polyrun provides:
12
12
 
13
- - Orchestration: `plan`, `run-shards`, and `parallel-rspec` (run-shards plus merge-coverage), with an optional on-disk queue and constraints for file lists and load balancing. For **GitHub Actions-style matrix sharding** (one job per global shard), use `ci-shard-run -- …` (any test runner) or `ci-shard-rspec`—not `run-shards` / `parallel-rspec`, which fan out N workers on one machine.
13
+ - Orchestration: `plan`, `run-shards`, and `parallel-rspec` / `parallel-minitest` / `parallel-quick` (run-shards plus merge-coverage), with an optional on-disk queue and constraints for file lists and load balancing. Bare `polyrun` (no subcommand) picks the suite from `partition.suite`, else `partition.paths_file`, else filesystem auto-detect. For **GitHub Actions-style matrix sharding** (one job per global shard), use `ci-shard-run -- …` (any test runner) or `ci-shard-rspec`—not `run-shards` / `parallel-*`, which fan out N workers on one machine.
14
14
  - Coverage: merge SimpleCov-compatible JSON fragments; emit JSON, LCOV, Cobertura, or console summaries (you can drop separate SimpleCov merge plugins for this path).
15
15
  - CI reporting: JUnit XML from RSpec JSON; slow-file reports from merged timing JSON.
16
16
  - Parallel hygiene: asset digest markers, SQL snapshots, YAML fixture batches, and DB URL or shard helpers aligned with `POLYRUN_SHARD_*`.
@@ -92,6 +92,8 @@ bin/polyrun build-paths # write spec/spec_paths.txt from partition.paths_build
92
92
  bin/polyrun ci-shard-run -- bundle exec rspec # CI matrix: shard plan + append paths to the command after --
93
93
  bin/polyrun ci-shard-rspec # same as ci-shard-run -- bundle exec rspec
94
94
  bin/polyrun parallel-rspec --workers 5 # run-shards + merge-coverage (default: bundle exec rspec)
95
+ bin/polyrun parallel-minitest --workers 5 # same; default rails test or ruby -I test
96
+ bin/polyrun parallel-quick --workers 5 # same; default bundle exec polyrun quick
95
97
  bin/polyrun run-shards --workers 5 --merge-coverage -- bundle exec rspec
96
98
  bin/polyrun merge-coverage -i cov1.json -i cov2.json -o merged.json --format json,lcov,cobertura,console
97
99
  bin/polyrun env --shard 0 --total 4 # print DATABASE_URL exports from polyrun.yml in cwd
@@ -145,6 +147,7 @@ That single require loads the CLI and core library **without** loading RSpec or
145
147
  | `Polyrun::Prepare::Assets` | Digest trees, marker file, `assets:precompile`. |
146
148
  | `Polyrun::Database::Shard` | Shard env map, `%{shard}` DB names, URL path suffix for `postgres://`, `mysql2://`, `mongodb://`, etc. |
147
149
  | `Polyrun::Database::UrlBuilder` | URLs from `polyrun.yml` `databases:` — nested blocks or `adapter:` for common Rails stacks (`postgresql`, `mysql`/`mysql2`, `trilogy`, `sqlserver`/`mssql`, `sqlite3`/`sqlite`, `mongodb`/`mongo`). |
150
+ | `Polyrun::Database::CloneShards` / `Provision` | PostgreSQL-only template + shard `CREATE DATABASE … TEMPLATE` via `psql`. Non-Postgres configs still use UrlBuilder for env; set `start.databases: false` and provision shards in prepare. |
148
151
  | `Polyrun::Hooks` | Load from `Config#hooks`; `run_phase` / `run_phase_if_enabled`; `build_worker_shell_script` wraps the worker command. |
149
152
  | `Polyrun::Hooks::Dsl` | Ruby hook file (`hooks.ruby`); `before(:suite)` / `after(:each)` etc. in `config/polyrun_hooks.rb` (see README). |
150
153
 
@@ -25,6 +25,7 @@ Do not mix “fan out N workers inside one job” with “matrix shard index”
25
25
  |--------|---------|
26
26
  | Single test DB | Omitting `databases:` in `polyrun.yml` may be enough; ensure parallel workers do not share one DB if they mutate data |
27
27
  | Multi-DB or shard suffixes | `databases:` in `polyrun.yml` (`shard_db_pattern`, `template_db`, optional `connections` and `env_key`). `database.yml` or `DATABASE_URL` may use `%{shard}` or `POLYRUN_SHARD_INDEX` suffixes—the same convention as `polyrun env` |
28
+ | Non-Postgres adapters (mysql2, trilogy, …) | `UrlBuilder` still emits URLs from `template_db` / `shard_db_pattern`. Auto provision on `polyrun start` and `db:clone-shards` / `db:setup-shard` are PostgreSQL-only (`CREATE DATABASE … TEMPLATE`). For MySQL/MariaDB, set `start.databases: false` (or rely on the skip warning) and provision shards in `prepare` |
28
29
  | External provisioning | Sometimes `db:prepare` plus shell clone scripts (multi-DB apps)—document ordering: prepare databases before `run-shards` |
29
30
 
30
31
  ## 4. Prepare (run once before workers)
@@ -47,26 +47,32 @@ module Polyrun
47
47
  def cmd_db_setup_shard(argv, config_path)
48
48
  dry = db_setup_shard_parse_options!(argv)
49
49
  cfg = Polyrun::Config.load(path: config_path || ENV["POLYRUN_CONFIG"])
50
- dh = cfg.databases
51
- if !dh.is_a?(Hash) || dh.empty?
50
+ databases = cfg.databases
51
+ if !databases.is_a?(Hash) || databases.empty?
52
52
  Polyrun::Log.warn "db:setup-shard: configure databases: in polyrun.yml"
53
53
  return 2
54
54
  end
55
55
 
56
+ unless Polyrun::Database::UrlBuilder::ConnectionInfer.postgresql?(databases)
57
+ adapter = Polyrun::Database::UrlBuilder::ConnectionInfer.infer_adapter_name(databases)
58
+ Polyrun::Log.warn "db:setup-shard: PostgreSQL-only (databases adapter is #{adapter})"
59
+ return 2
60
+ end
61
+
56
62
  shard = resolve_shard_index(cfg.partition)
57
- template = dh["template_db"] || dh[:template_db]
63
+ template = databases["template_db"] || databases[:template_db]
58
64
  if !template
59
65
  Polyrun::Log.warn "db:setup-shard: set databases.template_db"
60
66
  return 2
61
67
  end
62
68
 
63
- plan = Polyrun::Database::UrlBuilder.shard_database_plan(dh, shard_index: shard)
69
+ plan = Polyrun::Database::UrlBuilder.shard_database_plan(databases, shard_index: shard)
64
70
  if plan.empty?
65
71
  Polyrun::Log.warn "db:setup-shard: could not derive shard database names from polyrun.yml"
66
72
  return 2
67
73
  end
68
74
 
69
- db_setup_shard_run_plan(plan, dry: dry)
75
+ db_setup_shard_run_plan(plan, databases, dry: dry)
70
76
  end
71
77
 
72
78
  def db_setup_shard_parse_options!(argv)
@@ -77,7 +83,7 @@ module Polyrun
77
83
  dry
78
84
  end
79
85
 
80
- def db_setup_shard_run_plan(plan, dry:)
86
+ def db_setup_shard_run_plan(plan, databases, dry:)
81
87
  if dry
82
88
  plan.each do |row|
83
89
  Polyrun::Log.warn "would: CREATE DATABASE #{row[:new_db]} TEMPLATE #{row[:template_db]}"
@@ -85,11 +91,19 @@ module Polyrun
85
91
  return 0
86
92
  end
87
93
 
94
+ conn = Polyrun::Database::UrlBuilder::Connection.resolve_connection(databases)
95
+ opts = {
96
+ host: conn[:host],
97
+ port: conn[:port],
98
+ username: conn[:user],
99
+ password: conn[:password]
100
+ }
88
101
  threads = plan.map do |row|
89
102
  Thread.new do
90
103
  Polyrun::Database::Provision.create_database_from_template!(
91
104
  new_db: row[:new_db],
92
- template_db: row[:template_db].to_s
105
+ template_db: row[:template_db].to_s,
106
+ **opts
93
107
  )
94
108
  end
95
109
  end
@@ -7,20 +7,32 @@ module Polyrun
7
7
  private
8
8
 
9
9
  def dispatch_default_parallel!(config_path)
10
- suite = Polyrun::Partition::Paths.detect_auto_suite(Dir.pwd)
11
- unless suite
12
- Polyrun::Log.warn "polyrun: no tests found (spec/**/*_spec.rb, test/**/*_test.rb, or Polyrun quick files). See polyrun help."
10
+ cfg = Polyrun::Config.load(path: config_path || ENV["POLYRUN_CONFIG"])
11
+ pc = cfg.partition
12
+ code = Polyrun::Partition::PathsBuild.apply!(partition: pc, cwd: Dir.pwd)
13
+ return code if code != 0
14
+
15
+ resolved = Polyrun::Partition::Suite.resolve_default(cwd: Dir.pwd, partition: pc)
16
+ if resolved[:error]
17
+ Polyrun::Log.warn "polyrun: #{resolved[:error]}"
13
18
  return 2
14
19
  end
15
20
 
21
+ suite = resolved[:suite]
16
22
  Polyrun::Log.warn "polyrun: default → parallel #{suite} (use `polyrun help` for subcommands)" if @verbose
17
23
 
18
24
  case suite
19
25
  when :rspec
20
26
  cmd_start([], config_path)
21
27
  when :minitest
28
+ code = start_bootstrap!(cfg, [], config_path)
29
+ return code if code != 0
30
+
22
31
  cmd_parallel_minitest([], config_path)
23
32
  when :quick
33
+ code = start_bootstrap!(cfg, [], config_path)
34
+ return code if code != 0
35
+
24
36
  cmd_parallel_quick([], config_path)
25
37
  else
26
38
  2
@@ -48,8 +60,7 @@ module Polyrun
48
60
  end
49
61
 
50
62
  def dispatch_implicit_parallel_targets!(argv, config_path)
51
- path_tokens = argv.select { |a| cli_implicit_path_token?(a) }
52
- head = argv.reject { |a| cli_implicit_path_token?(a) }
63
+ path_tokens, head, after = split_implicit_arguments(argv)
53
64
  expanded = expand_implicit_target_paths(path_tokens)
54
65
  if expanded.empty?
55
66
  Polyrun::Log.warn "polyrun: no files matched path arguments"
@@ -70,7 +81,8 @@ module Polyrun
70
81
  begin
71
82
  tmp.write(expanded.join("\n") + "\n")
72
83
  tmp.close
73
- combined = head + ["--paths-file", tmp.path]
84
+ # Keep --paths-file before `--` so OptionParser sees it; after `--` it becomes a worker argv token.
85
+ combined = head + ["--paths-file", tmp.path] + after
74
86
  case suite
75
87
  when :rspec
76
88
  cmd_start(combined, config_path)
@@ -91,6 +103,16 @@ module Polyrun
91
103
  end
92
104
  end
93
105
 
106
+ def split_implicit_arguments(argv)
107
+ # Only tokens before `--` are suite paths; everything from `--` onward is the worker command.
108
+ sep = argv.index("--")
109
+ before = sep ? argv[0...sep] : argv
110
+ after = sep ? argv[sep..] : []
111
+ path_tokens = before.select { |a| cli_implicit_path_token?(a) }
112
+ head = before.reject { |a| cli_implicit_path_token?(a) }
113
+ [path_tokens, head, after]
114
+ end
115
+
94
116
  def expand_implicit_target_paths(path_tokens)
95
117
  path_tokens.flat_map do |p|
96
118
  abs = File.expand_path(p)
@@ -5,7 +5,7 @@ module Polyrun
5
5
  Polyrun::Log.puts <<~HELP
6
6
  usage: polyrun [global options] [<command> | <paths...>]
7
7
 
8
- With no command, runs parallel tests for the detected suite: RSpec under spec/, Minitest under test/, or Polyrun Quick (same discovery as polyrun quick). If the first argument is a known subcommand name, it is dispatched. Otherwise, path-like tokens (optionally with run-shards flags such as --workers) shard those files in parallel; see commands below.
8
+ With no command, runs parallel tests for the resolved suite: partition.suite when set; else suite inferred from partition.paths_file; else filesystem auto-detect (RSpec under spec/ before Minitest under test/, then Polyrun Quick). If the first argument is a known subcommand name, it is dispatched. Otherwise, path-like tokens (optionally with run-shards flags such as --workers) shard those files in parallel; see commands below.
9
9
 
10
10
  global:
11
11
  -c, --config PATH polyrun.yml path (or POLYRUN_CONFIG)
@@ -40,7 +40,9 @@ module Polyrun
40
40
  merge-coverage merge SimpleCov JSON fragments (json/lcov/cobertura/console/html/csv/markdown)
41
41
  merge-failures merge per-shard failure JSONL fragments or RSpec JSON files (jsonl/json/csv/markdown)
42
42
  run-shards fan out N parallel OS processes (POLYRUN_SHARD_*; not Ruby threads); optional --merge-coverage / --merge-failures / --merge-spec-quality
43
- parallel-rspec run-shards + merge-coverage (defaults to: bundle exec rspec after --)
43
+ parallel-rspec run-shards + merge-coverage (defaults to: bundle exec rspec after --); no start prepare/DB bootstrap
44
+ parallel-minitest run-shards + merge-coverage (defaults to: bundle exec rails test or ruby -I test after --); no start prepare/DB bootstrap
45
+ parallel-quick run-shards + merge-coverage (defaults to: bundle exec polyrun quick after --); no start prepare/DB bootstrap
44
46
  start parallel-rspec; auto-runs prepare (shell/assets) and db:setup-* when polyrun.yml configures them; legacy script/build_spec_paths.rb if paths_build absent
45
47
  ci-shard-run CI matrix: build-paths + plan for POLYRUN_SHARD_INDEX / POLYRUN_SHARD_TOTAL (or config), then run your command with that shard's paths after --; optional --shard-processes M or --workers M (POLYRUN_SHARD_PROCESSES; not POLYRUN_WORKERS) for N×M jobs × processes on this host
46
48
  ci-shard-rspec same as ci-shard-run -- bundle exec rspec; optional --shard-processes / --workers / -- [rspec-only flags]
@@ -11,6 +11,46 @@ module Polyrun
11
11
  Polyrun::Config::Resolver.env_int(name, fallback)
12
12
  end
13
13
 
14
+ # Shared timing → costs/strategy resolution for +plan+ and +run-shards+.
15
+ # @return [Array] +[costs, strategy, error_code]+ where +error_code+ is nil on success
16
+ def partition_resolve_costs_and_strategy(
17
+ timing_path,
18
+ strategy,
19
+ timing_granularity,
20
+ strategy_explicit: false,
21
+ log_prefix: "polyrun"
22
+ )
23
+ strategy = strategy.to_s
24
+ if timing_path
25
+ if strategy_explicit && strategy == "round_robin"
26
+ return [nil, strategy, nil]
27
+ end
28
+
29
+ costs = Polyrun::Partition::Plan.load_timing_costs(
30
+ File.expand_path(timing_path.to_s, Dir.pwd),
31
+ granularity: timing_granularity
32
+ )
33
+ if costs.empty?
34
+ Polyrun::Log.warn "#{log_prefix}: timing file missing or empty: #{timing_path}"
35
+ return [nil, nil, 2]
36
+ end
37
+ if Polyrun::Partition::Plan.timing_load_strategy?(strategy)
38
+ return [costs, strategy, nil]
39
+ end
40
+ unless strategy_explicit
41
+ Polyrun::Log.warn "#{log_prefix}: using cost_binpack (timing data present)" if @verbose
42
+ return [costs, "cost_binpack", nil]
43
+ end
44
+
45
+ [nil, strategy, nil]
46
+ elsif Polyrun::Partition::Plan.cost_strategy?(strategy) || Polyrun::Partition::Plan.lazy_robin_strategy?(strategy)
47
+ Polyrun::Log.warn "#{log_prefix}: --timing or partition.timing_file required for strategy #{strategy}"
48
+ [nil, nil, 2]
49
+ else
50
+ [nil, strategy, nil]
51
+ end
52
+ end
53
+
14
54
  # Per-worker wall clock (from spawn) for run-shards / ci-shard fan-out; unset or invalid means no limit.
15
55
  def env_worker_timeout_sec
16
56
  s = ENV["POLYRUN_WORKER_TIMEOUT_SEC"].to_s.strip
@@ -97,12 +137,7 @@ module Polyrun
97
137
  # {Partition::Plan#default_weight} semantics: mean of known timing costs for missing paths.
98
138
  def queue_weight_for(path, costs, default_weight = nil, granularity: :file)
99
139
  g = Polyrun::Partition::TimingKeys.normalize_granularity(granularity)
100
- key =
101
- if g == :example
102
- Polyrun::Partition::TimingKeys.normalize_locator(path.to_s, Dir.pwd, :example)
103
- else
104
- File.expand_path(path.to_s, Dir.pwd)
105
- end
140
+ key = Polyrun::Partition::TimingKeys.normalize_locator(path.to_s, Dir.pwd, g)
106
141
  return costs[key] if costs.key?(key)
107
142
 
108
143
  unless default_weight.nil?
@@ -167,35 +167,16 @@ module Polyrun
167
167
  end
168
168
 
169
169
  def plan_load_costs_and_strategy(timing_path, strategy, timing_granularity, strategy_explicit: false)
170
- strategy = strategy.to_s
171
- if timing_path
172
- if strategy_explicit && strategy == "round_robin"
173
- return [nil, strategy]
174
- end
175
-
176
- costs = Polyrun::Partition::Plan.load_timing_costs(
177
- File.expand_path(timing_path.to_s, Dir.pwd),
178
- granularity: timing_granularity
179
- )
180
- if costs.empty?
181
- Polyrun::Log.warn "polyrun plan: timing file missing or has no entries: #{timing_path}"
182
- return nil
183
- end
184
- if Polyrun::Partition::Plan.timing_load_strategy?(strategy)
185
- return [costs, strategy]
186
- end
187
- unless strategy_explicit
188
- Polyrun::Log.warn "polyrun plan: using cost_binpack (timing data present)" if @verbose
189
- return [costs, "cost_binpack"]
190
- end
191
-
192
- [nil, strategy]
193
- elsif Polyrun::Partition::Plan.cost_strategy?(strategy) || Polyrun::Partition::Plan.lazy_robin_strategy?(strategy)
194
- Polyrun::Log.warn "polyrun plan: --timing or partition.timing_file required for strategy #{strategy}"
195
- nil
196
- else
197
- [nil, strategy]
198
- end
170
+ costs, resolved, err = partition_resolve_costs_and_strategy(
171
+ timing_path,
172
+ strategy,
173
+ timing_granularity,
174
+ strategy_explicit: strategy_explicit,
175
+ log_prefix: "polyrun plan"
176
+ )
177
+ return nil if err
178
+
179
+ [costs, resolved]
199
180
  end
200
181
  end
201
182
  end
@@ -43,6 +43,14 @@ module Polyrun
43
43
  end.parse!(head)
44
44
 
45
45
  workers ||= env_int("POLYRUN_WORKERS", Polyrun::Config::DEFAULT_PARALLEL_WORKERS)
46
+ unless workers.is_a?(Integer) && workers.between?(1, Polyrun::Config::MAX_PARALLEL_WORKERS)
47
+ Polyrun::Log.warn "polyrun run-queue: --workers must be between 1 and #{Polyrun::Config::MAX_PARALLEL_WORKERS}"
48
+ return 2
49
+ end
50
+ unless batch.is_a?(Integer) && batch >= 1
51
+ Polyrun::Log.warn "polyrun run-queue: --batch must be >= 1"
52
+ return 2
53
+ end
46
54
  paths_file ||= pc["paths_file"] || pc[:paths_file]
47
55
  unless paths_file
48
56
  Polyrun::Log.warn "polyrun run-queue: need --paths-file or partition.paths_file"
@@ -87,7 +95,8 @@ module Polyrun
87
95
  result = Polyrun::Queue::WorkerLoop.run(
88
96
  store: store, worker_id: wid, batch: batch, cmd: cmd, on_failure: on_failure
89
97
  )
90
- exit result[:exit_code]
98
+ # exit! skips parent at_exit hooks (Coverage finish, RSpec autorun) loaded in this process.
99
+ exit! result[:exit_code]
91
100
  end
92
101
  pids << {pid: pid, worker_id: wid}
93
102
  end
@@ -40,11 +40,8 @@ module Polyrun
40
40
  end
41
41
 
42
42
  # Same as parallel-rspec but runs +bundle exec rails test+ or +bundle exec ruby -I test+ after +--+.
43
+ # Does not run +start+ prepare/database bootstrap (use +polyrun start+ or bare +polyrun+ for that).
43
44
  def cmd_parallel_minitest(argv, config_path)
44
- cfg = Polyrun::Config.load(path: config_path || ENV["POLYRUN_CONFIG"])
45
- code = start_bootstrap!(cfg, argv, config_path)
46
- return code if code != 0
47
-
48
45
  sep = argv.index("--")
49
46
  combined =
50
47
  if sep
@@ -60,11 +57,8 @@ module Polyrun
60
57
 
61
58
  # Same as parallel-rspec but runs +bundle exec polyrun quick+ after +--+ (one Quick process per shard).
62
59
  # Run from the app root with +bundle exec+ so workers resolve the same gem as the parent (same concern as +bundle exec rspec+).
60
+ # Does not run +start+ prepare/database bootstrap (use +polyrun start+ or bare +polyrun+ for that).
63
61
  def cmd_parallel_quick(argv, config_path)
64
- cfg = Polyrun::Config.load(path: config_path || ENV["POLYRUN_CONFIG"])
65
- code = start_bootstrap!(cfg, argv, config_path)
66
- return code if code != 0
67
-
68
62
  sep = argv.index("--")
69
63
  combined =
70
64
  if sep
@@ -24,49 +24,55 @@ module Polyrun
24
24
 
25
25
  pids = []
26
26
  Polyrun::WorkerOutput.prepare_log_dir! if Polyrun::WorkerOutput.routing_enabled?
27
- workers.times do |shard|
28
- paths = plan.shard(shard)
29
- if paths.empty?
30
- Polyrun::Log.warn "polyrun run-shards: shard #{shard} skipped (no paths)" if @verbose || parallel
31
- next
32
- end
27
+ begin
28
+ workers.times do |shard|
29
+ paths = plan.shard(shard)
30
+ if paths.empty?
31
+ Polyrun::Log.warn "polyrun run-shards: shard #{shard} skipped (no paths)" if @verbose || parallel
32
+ next
33
+ end
33
34
 
34
- env_shard = ENV.to_h.merge(
35
- "POLYRUN_HOOK_ORCHESTRATOR" => "1",
36
- "POLYRUN_SHARD_INDEX" => shard.to_s,
37
- "POLYRUN_SHARD_TOTAL" => workers.to_s
38
- )
39
- code = hook_cfg.run_phase_if_enabled(:before_shard, env_shard)
40
- if code != 0
41
- run_shards_terminate_children!(pids)
42
- return [pids, code]
43
- end
35
+ env_shard = ENV.to_h.merge(
36
+ "POLYRUN_HOOK_ORCHESTRATOR" => "1",
37
+ "POLYRUN_SHARD_INDEX" => shard.to_s,
38
+ "POLYRUN_SHARD_TOTAL" => workers.to_s
39
+ )
40
+ code = hook_cfg.run_phase_if_enabled(:before_shard, env_shard)
41
+ if code != 0
42
+ run_shards_terminate_children!(pids)
43
+ return [pids, code]
44
+ end
44
45
 
45
- child_env = shard_child_env(
46
- cfg: cfg,
47
- workers: workers,
48
- shard: shard,
49
- matrix_index: mx,
50
- matrix_total: mt,
51
- failure_fragments: ctx[:merge_failures],
52
- spec_quality_fragments: ctx[:merge_spec_quality]
53
- )
54
- child_env = child_env.merge("POLYRUN_HOOK_ORCHESTRATOR" => "0")
55
- child_env = hook_cfg.merge_worker_ruby_env(child_env)
46
+ child_env = shard_child_env(
47
+ cfg: cfg,
48
+ workers: workers,
49
+ shard: shard,
50
+ matrix_index: mx,
51
+ matrix_total: mt,
52
+ failure_fragments: ctx[:merge_failures],
53
+ spec_quality_fragments: ctx[:merge_spec_quality]
54
+ )
55
+ child_env = child_env.merge("POLYRUN_HOOK_ORCHESTRATOR" => "0")
56
+ child_env = hook_cfg.merge_worker_ruby_env(child_env)
56
57
 
57
- ping_path = run_shards_prepare_worker_ping!(ctx, child_env, shard)
58
+ ping_path = run_shards_prepare_worker_ping!(ctx, child_env, shard)
58
59
 
59
- Polyrun::Log.warn "polyrun run-shards: shard #{shard} → #{paths.size} file(s)" if @verbose
60
- spawned_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
61
- pid = run_shards_spawn_one_worker(child_env, cmd, paths, hook_cfg)
62
- pids << {pid: pid, shard: shard, spawned_at: spawned_at, ping_path: ping_path}
63
- Polyrun::Debug.log("[parent pid=#{$$}] run-shards: Process.spawn shard=#{shard} child_pid=#{pid} spec_files=#{paths.size}")
64
- Polyrun::Log.warn "polyrun run-shards: started shard #{shard} pid=#{pid} (#{paths.size} file(s))" if parallel
65
- if Polyrun::WorkerOutput.routing_enabled? && parallel
66
- Polyrun::Log.warn "polyrun run-shards: shard #{shard} log → #{Polyrun::WorkerOutput.log_path_for(shard)}"
60
+ Polyrun::Log.warn "polyrun run-shards: shard #{shard} → #{paths.size} file(s)" if @verbose
61
+ spawned_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
62
+ pid = run_shards_spawn_one_worker(child_env, cmd, paths, hook_cfg)
63
+ pids << {pid: pid, shard: shard, spawned_at: spawned_at, ping_path: ping_path}
64
+ Polyrun::Debug.log("[parent pid=#{$$}] run-shards: Process.spawn shard=#{shard} child_pid=#{pid} spec_files=#{paths.size}")
65
+ Polyrun::Log.warn "polyrun run-shards: started shard #{shard} pid=#{pid} (#{paths.size} file(s))" if parallel
66
+ if Polyrun::WorkerOutput.routing_enabled? && parallel
67
+ Polyrun::Log.warn "polyrun run-shards: shard #{shard} log → #{Polyrun::WorkerOutput.log_path_for(shard)}"
68
+ end
67
69
  end
70
+ [pids, nil]
71
+ rescue StandardError, ScriptError => e
72
+ Polyrun::Log.warn "polyrun run-shards: spawn failed (#{e.class}: #{e.message}); stopping #{pids.size} worker(s)"
73
+ run_shards_terminate_children!(pids)
74
+ [pids, 1]
68
75
  end
69
- [pids, nil]
70
76
  end
71
77
  # rubocop:enable Metrics/AbcSize
72
78
 
@@ -85,13 +91,20 @@ module Polyrun
85
91
  path
86
92
  end
87
93
 
94
+ # Workers start in their own process group so TERM/KILL can reach hook shell grandchildren.
88
95
  def run_shards_spawn_one_worker(child_env, cmd, paths, hook_cfg)
89
96
  return Polyrun::WorkerOutput.spawn_worker(child_env, cmd, paths, hook_cfg) if Polyrun::WorkerOutput.routing_enabled?
90
97
 
91
98
  if hook_cfg.worker_hooks? && !Polyrun::Hooks.disabled?
92
- Process.spawn(child_env, "sh", "-c", hook_cfg.build_worker_shell_script(cmd, paths))
99
+ Process.spawn(
100
+ child_env,
101
+ "sh",
102
+ "-c",
103
+ hook_cfg.build_worker_shell_script(cmd, paths),
104
+ pgroup: true
105
+ )
93
106
  else
94
- Process.spawn(child_env, *cmd, *paths)
107
+ Process.spawn(child_env, *cmd, *paths, pgroup: true)
95
108
  end
96
109
  end
97
110
  end
@@ -234,7 +234,7 @@ module Polyrun
234
234
  end
235
235
 
236
236
  def run_shards_force_stop_pid_status(pid)
237
- Process.kill(:KILL, pid)
237
+ run_shards_kill_process_group!(:KILL, pid)
238
238
  st = nil
239
239
  begin
240
240
  Process.wait(pid)
@@ -243,7 +243,7 @@ module Polyrun
243
243
  # child already reaped
244
244
  end
245
245
  st
246
- rescue Errno::ESRCH
246
+ rescue Errno::ESRCH, Errno::EPERM
247
247
  begin
248
248
  Process.wait(pid)
249
249
  $?
@@ -33,6 +33,9 @@ module Polyrun
33
33
  items, paths_source, err = run_shards_resolve_items(o[:paths_file], pc)
34
34
  return [err, nil] if err
35
35
 
36
+ err = run_shards_validate_suite_vs_command!(items, cmd)
37
+ return [err, nil] if err
38
+
36
39
  costs, strategy, err = run_shards_resolve_costs(
37
40
  o[:timing_path],
38
41
  o[:strategy],
@@ -64,39 +64,26 @@ module Polyrun
64
64
  Polyrun::Log.warn "polyrun run-shards: no paths (empty paths file or list)"
65
65
  return [nil, nil, 2]
66
66
  end
67
+
67
68
  [items, paths_source, nil]
68
69
  end
69
70
 
71
+ def run_shards_validate_suite_vs_command!(items, cmd)
72
+ msg = Polyrun::Partition::Suite.command_mismatch_message(items, cmd)
73
+ return nil unless msg
74
+
75
+ Polyrun::Log.warn "polyrun run-shards: #{msg}"
76
+ 2
77
+ end
78
+
70
79
  def run_shards_resolve_costs(timing_path, strategy, timing_granularity, strategy_explicit: false)
71
- strategy = strategy.to_s
72
- if timing_path
73
- if strategy_explicit && strategy == "round_robin"
74
- return [nil, strategy, nil]
75
- end
76
-
77
- costs = Polyrun::Partition::Plan.load_timing_costs(
78
- File.expand_path(timing_path.to_s, Dir.pwd),
79
- granularity: timing_granularity
80
- )
81
- if costs.empty?
82
- Polyrun::Log.warn "polyrun run-shards: timing file missing or empty: #{timing_path}"
83
- return [nil, nil, 2]
84
- end
85
- if Polyrun::Partition::Plan.timing_load_strategy?(strategy)
86
- return [costs, strategy, nil]
87
- end
88
- unless strategy_explicit
89
- Polyrun::Log.warn "polyrun run-shards: using cost_binpack (timing data present)" if @verbose
90
- return [costs, "cost_binpack", nil]
91
- end
92
-
93
- [nil, strategy, nil]
94
- elsif Polyrun::Partition::Plan.cost_strategy?(strategy) || Polyrun::Partition::Plan.lazy_robin_strategy?(strategy)
95
- Polyrun::Log.warn "polyrun run-shards: --timing or partition.timing_file required for strategy #{strategy}"
96
- [nil, nil, 2]
97
- else
98
- [nil, strategy, nil]
99
- end
80
+ partition_resolve_costs_and_strategy(
81
+ timing_path,
82
+ strategy,
83
+ timing_granularity,
84
+ strategy_explicit: strategy_explicit,
85
+ log_prefix: "polyrun run-shards"
86
+ )
100
87
  end
101
88
 
102
89
  def run_shards_make_plan(items, workers, strategy, seed, costs, constraints, timing_granularity, stable_assignment = nil, shard_weights: nil)
@@ -127,7 +127,7 @@ module Polyrun
127
127
  return if Polyrun::WorkerOutput.routing_enabled?
128
128
 
129
129
  Polyrun::Log.warn "polyrun run-shards: #{pid_count} children running; RSpec output below may be interleaved."
130
- Polyrun::Log.warn "polyrun run-shards: each worker prints its own summary line; the last \"N examples\" line is not a total across shards."
130
+ Polyrun::Log.warn "polyrun run-shards: RSpec summary counts apply to individual shards."
131
131
  end
132
132
 
133
133
  def run_shards_merge_or_hint_coverage(ctx)