polyrun 2.2.4 → 2.2.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +28 -0
- data/README.md +4 -1
- data/docs/SETUP_PROFILE.md +1 -0
- data/lib/polyrun/cli/database_commands.rb +21 -7
- data/lib/polyrun/cli/default_run.rb +28 -6
- data/lib/polyrun/cli/help.rb +4 -2
- data/lib/polyrun/cli/helpers.rb +41 -6
- data/lib/polyrun/cli/plan_command.rb +10 -29
- data/lib/polyrun/cli/run_queue_command.rb +10 -1
- data/lib/polyrun/cli/run_shards_command.rb +2 -8
- data/lib/polyrun/cli/run_shards_parallel_children.rb +52 -39
- data/lib/polyrun/cli/run_shards_parallel_wait.rb +2 -2
- data/lib/polyrun/cli/run_shards_plan_boot_phases.rb +3 -0
- data/lib/polyrun/cli/run_shards_planning.rb +16 -29
- data/lib/polyrun/cli/run_shards_run.rb +1 -1
- data/lib/polyrun/cli/run_shards_worker_interrupt.rb +16 -14
- data/lib/polyrun/cli/start_bootstrap.rb +51 -22
- data/lib/polyrun/cli.rb +8 -1
- data/lib/polyrun/config.rb +9 -4
- data/lib/polyrun/coverage/example_diff.rb +6 -7
- data/lib/polyrun/coverage/merge/formatters_html.rb +20 -2
- data/lib/polyrun/coverage/merge.rb +9 -3
- data/lib/polyrun/coverage/merge_fragment_meta.rb +6 -1
- data/lib/polyrun/coverage/rails.rb +1 -1
- data/lib/polyrun/database/clone_shards.rb +28 -5
- data/lib/polyrun/database/provision.rb +15 -9
- data/lib/polyrun/database/url_builder/connection/infer.rb +4 -0
- data/lib/polyrun/database/url_builder.rb +31 -3
- data/lib/polyrun/partition/paths.rb +23 -4
- data/lib/polyrun/partition/plan.rb +6 -24
- data/lib/polyrun/partition/suite.rb +116 -0
- data/lib/polyrun/partition/timing_diagnostics.rb +2 -3
- data/lib/polyrun/queue/file_store.rb +133 -30
- data/lib/polyrun/queue/file_store_intent.rb +188 -0
- data/lib/polyrun/queue/worker_loop.rb +41 -7
- data/lib/polyrun/spec_quality/config.rb +1 -1
- data/lib/polyrun/spec_quality/profile.rb +0 -1
- data/lib/polyrun/spec_quality/report.rb +16 -7
- data/lib/polyrun/templates/rails_prepare.polyrun.yml +3 -1
- data/lib/polyrun/version.rb +1 -1
- data/lib/polyrun/worker_output.rb +30 -13
- data/lib/polyrun.rb +1 -0
- metadata +3 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 69dd2432a149cc9790756d2d54b6481e898604a7ae84219b8b5cfbd65997e231
|
|
4
|
+
data.tar.gz: ab1ea9013d606dd53e717cc421e353c668b47382871e3494f03fa689333e422c
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 70c45810383e893e6d4ed8a9b2288f3540bb85e37345149283cd9992e2f23c5fa1d475b9e43766d20ddc6f93eaae562c258ff99769284bf606aa2858bbaae515
|
|
7
|
+
data.tar.gz: 17a14844465eade2840dcc968d1c432e4462991ac099831cd03cd19f262aec9b42d5bd129efcc8e0b843534357ae281380507efb7c40f530e772a7bcb78d9f40
|
data/CHANGELOG.md
CHANGED
|
@@ -2,6 +2,34 @@
|
|
|
2
2
|
|
|
3
3
|
## Unreleased
|
|
4
4
|
|
|
5
|
+
## 2.2.6 (2026-10-29)
|
|
6
|
+
|
|
7
|
+
- Fix `polyrun start` auto database provision when `databases:` uses a non-Postgres adapter (`mysql2`, trilogy, …): skip CloneShards with a clear warning (or exit 1 if `start.databases: true`); keep `template_db` / `shard_db_pattern` for URL shaping.
|
|
8
|
+
- Fail `db:clone-shards` / `db:setup-shard` before `psql` when the inferred adapter is not PostgreSQL.
|
|
9
|
+
- Pass `postgresql:` host, port, username, and password from `polyrun.yml` into `psql` during shard provision (password via child `PGPASSWORD`).
|
|
10
|
+
- Fix implicit path runs so tokens after `--` stay the worker command, and `--paths-file` is passed before `--`.
|
|
11
|
+
- Accept `polyrun start --workers=N` (equals form) so shard DB provision matches worker count.
|
|
12
|
+
- Reject `databases.shard_db_pattern` without `%{shard}` / `%<shard>d`, and refuse patterns that resolve to `template_db`.
|
|
13
|
+
- Redact database passwords in `db:*` / CloneShards dry-run shell logs.
|
|
14
|
+
- Raise when an explicit `-c` / `POLYRUN_CONFIG` path is missing; map `Polyrun::Error` to exit 2.
|
|
15
|
+
- Validate `run-queue --workers` (1..max) and exit forked queue workers with `exit!`.
|
|
16
|
+
- Fix SpecQuality `minimum_unique_lines_per_example` gate for thresholds above 1; keep only positive or CPU profile deltas.
|
|
17
|
+
- Align file-granularity timing cost keys with realpath normalization so cost binpack hits under symlink roots.
|
|
18
|
+
- TERM shard workers before draining routed worker output on SIGINT/SIGTERM; skip empty queue leases.
|
|
19
|
+
- Recover interrupted queue claim/ack/reclaim via `intent.json`; age leases by `heartbeat_at`; quarantine poison paths after `POLYRUN_QUEUE_MAX_ATTEMPTS` (default 3).
|
|
20
|
+
- On a failed `run-queue` batch, re-run paths one-by-one and ack successes so a poison path does not requeue neighbors.
|
|
21
|
+
- Spawn shard workers in their own process group and signal the group on timeout/interrupt; abort already-started workers if a later spawn fails.
|
|
22
|
+
- Cache coverage HTML ERB templates; fold merge fragments without retaining full JSON documents; share plan/run-shards timing cost resolution.
|
|
23
|
+
- Clarify `run-shards` interleaved-output notice: RSpec summary counts apply to individual shards (no false per-worker total).
|
|
24
|
+
|
|
25
|
+
## 2.2.5 (2026-09-24)
|
|
26
|
+
|
|
27
|
+
- Fix bare `polyrun` suite selection when both `spec/` and `test/` exist: honor `partition.suite`, else infer from `partition.paths_file`, else filesystem auto-detect (RSpec first).
|
|
28
|
+
- Exit 2 when `partition.suite` disagrees with `paths_file`, or when `run-shards` paths infer a different suite than a recognizable command after `--`.
|
|
29
|
+
- Add `parallel-minitest` and `parallel-quick` CLI subcommands (same shape as `parallel-rspec`); they do not run `start` prepare or database bootstrap.
|
|
30
|
+
- Recognize `bin/rails test` and compact `ruby -Itest` in the suite/command mismatch guard; strip `path:line` locators before classifying paths.
|
|
31
|
+
- Skip legacy `spec/spec_paths.txt` when `partition.suite` is `minitest` or `quick`; allow mixed path lists only with suite-agnostic custom runners.
|
|
32
|
+
|
|
5
33
|
## 2.2.4 (2026-09-04)
|
|
6
34
|
|
|
7
35
|
- Fix `DEBUG_SQL` / `POLYRUN_DEBUG_SQL` raising `NoMethodError` for `loggable_sql?` when the SQL subscriber runs on an RSpec example group.
|
data/README.md
CHANGED
|
@@ -10,7 +10,7 @@ Running tests in parallel across processes still requires a single merged covera
|
|
|
10
10
|
|
|
11
11
|
Polyrun provides:
|
|
12
12
|
|
|
13
|
-
- Orchestration: `plan`, `run-shards`, and `parallel-rspec` (run-shards plus merge-coverage), with an optional on-disk queue and constraints for file lists and load balancing. For **GitHub Actions-style matrix sharding** (one job per global shard), use `ci-shard-run -- …` (any test runner) or `ci-shard-rspec`—not `run-shards` / `parallel
|
|
13
|
+
- Orchestration: `plan`, `run-shards`, and `parallel-rspec` / `parallel-minitest` / `parallel-quick` (run-shards plus merge-coverage), with an optional on-disk queue and constraints for file lists and load balancing. Bare `polyrun` (no subcommand) picks the suite from `partition.suite`, else `partition.paths_file`, else filesystem auto-detect. For **GitHub Actions-style matrix sharding** (one job per global shard), use `ci-shard-run -- …` (any test runner) or `ci-shard-rspec`—not `run-shards` / `parallel-*`, which fan out N workers on one machine.
|
|
14
14
|
- Coverage: merge SimpleCov-compatible JSON fragments; emit JSON, LCOV, Cobertura, or console summaries (you can drop separate SimpleCov merge plugins for this path).
|
|
15
15
|
- CI reporting: JUnit XML from RSpec JSON; slow-file reports from merged timing JSON.
|
|
16
16
|
- Parallel hygiene: asset digest markers, SQL snapshots, YAML fixture batches, and DB URL or shard helpers aligned with `POLYRUN_SHARD_*`.
|
|
@@ -92,6 +92,8 @@ bin/polyrun build-paths # write spec/spec_paths.txt from partition.paths_build
|
|
|
92
92
|
bin/polyrun ci-shard-run -- bundle exec rspec # CI matrix: shard plan + append paths to the command after --
|
|
93
93
|
bin/polyrun ci-shard-rspec # same as ci-shard-run -- bundle exec rspec
|
|
94
94
|
bin/polyrun parallel-rspec --workers 5 # run-shards + merge-coverage (default: bundle exec rspec)
|
|
95
|
+
bin/polyrun parallel-minitest --workers 5 # same; default rails test or ruby -I test
|
|
96
|
+
bin/polyrun parallel-quick --workers 5 # same; default bundle exec polyrun quick
|
|
95
97
|
bin/polyrun run-shards --workers 5 --merge-coverage -- bundle exec rspec
|
|
96
98
|
bin/polyrun merge-coverage -i cov1.json -i cov2.json -o merged.json --format json,lcov,cobertura,console
|
|
97
99
|
bin/polyrun env --shard 0 --total 4 # print DATABASE_URL exports from polyrun.yml in cwd
|
|
@@ -145,6 +147,7 @@ That single require loads the CLI and core library **without** loading RSpec or
|
|
|
145
147
|
| `Polyrun::Prepare::Assets` | Digest trees, marker file, `assets:precompile`. |
|
|
146
148
|
| `Polyrun::Database::Shard` | Shard env map, `%{shard}` DB names, URL path suffix for `postgres://`, `mysql2://`, `mongodb://`, etc. |
|
|
147
149
|
| `Polyrun::Database::UrlBuilder` | URLs from `polyrun.yml` `databases:` — nested blocks or `adapter:` for common Rails stacks (`postgresql`, `mysql`/`mysql2`, `trilogy`, `sqlserver`/`mssql`, `sqlite3`/`sqlite`, `mongodb`/`mongo`). |
|
|
150
|
+
| `Polyrun::Database::CloneShards` / `Provision` | PostgreSQL-only template + shard `CREATE DATABASE … TEMPLATE` via `psql`. Non-Postgres configs still use UrlBuilder for env; set `start.databases: false` and provision shards in prepare. |
|
|
148
151
|
| `Polyrun::Hooks` | Load from `Config#hooks`; `run_phase` / `run_phase_if_enabled`; `build_worker_shell_script` wraps the worker command. |
|
|
149
152
|
| `Polyrun::Hooks::Dsl` | Ruby hook file (`hooks.ruby`); `before(:suite)` / `after(:each)` etc. in `config/polyrun_hooks.rb` (see README). |
|
|
150
153
|
|
data/docs/SETUP_PROFILE.md
CHANGED
|
@@ -25,6 +25,7 @@ Do not mix “fan out N workers inside one job” with “matrix shard index”
|
|
|
25
25
|
|--------|---------|
|
|
26
26
|
| Single test DB | Omitting `databases:` in `polyrun.yml` may be enough; ensure parallel workers do not share one DB if they mutate data |
|
|
27
27
|
| Multi-DB or shard suffixes | `databases:` in `polyrun.yml` (`shard_db_pattern`, `template_db`, optional `connections` and `env_key`). `database.yml` or `DATABASE_URL` may use `%{shard}` or `POLYRUN_SHARD_INDEX` suffixes—the same convention as `polyrun env` |
|
|
28
|
+
| Non-Postgres adapters (mysql2, trilogy, …) | `UrlBuilder` still emits URLs from `template_db` / `shard_db_pattern`. Auto provision on `polyrun start` and `db:clone-shards` / `db:setup-shard` are PostgreSQL-only (`CREATE DATABASE … TEMPLATE`). For MySQL/MariaDB, set `start.databases: false` (or rely on the skip warning) and provision shards in `prepare` |
|
|
28
29
|
| External provisioning | Sometimes `db:prepare` plus shell clone scripts (multi-DB apps)—document ordering: prepare databases before `run-shards` |
|
|
29
30
|
|
|
30
31
|
## 4. Prepare (run once before workers)
|
|
@@ -47,26 +47,32 @@ module Polyrun
|
|
|
47
47
|
def cmd_db_setup_shard(argv, config_path)
|
|
48
48
|
dry = db_setup_shard_parse_options!(argv)
|
|
49
49
|
cfg = Polyrun::Config.load(path: config_path || ENV["POLYRUN_CONFIG"])
|
|
50
|
-
|
|
51
|
-
if !
|
|
50
|
+
databases = cfg.databases
|
|
51
|
+
if !databases.is_a?(Hash) || databases.empty?
|
|
52
52
|
Polyrun::Log.warn "db:setup-shard: configure databases: in polyrun.yml"
|
|
53
53
|
return 2
|
|
54
54
|
end
|
|
55
55
|
|
|
56
|
+
unless Polyrun::Database::UrlBuilder::ConnectionInfer.postgresql?(databases)
|
|
57
|
+
adapter = Polyrun::Database::UrlBuilder::ConnectionInfer.infer_adapter_name(databases)
|
|
58
|
+
Polyrun::Log.warn "db:setup-shard: PostgreSQL-only (databases adapter is #{adapter})"
|
|
59
|
+
return 2
|
|
60
|
+
end
|
|
61
|
+
|
|
56
62
|
shard = resolve_shard_index(cfg.partition)
|
|
57
|
-
template =
|
|
63
|
+
template = databases["template_db"] || databases[:template_db]
|
|
58
64
|
if !template
|
|
59
65
|
Polyrun::Log.warn "db:setup-shard: set databases.template_db"
|
|
60
66
|
return 2
|
|
61
67
|
end
|
|
62
68
|
|
|
63
|
-
plan = Polyrun::Database::UrlBuilder.shard_database_plan(
|
|
69
|
+
plan = Polyrun::Database::UrlBuilder.shard_database_plan(databases, shard_index: shard)
|
|
64
70
|
if plan.empty?
|
|
65
71
|
Polyrun::Log.warn "db:setup-shard: could not derive shard database names from polyrun.yml"
|
|
66
72
|
return 2
|
|
67
73
|
end
|
|
68
74
|
|
|
69
|
-
db_setup_shard_run_plan(plan, dry: dry)
|
|
75
|
+
db_setup_shard_run_plan(plan, databases, dry: dry)
|
|
70
76
|
end
|
|
71
77
|
|
|
72
78
|
def db_setup_shard_parse_options!(argv)
|
|
@@ -77,7 +83,7 @@ module Polyrun
|
|
|
77
83
|
dry
|
|
78
84
|
end
|
|
79
85
|
|
|
80
|
-
def db_setup_shard_run_plan(plan, dry:)
|
|
86
|
+
def db_setup_shard_run_plan(plan, databases, dry:)
|
|
81
87
|
if dry
|
|
82
88
|
plan.each do |row|
|
|
83
89
|
Polyrun::Log.warn "would: CREATE DATABASE #{row[:new_db]} TEMPLATE #{row[:template_db]}"
|
|
@@ -85,11 +91,19 @@ module Polyrun
|
|
|
85
91
|
return 0
|
|
86
92
|
end
|
|
87
93
|
|
|
94
|
+
conn = Polyrun::Database::UrlBuilder::Connection.resolve_connection(databases)
|
|
95
|
+
opts = {
|
|
96
|
+
host: conn[:host],
|
|
97
|
+
port: conn[:port],
|
|
98
|
+
username: conn[:user],
|
|
99
|
+
password: conn[:password]
|
|
100
|
+
}
|
|
88
101
|
threads = plan.map do |row|
|
|
89
102
|
Thread.new do
|
|
90
103
|
Polyrun::Database::Provision.create_database_from_template!(
|
|
91
104
|
new_db: row[:new_db],
|
|
92
|
-
template_db: row[:template_db].to_s
|
|
105
|
+
template_db: row[:template_db].to_s,
|
|
106
|
+
**opts
|
|
93
107
|
)
|
|
94
108
|
end
|
|
95
109
|
end
|
|
@@ -7,20 +7,32 @@ module Polyrun
|
|
|
7
7
|
private
|
|
8
8
|
|
|
9
9
|
def dispatch_default_parallel!(config_path)
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
10
|
+
cfg = Polyrun::Config.load(path: config_path || ENV["POLYRUN_CONFIG"])
|
|
11
|
+
pc = cfg.partition
|
|
12
|
+
code = Polyrun::Partition::PathsBuild.apply!(partition: pc, cwd: Dir.pwd)
|
|
13
|
+
return code if code != 0
|
|
14
|
+
|
|
15
|
+
resolved = Polyrun::Partition::Suite.resolve_default(cwd: Dir.pwd, partition: pc)
|
|
16
|
+
if resolved[:error]
|
|
17
|
+
Polyrun::Log.warn "polyrun: #{resolved[:error]}"
|
|
13
18
|
return 2
|
|
14
19
|
end
|
|
15
20
|
|
|
21
|
+
suite = resolved[:suite]
|
|
16
22
|
Polyrun::Log.warn "polyrun: default → parallel #{suite} (use `polyrun help` for subcommands)" if @verbose
|
|
17
23
|
|
|
18
24
|
case suite
|
|
19
25
|
when :rspec
|
|
20
26
|
cmd_start([], config_path)
|
|
21
27
|
when :minitest
|
|
28
|
+
code = start_bootstrap!(cfg, [], config_path)
|
|
29
|
+
return code if code != 0
|
|
30
|
+
|
|
22
31
|
cmd_parallel_minitest([], config_path)
|
|
23
32
|
when :quick
|
|
33
|
+
code = start_bootstrap!(cfg, [], config_path)
|
|
34
|
+
return code if code != 0
|
|
35
|
+
|
|
24
36
|
cmd_parallel_quick([], config_path)
|
|
25
37
|
else
|
|
26
38
|
2
|
|
@@ -48,8 +60,7 @@ module Polyrun
|
|
|
48
60
|
end
|
|
49
61
|
|
|
50
62
|
def dispatch_implicit_parallel_targets!(argv, config_path)
|
|
51
|
-
path_tokens
|
|
52
|
-
head = argv.reject { |a| cli_implicit_path_token?(a) }
|
|
63
|
+
path_tokens, head, after = split_implicit_arguments(argv)
|
|
53
64
|
expanded = expand_implicit_target_paths(path_tokens)
|
|
54
65
|
if expanded.empty?
|
|
55
66
|
Polyrun::Log.warn "polyrun: no files matched path arguments"
|
|
@@ -70,7 +81,8 @@ module Polyrun
|
|
|
70
81
|
begin
|
|
71
82
|
tmp.write(expanded.join("\n") + "\n")
|
|
72
83
|
tmp.close
|
|
73
|
-
|
|
84
|
+
# Keep --paths-file before `--` so OptionParser sees it; after `--` it becomes a worker argv token.
|
|
85
|
+
combined = head + ["--paths-file", tmp.path] + after
|
|
74
86
|
case suite
|
|
75
87
|
when :rspec
|
|
76
88
|
cmd_start(combined, config_path)
|
|
@@ -91,6 +103,16 @@ module Polyrun
|
|
|
91
103
|
end
|
|
92
104
|
end
|
|
93
105
|
|
|
106
|
+
def split_implicit_arguments(argv)
|
|
107
|
+
# Only tokens before `--` are suite paths; everything from `--` onward is the worker command.
|
|
108
|
+
sep = argv.index("--")
|
|
109
|
+
before = sep ? argv[0...sep] : argv
|
|
110
|
+
after = sep ? argv[sep..] : []
|
|
111
|
+
path_tokens = before.select { |a| cli_implicit_path_token?(a) }
|
|
112
|
+
head = before.reject { |a| cli_implicit_path_token?(a) }
|
|
113
|
+
[path_tokens, head, after]
|
|
114
|
+
end
|
|
115
|
+
|
|
94
116
|
def expand_implicit_target_paths(path_tokens)
|
|
95
117
|
path_tokens.flat_map do |p|
|
|
96
118
|
abs = File.expand_path(p)
|
data/lib/polyrun/cli/help.rb
CHANGED
|
@@ -5,7 +5,7 @@ module Polyrun
|
|
|
5
5
|
Polyrun::Log.puts <<~HELP
|
|
6
6
|
usage: polyrun [global options] [<command> | <paths...>]
|
|
7
7
|
|
|
8
|
-
With no command, runs parallel tests for the
|
|
8
|
+
With no command, runs parallel tests for the resolved suite: partition.suite when set; else suite inferred from partition.paths_file; else filesystem auto-detect (RSpec under spec/ before Minitest under test/, then Polyrun Quick). If the first argument is a known subcommand name, it is dispatched. Otherwise, path-like tokens (optionally with run-shards flags such as --workers) shard those files in parallel; see commands below.
|
|
9
9
|
|
|
10
10
|
global:
|
|
11
11
|
-c, --config PATH polyrun.yml path (or POLYRUN_CONFIG)
|
|
@@ -40,7 +40,9 @@ module Polyrun
|
|
|
40
40
|
merge-coverage merge SimpleCov JSON fragments (json/lcov/cobertura/console/html/csv/markdown)
|
|
41
41
|
merge-failures merge per-shard failure JSONL fragments or RSpec JSON files (jsonl/json/csv/markdown)
|
|
42
42
|
run-shards fan out N parallel OS processes (POLYRUN_SHARD_*; not Ruby threads); optional --merge-coverage / --merge-failures / --merge-spec-quality
|
|
43
|
-
parallel-rspec run-shards + merge-coverage (defaults to: bundle exec rspec after --)
|
|
43
|
+
parallel-rspec run-shards + merge-coverage (defaults to: bundle exec rspec after --); no start prepare/DB bootstrap
|
|
44
|
+
parallel-minitest run-shards + merge-coverage (defaults to: bundle exec rails test or ruby -I test after --); no start prepare/DB bootstrap
|
|
45
|
+
parallel-quick run-shards + merge-coverage (defaults to: bundle exec polyrun quick after --); no start prepare/DB bootstrap
|
|
44
46
|
start parallel-rspec; auto-runs prepare (shell/assets) and db:setup-* when polyrun.yml configures them; legacy script/build_spec_paths.rb if paths_build absent
|
|
45
47
|
ci-shard-run CI matrix: build-paths + plan for POLYRUN_SHARD_INDEX / POLYRUN_SHARD_TOTAL (or config), then run your command with that shard's paths after --; optional --shard-processes M or --workers M (POLYRUN_SHARD_PROCESSES; not POLYRUN_WORKERS) for N×M jobs × processes on this host
|
|
46
48
|
ci-shard-rspec same as ci-shard-run -- bundle exec rspec; optional --shard-processes / --workers / -- [rspec-only flags]
|
data/lib/polyrun/cli/helpers.rb
CHANGED
|
@@ -11,6 +11,46 @@ module Polyrun
|
|
|
11
11
|
Polyrun::Config::Resolver.env_int(name, fallback)
|
|
12
12
|
end
|
|
13
13
|
|
|
14
|
+
# Shared timing → costs/strategy resolution for +plan+ and +run-shards+.
|
|
15
|
+
# @return [Array] +[costs, strategy, error_code]+ where +error_code+ is nil on success
|
|
16
|
+
def partition_resolve_costs_and_strategy(
|
|
17
|
+
timing_path,
|
|
18
|
+
strategy,
|
|
19
|
+
timing_granularity,
|
|
20
|
+
strategy_explicit: false,
|
|
21
|
+
log_prefix: "polyrun"
|
|
22
|
+
)
|
|
23
|
+
strategy = strategy.to_s
|
|
24
|
+
if timing_path
|
|
25
|
+
if strategy_explicit && strategy == "round_robin"
|
|
26
|
+
return [nil, strategy, nil]
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
costs = Polyrun::Partition::Plan.load_timing_costs(
|
|
30
|
+
File.expand_path(timing_path.to_s, Dir.pwd),
|
|
31
|
+
granularity: timing_granularity
|
|
32
|
+
)
|
|
33
|
+
if costs.empty?
|
|
34
|
+
Polyrun::Log.warn "#{log_prefix}: timing file missing or empty: #{timing_path}"
|
|
35
|
+
return [nil, nil, 2]
|
|
36
|
+
end
|
|
37
|
+
if Polyrun::Partition::Plan.timing_load_strategy?(strategy)
|
|
38
|
+
return [costs, strategy, nil]
|
|
39
|
+
end
|
|
40
|
+
unless strategy_explicit
|
|
41
|
+
Polyrun::Log.warn "#{log_prefix}: using cost_binpack (timing data present)" if @verbose
|
|
42
|
+
return [costs, "cost_binpack", nil]
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
[nil, strategy, nil]
|
|
46
|
+
elsif Polyrun::Partition::Plan.cost_strategy?(strategy) || Polyrun::Partition::Plan.lazy_robin_strategy?(strategy)
|
|
47
|
+
Polyrun::Log.warn "#{log_prefix}: --timing or partition.timing_file required for strategy #{strategy}"
|
|
48
|
+
[nil, nil, 2]
|
|
49
|
+
else
|
|
50
|
+
[nil, strategy, nil]
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
|
|
14
54
|
# Per-worker wall clock (from spawn) for run-shards / ci-shard fan-out; unset or invalid means no limit.
|
|
15
55
|
def env_worker_timeout_sec
|
|
16
56
|
s = ENV["POLYRUN_WORKER_TIMEOUT_SEC"].to_s.strip
|
|
@@ -97,12 +137,7 @@ module Polyrun
|
|
|
97
137
|
# {Partition::Plan#default_weight} semantics: mean of known timing costs for missing paths.
|
|
98
138
|
def queue_weight_for(path, costs, default_weight = nil, granularity: :file)
|
|
99
139
|
g = Polyrun::Partition::TimingKeys.normalize_granularity(granularity)
|
|
100
|
-
key =
|
|
101
|
-
if g == :example
|
|
102
|
-
Polyrun::Partition::TimingKeys.normalize_locator(path.to_s, Dir.pwd, :example)
|
|
103
|
-
else
|
|
104
|
-
File.expand_path(path.to_s, Dir.pwd)
|
|
105
|
-
end
|
|
140
|
+
key = Polyrun::Partition::TimingKeys.normalize_locator(path.to_s, Dir.pwd, g)
|
|
106
141
|
return costs[key] if costs.key?(key)
|
|
107
142
|
|
|
108
143
|
unless default_weight.nil?
|
|
@@ -167,35 +167,16 @@ module Polyrun
|
|
|
167
167
|
end
|
|
168
168
|
|
|
169
169
|
def plan_load_costs_and_strategy(timing_path, strategy, timing_granularity, strategy_explicit: false)
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
if costs.empty?
|
|
181
|
-
Polyrun::Log.warn "polyrun plan: timing file missing or has no entries: #{timing_path}"
|
|
182
|
-
return nil
|
|
183
|
-
end
|
|
184
|
-
if Polyrun::Partition::Plan.timing_load_strategy?(strategy)
|
|
185
|
-
return [costs, strategy]
|
|
186
|
-
end
|
|
187
|
-
unless strategy_explicit
|
|
188
|
-
Polyrun::Log.warn "polyrun plan: using cost_binpack (timing data present)" if @verbose
|
|
189
|
-
return [costs, "cost_binpack"]
|
|
190
|
-
end
|
|
191
|
-
|
|
192
|
-
[nil, strategy]
|
|
193
|
-
elsif Polyrun::Partition::Plan.cost_strategy?(strategy) || Polyrun::Partition::Plan.lazy_robin_strategy?(strategy)
|
|
194
|
-
Polyrun::Log.warn "polyrun plan: --timing or partition.timing_file required for strategy #{strategy}"
|
|
195
|
-
nil
|
|
196
|
-
else
|
|
197
|
-
[nil, strategy]
|
|
198
|
-
end
|
|
170
|
+
costs, resolved, err = partition_resolve_costs_and_strategy(
|
|
171
|
+
timing_path,
|
|
172
|
+
strategy,
|
|
173
|
+
timing_granularity,
|
|
174
|
+
strategy_explicit: strategy_explicit,
|
|
175
|
+
log_prefix: "polyrun plan"
|
|
176
|
+
)
|
|
177
|
+
return nil if err
|
|
178
|
+
|
|
179
|
+
[costs, resolved]
|
|
199
180
|
end
|
|
200
181
|
end
|
|
201
182
|
end
|
|
@@ -43,6 +43,14 @@ module Polyrun
|
|
|
43
43
|
end.parse!(head)
|
|
44
44
|
|
|
45
45
|
workers ||= env_int("POLYRUN_WORKERS", Polyrun::Config::DEFAULT_PARALLEL_WORKERS)
|
|
46
|
+
unless workers.is_a?(Integer) && workers.between?(1, Polyrun::Config::MAX_PARALLEL_WORKERS)
|
|
47
|
+
Polyrun::Log.warn "polyrun run-queue: --workers must be between 1 and #{Polyrun::Config::MAX_PARALLEL_WORKERS}"
|
|
48
|
+
return 2
|
|
49
|
+
end
|
|
50
|
+
unless batch.is_a?(Integer) && batch >= 1
|
|
51
|
+
Polyrun::Log.warn "polyrun run-queue: --batch must be >= 1"
|
|
52
|
+
return 2
|
|
53
|
+
end
|
|
46
54
|
paths_file ||= pc["paths_file"] || pc[:paths_file]
|
|
47
55
|
unless paths_file
|
|
48
56
|
Polyrun::Log.warn "polyrun run-queue: need --paths-file or partition.paths_file"
|
|
@@ -87,7 +95,8 @@ module Polyrun
|
|
|
87
95
|
result = Polyrun::Queue::WorkerLoop.run(
|
|
88
96
|
store: store, worker_id: wid, batch: batch, cmd: cmd, on_failure: on_failure
|
|
89
97
|
)
|
|
90
|
-
exit
|
|
98
|
+
# exit! skips parent at_exit hooks (Coverage finish, RSpec autorun) loaded in this process.
|
|
99
|
+
exit! result[:exit_code]
|
|
91
100
|
end
|
|
92
101
|
pids << {pid: pid, worker_id: wid}
|
|
93
102
|
end
|
|
@@ -40,11 +40,8 @@ module Polyrun
|
|
|
40
40
|
end
|
|
41
41
|
|
|
42
42
|
# Same as parallel-rspec but runs +bundle exec rails test+ or +bundle exec ruby -I test+ after +--+.
|
|
43
|
+
# Does not run +start+ prepare/database bootstrap (use +polyrun start+ or bare +polyrun+ for that).
|
|
43
44
|
def cmd_parallel_minitest(argv, config_path)
|
|
44
|
-
cfg = Polyrun::Config.load(path: config_path || ENV["POLYRUN_CONFIG"])
|
|
45
|
-
code = start_bootstrap!(cfg, argv, config_path)
|
|
46
|
-
return code if code != 0
|
|
47
|
-
|
|
48
45
|
sep = argv.index("--")
|
|
49
46
|
combined =
|
|
50
47
|
if sep
|
|
@@ -60,11 +57,8 @@ module Polyrun
|
|
|
60
57
|
|
|
61
58
|
# Same as parallel-rspec but runs +bundle exec polyrun quick+ after +--+ (one Quick process per shard).
|
|
62
59
|
# Run from the app root with +bundle exec+ so workers resolve the same gem as the parent (same concern as +bundle exec rspec+).
|
|
60
|
+
# Does not run +start+ prepare/database bootstrap (use +polyrun start+ or bare +polyrun+ for that).
|
|
63
61
|
def cmd_parallel_quick(argv, config_path)
|
|
64
|
-
cfg = Polyrun::Config.load(path: config_path || ENV["POLYRUN_CONFIG"])
|
|
65
|
-
code = start_bootstrap!(cfg, argv, config_path)
|
|
66
|
-
return code if code != 0
|
|
67
|
-
|
|
68
62
|
sep = argv.index("--")
|
|
69
63
|
combined =
|
|
70
64
|
if sep
|
|
@@ -24,49 +24,55 @@ module Polyrun
|
|
|
24
24
|
|
|
25
25
|
pids = []
|
|
26
26
|
Polyrun::WorkerOutput.prepare_log_dir! if Polyrun::WorkerOutput.routing_enabled?
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
27
|
+
begin
|
|
28
|
+
workers.times do |shard|
|
|
29
|
+
paths = plan.shard(shard)
|
|
30
|
+
if paths.empty?
|
|
31
|
+
Polyrun::Log.warn "polyrun run-shards: shard #{shard} skipped (no paths)" if @verbose || parallel
|
|
32
|
+
next
|
|
33
|
+
end
|
|
33
34
|
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
35
|
+
env_shard = ENV.to_h.merge(
|
|
36
|
+
"POLYRUN_HOOK_ORCHESTRATOR" => "1",
|
|
37
|
+
"POLYRUN_SHARD_INDEX" => shard.to_s,
|
|
38
|
+
"POLYRUN_SHARD_TOTAL" => workers.to_s
|
|
39
|
+
)
|
|
40
|
+
code = hook_cfg.run_phase_if_enabled(:before_shard, env_shard)
|
|
41
|
+
if code != 0
|
|
42
|
+
run_shards_terminate_children!(pids)
|
|
43
|
+
return [pids, code]
|
|
44
|
+
end
|
|
44
45
|
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
46
|
+
child_env = shard_child_env(
|
|
47
|
+
cfg: cfg,
|
|
48
|
+
workers: workers,
|
|
49
|
+
shard: shard,
|
|
50
|
+
matrix_index: mx,
|
|
51
|
+
matrix_total: mt,
|
|
52
|
+
failure_fragments: ctx[:merge_failures],
|
|
53
|
+
spec_quality_fragments: ctx[:merge_spec_quality]
|
|
54
|
+
)
|
|
55
|
+
child_env = child_env.merge("POLYRUN_HOOK_ORCHESTRATOR" => "0")
|
|
56
|
+
child_env = hook_cfg.merge_worker_ruby_env(child_env)
|
|
56
57
|
|
|
57
|
-
|
|
58
|
+
ping_path = run_shards_prepare_worker_ping!(ctx, child_env, shard)
|
|
58
59
|
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
60
|
+
Polyrun::Log.warn "polyrun run-shards: shard #{shard} → #{paths.size} file(s)" if @verbose
|
|
61
|
+
spawned_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
62
|
+
pid = run_shards_spawn_one_worker(child_env, cmd, paths, hook_cfg)
|
|
63
|
+
pids << {pid: pid, shard: shard, spawned_at: spawned_at, ping_path: ping_path}
|
|
64
|
+
Polyrun::Debug.log("[parent pid=#{$$}] run-shards: Process.spawn shard=#{shard} child_pid=#{pid} spec_files=#{paths.size}")
|
|
65
|
+
Polyrun::Log.warn "polyrun run-shards: started shard #{shard} pid=#{pid} (#{paths.size} file(s))" if parallel
|
|
66
|
+
if Polyrun::WorkerOutput.routing_enabled? && parallel
|
|
67
|
+
Polyrun::Log.warn "polyrun run-shards: shard #{shard} log → #{Polyrun::WorkerOutput.log_path_for(shard)}"
|
|
68
|
+
end
|
|
67
69
|
end
|
|
70
|
+
[pids, nil]
|
|
71
|
+
rescue StandardError, ScriptError => e
|
|
72
|
+
Polyrun::Log.warn "polyrun run-shards: spawn failed (#{e.class}: #{e.message}); stopping #{pids.size} worker(s)"
|
|
73
|
+
run_shards_terminate_children!(pids)
|
|
74
|
+
[pids, 1]
|
|
68
75
|
end
|
|
69
|
-
[pids, nil]
|
|
70
76
|
end
|
|
71
77
|
# rubocop:enable Metrics/AbcSize
|
|
72
78
|
|
|
@@ -85,13 +91,20 @@ module Polyrun
|
|
|
85
91
|
path
|
|
86
92
|
end
|
|
87
93
|
|
|
94
|
+
# Workers start in their own process group so TERM/KILL can reach hook shell grandchildren.
|
|
88
95
|
def run_shards_spawn_one_worker(child_env, cmd, paths, hook_cfg)
|
|
89
96
|
return Polyrun::WorkerOutput.spawn_worker(child_env, cmd, paths, hook_cfg) if Polyrun::WorkerOutput.routing_enabled?
|
|
90
97
|
|
|
91
98
|
if hook_cfg.worker_hooks? && !Polyrun::Hooks.disabled?
|
|
92
|
-
Process.spawn(
|
|
99
|
+
Process.spawn(
|
|
100
|
+
child_env,
|
|
101
|
+
"sh",
|
|
102
|
+
"-c",
|
|
103
|
+
hook_cfg.build_worker_shell_script(cmd, paths),
|
|
104
|
+
pgroup: true
|
|
105
|
+
)
|
|
93
106
|
else
|
|
94
|
-
Process.spawn(child_env, *cmd, *paths)
|
|
107
|
+
Process.spawn(child_env, *cmd, *paths, pgroup: true)
|
|
95
108
|
end
|
|
96
109
|
end
|
|
97
110
|
end
|
|
@@ -234,7 +234,7 @@ module Polyrun
|
|
|
234
234
|
end
|
|
235
235
|
|
|
236
236
|
def run_shards_force_stop_pid_status(pid)
|
|
237
|
-
|
|
237
|
+
run_shards_kill_process_group!(:KILL, pid)
|
|
238
238
|
st = nil
|
|
239
239
|
begin
|
|
240
240
|
Process.wait(pid)
|
|
@@ -243,7 +243,7 @@ module Polyrun
|
|
|
243
243
|
# child already reaped
|
|
244
244
|
end
|
|
245
245
|
st
|
|
246
|
-
rescue Errno::ESRCH
|
|
246
|
+
rescue Errno::ESRCH, Errno::EPERM
|
|
247
247
|
begin
|
|
248
248
|
Process.wait(pid)
|
|
249
249
|
$?
|
|
@@ -33,6 +33,9 @@ module Polyrun
|
|
|
33
33
|
items, paths_source, err = run_shards_resolve_items(o[:paths_file], pc)
|
|
34
34
|
return [err, nil] if err
|
|
35
35
|
|
|
36
|
+
err = run_shards_validate_suite_vs_command!(items, cmd)
|
|
37
|
+
return [err, nil] if err
|
|
38
|
+
|
|
36
39
|
costs, strategy, err = run_shards_resolve_costs(
|
|
37
40
|
o[:timing_path],
|
|
38
41
|
o[:strategy],
|
|
@@ -64,39 +64,26 @@ module Polyrun
|
|
|
64
64
|
Polyrun::Log.warn "polyrun run-shards: no paths (empty paths file or list)"
|
|
65
65
|
return [nil, nil, 2]
|
|
66
66
|
end
|
|
67
|
+
|
|
67
68
|
[items, paths_source, nil]
|
|
68
69
|
end
|
|
69
70
|
|
|
71
|
+
def run_shards_validate_suite_vs_command!(items, cmd)
|
|
72
|
+
msg = Polyrun::Partition::Suite.command_mismatch_message(items, cmd)
|
|
73
|
+
return nil unless msg
|
|
74
|
+
|
|
75
|
+
Polyrun::Log.warn "polyrun run-shards: #{msg}"
|
|
76
|
+
2
|
|
77
|
+
end
|
|
78
|
+
|
|
70
79
|
def run_shards_resolve_costs(timing_path, strategy, timing_granularity, strategy_explicit: false)
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
File.expand_path(timing_path.to_s, Dir.pwd),
|
|
79
|
-
granularity: timing_granularity
|
|
80
|
-
)
|
|
81
|
-
if costs.empty?
|
|
82
|
-
Polyrun::Log.warn "polyrun run-shards: timing file missing or empty: #{timing_path}"
|
|
83
|
-
return [nil, nil, 2]
|
|
84
|
-
end
|
|
85
|
-
if Polyrun::Partition::Plan.timing_load_strategy?(strategy)
|
|
86
|
-
return [costs, strategy, nil]
|
|
87
|
-
end
|
|
88
|
-
unless strategy_explicit
|
|
89
|
-
Polyrun::Log.warn "polyrun run-shards: using cost_binpack (timing data present)" if @verbose
|
|
90
|
-
return [costs, "cost_binpack", nil]
|
|
91
|
-
end
|
|
92
|
-
|
|
93
|
-
[nil, strategy, nil]
|
|
94
|
-
elsif Polyrun::Partition::Plan.cost_strategy?(strategy) || Polyrun::Partition::Plan.lazy_robin_strategy?(strategy)
|
|
95
|
-
Polyrun::Log.warn "polyrun run-shards: --timing or partition.timing_file required for strategy #{strategy}"
|
|
96
|
-
[nil, nil, 2]
|
|
97
|
-
else
|
|
98
|
-
[nil, strategy, nil]
|
|
99
|
-
end
|
|
80
|
+
partition_resolve_costs_and_strategy(
|
|
81
|
+
timing_path,
|
|
82
|
+
strategy,
|
|
83
|
+
timing_granularity,
|
|
84
|
+
strategy_explicit: strategy_explicit,
|
|
85
|
+
log_prefix: "polyrun run-shards"
|
|
86
|
+
)
|
|
100
87
|
end
|
|
101
88
|
|
|
102
89
|
def run_shards_make_plan(items, workers, strategy, seed, costs, constraints, timing_granularity, stable_assignment = nil, shard_weights: nil)
|
|
@@ -127,7 +127,7 @@ module Polyrun
|
|
|
127
127
|
return if Polyrun::WorkerOutput.routing_enabled?
|
|
128
128
|
|
|
129
129
|
Polyrun::Log.warn "polyrun run-shards: #{pid_count} children running; RSpec output below may be interleaved."
|
|
130
|
-
Polyrun::Log.warn "polyrun run-shards:
|
|
130
|
+
Polyrun::Log.warn "polyrun run-shards: RSpec summary counts apply to individual shards."
|
|
131
131
|
end
|
|
132
132
|
|
|
133
133
|
def run_shards_merge_or_hint_coverage(ctx)
|