polyrun 1.2.0 → 1.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -7,11 +7,18 @@ module Polyrun
7
7
  private
8
8
 
9
9
  # File-backed queue (spec_queue.md): init → claim batches → ack (ledger append-only).
10
+ #
11
+ # *N×M and load balancing*: +queue init+ uses the same +Partition::Plan+ slice as +polyrun plan+
12
+ # when +--shard+ / +--total+ (or +partition.shard_index+ / +shard_total+ / CI env) define a matrix.
13
+ # Each CI job runs +init+ once for its shard; the queue holds **only** that shard's paths (no duplicate
14
+ # work across matrix jobs). **M** local workers use +queue claim+ on a **shared** queue directory
15
+ # (NFS or the runner disk): claims are mutually exclusive — **dynamic** balance (fast workers pull more
16
+ # batches), **not** the same static per-worker wall time as +cost_binpack+ on the full suite.
17
+ # +--timing+ on init sorts dequeue order (heavy items first when weights are known); it does **not**
18
+ # replace binpack across workers — for that, use static +plan+ / +run-shards+ with +cost_binpack+.
10
19
  def cmd_queue(argv)
11
20
  dir = ".polyrun-queue"
12
21
  paths_file = nil
13
- timing_path = nil
14
- timing_granularity = nil
15
22
  worker = ENV["USER"] || "worker"
16
23
  batch = 5
17
24
  lease_id = nil
@@ -20,7 +27,7 @@ module Polyrun
20
27
  Polyrun::Debug.log("queue: subcommand=#{sub.inspect}")
21
28
  case sub
22
29
  when "init"
23
- queue_cmd_init(argv, dir, paths_file, timing_path, timing_granularity)
30
+ queue_cmd_init(argv, dir)
24
31
  when "claim"
25
32
  queue_cmd_claim(argv, dir, worker, batch)
26
33
  when "ack"
@@ -33,34 +40,54 @@ module Polyrun
33
40
  end
34
41
  end
35
42
 
36
- def queue_cmd_init(argv, dir, paths_file, timing_path, timing_granularity)
43
+ def queue_cmd_init(argv, dir)
44
+ cfg = Polyrun::Config.load(path: ENV["POLYRUN_CONFIG"])
45
+ pc = cfg.partition
46
+ ctx = plan_command_initial_context(pc)
47
+ paths_file = nil
37
48
  OptionParser.new do |opts|
38
- opts.banner = "usage: polyrun queue init --paths-file P [--timing PATH] [--timing-granularity VAL] [--dir DIR]"
49
+ opts.banner = "usage: polyrun queue init --paths-file P [--dir DIR] [same partition options as polyrun plan]"
39
50
  opts.on("--dir PATH") { |v| dir = v }
40
- opts.on("--paths-file PATH") { |v| paths_file = v }
41
- opts.on("--timing PATH") { |v| timing_path = v }
42
- opts.on("--timing-granularity VAL") { |v| timing_granularity = v }
51
+ opts.on("--paths-file PATH", String) { |v| paths_file = v }
52
+ plan_command_register_partition_options!(opts, ctx)
43
53
  end.parse!(argv)
44
54
  unless paths_file
45
55
  Polyrun::Log.warn "queue init: need --paths-file"
46
56
  return 2
47
57
  end
48
- cfg = Polyrun::Config.load(path: ENV["POLYRUN_CONFIG"])
49
- g = resolve_partition_timing_granularity(cfg.partition, timing_granularity)
50
- items = Polyrun::Partition::Paths.read_lines(paths_file)
51
- costs =
52
- if timing_path
53
- Polyrun::Partition::Plan.load_timing_costs(
54
- File.expand_path(timing_path, Dir.pwd),
55
- granularity: g
56
- )
57
- end
58
- ordered = queue_init_ordered_items(items, costs, g)
58
+ code = Polyrun::Partition::PathsBuild.apply!(partition: pc, cwd: Dir.pwd)
59
+ return code if code != 0
60
+
61
+ ordered, code = queue_partition_manifest_and_ordered_paths(cfg, pc, ctx, paths_file)
62
+ return code if code != 0
63
+
59
64
  Polyrun::Queue::FileStore.new(dir).init!(ordered)
60
65
  Polyrun::Log.puts JSON.generate({"dir" => File.expand_path(dir), "count" => ordered.size})
61
66
  0
62
67
  end
63
68
 
69
+ def queue_partition_manifest_and_ordered_paths(cfg, pc, ctx, paths_file)
70
+ Polyrun::Log.warn "polyrun queue init: using #{cfg.path}" if @verbose && cfg.path
71
+
72
+ manifest, code = plan_command_manifest_from_paths(cfg, pc, [], ctx, paths_file)
73
+ return [nil, code] if code != 0
74
+
75
+ paths = manifest["paths"] || []
76
+ g = resolve_partition_timing_granularity(pc, ctx[:timing_granularity])
77
+ timing_for_sort = plan_resolve_timing_path(pc, ctx[:timing_path], ctx[:strategy])
78
+ costs = queue_init_timing_costs(timing_for_sort, g)
79
+ [queue_init_ordered_items(paths, costs, g), 0]
80
+ end
81
+
82
+ def queue_init_timing_costs(timing_for_sort, g)
83
+ return nil unless timing_for_sort
84
+
85
+ Polyrun::Partition::Plan.load_timing_costs(
86
+ File.expand_path(timing_for_sort.to_s, Dir.pwd),
87
+ granularity: g
88
+ )
89
+ end
90
+
64
91
  def queue_init_ordered_items(items, costs, granularity = :file)
65
92
  if costs && !costs.empty?
66
93
  dw = costs.values.sum / costs.size.to_f
@@ -15,7 +15,7 @@ module Polyrun
15
15
  # Default and upper bound for parallel OS processes (POLYRUN_WORKERS / --workers); see {Polyrun::Config}.
16
16
 
17
17
  # Spawns N OS processes (not Ruby threads) with POLYRUN_SHARD_INDEX / POLYRUN_SHARD_TOTAL so
18
- # {Coverage::Collector} writes coverage/polyrun-fragment-<shard>.json. Merge with merge-coverage.
18
+ # {Coverage::Collector} writes coverage/polyrun-fragment-worker<N>.json (or shard<S>-worker<W>.json in N×M CI). Merge with merge-coverage.
19
19
  def cmd_run_shards(argv, config_path)
20
20
  run_shards_run!(argv, config_path)
21
21
  end
@@ -112,10 +112,21 @@ module Polyrun
112
112
  end
113
113
 
114
114
  # ENV for a worker process: POLYRUN_SHARD_* plus per-shard database URLs from polyrun.yml or DATABASE_URL.
115
- def shard_child_env(cfg:, workers:, shard:)
115
+ # When +matrix_total+ > 1 with multiple local workers, sets +POLYRUN_SHARD_MATRIX_INDEX+ / +POLYRUN_SHARD_MATRIX_TOTAL+
116
+ # so {Coverage::Collector} can name fragments uniquely across CI matrix jobs (NxM sharding).
117
+ def shard_child_env(cfg:, workers:, shard:, matrix_index: nil, matrix_total: nil)
116
118
  child_env = ENV.to_h.merge(
117
119
  Polyrun::Database::Shard.env_map(shard_index: shard, shard_total: workers)
118
120
  )
121
+ mt = matrix_total.nil? ? 0 : Integer(matrix_total)
122
+ if mt > 1
123
+ if matrix_index.nil?
124
+ Polyrun::Log.warn "polyrun run-shards: matrix_total=#{mt} but matrix_index is nil; omit POLYRUN_SHARD_MATRIX_*"
125
+ else
126
+ child_env["POLYRUN_SHARD_MATRIX_INDEX"] = Integer(matrix_index).to_s
127
+ child_env["POLYRUN_SHARD_MATRIX_TOTAL"] = mt.to_s
128
+ end
129
+ end
119
130
  dh = cfg.databases
120
131
  if dh.is_a?(Hash) && !dh.empty?
121
132
  child_env.merge!(Polyrun::Database::UrlBuilder.env_exports_for_databases(dh, shard_index: shard))
@@ -0,0 +1,92 @@
1
+ module Polyrun
2
+ class CLI
3
+ # Spawns and waits on worker processes for +run-shards+ / +ci-shard-*+ fan-out.
4
+ module RunShardsParallelChildren
5
+ private
6
+
7
+ # @return [Array(Array, Integer, nil)] +[pids, spawn_error_code]+; +spawn_error_code+ is +nil+ when all spawns succeeded
8
+ # rubocop:disable Metrics/AbcSize -- shard loop: spawn + shard hooks + env
9
+ def run_shards_spawn_workers(ctx, hook_cfg)
10
+ workers = ctx[:workers]
11
+ cmd = ctx[:cmd]
12
+ cfg = ctx[:cfg]
13
+ plan = ctx[:plan]
14
+ parallel = ctx[:parallel]
15
+ mx = ctx[:matrix_shard_index]
16
+ mt = ctx[:matrix_shard_total]
17
+
18
+ pids = []
19
+ workers.times do |shard|
20
+ paths = plan.shard(shard)
21
+ if paths.empty?
22
+ Polyrun::Log.warn "polyrun run-shards: shard #{shard} skipped (no paths)" if @verbose || parallel
23
+ next
24
+ end
25
+
26
+ env_shard = ENV.to_h.merge(
27
+ "POLYRUN_HOOK_ORCHESTRATOR" => "1",
28
+ "POLYRUN_SHARD_INDEX" => shard.to_s,
29
+ "POLYRUN_SHARD_TOTAL" => workers.to_s
30
+ )
31
+ code = hook_cfg.run_phase_if_enabled(:before_shard, env_shard)
32
+ if code != 0
33
+ run_shards_terminate_children!(pids)
34
+ return [pids, code]
35
+ end
36
+
37
+ child_env = shard_child_env(cfg: cfg, workers: workers, shard: shard, matrix_index: mx, matrix_total: mt)
38
+ child_env = child_env.merge("POLYRUN_HOOK_ORCHESTRATOR" => "0")
39
+ child_env = hook_cfg.merge_worker_ruby_env(child_env)
40
+
41
+ Polyrun::Log.warn "polyrun run-shards: shard #{shard} → #{paths.size} file(s)" if @verbose
42
+ pid = run_shards_spawn_one_worker(child_env, cmd, paths, hook_cfg)
43
+ pids << {pid: pid, shard: shard}
44
+ Polyrun::Debug.log("[parent pid=#{$$}] run-shards: Process.spawn shard=#{shard} child_pid=#{pid} spec_files=#{paths.size}")
45
+ Polyrun::Log.warn "polyrun run-shards: started shard #{shard} pid=#{pid} (#{paths.size} file(s))" if parallel
46
+ end
47
+ [pids, nil]
48
+ end
49
+ # rubocop:enable Metrics/AbcSize
50
+
51
+ def run_shards_spawn_one_worker(child_env, cmd, paths, hook_cfg)
52
+ if hook_cfg.worker_hooks? && !Polyrun::Hooks.disabled?
53
+ Process.spawn(child_env, "sh", "-c", hook_cfg.build_worker_shell_script(cmd, paths))
54
+ else
55
+ Process.spawn(child_env, *cmd, *paths)
56
+ end
57
+ end
58
+
59
+ # @return [Array(Array, Integer)] +[shard_results, after_shard_hook_error_code]+ (0 when all +after_shard+ hooks passed)
60
+ def run_shards_wait_all_children(pids, hook_cfg, ctx)
61
+ workers = ctx[:workers]
62
+ shard_results = []
63
+ after_hook_err = 0
64
+ Polyrun::Debug.time("Process.wait (#{pids.size} worker process(es))") do
65
+ pids.each do |h|
66
+ Process.wait(h[:pid])
67
+ exitstatus = $?.exitstatus
68
+ ok = $?.success?
69
+ Polyrun::Debug.log("[parent pid=#{$$}] run-shards: Process.wait child_pid=#{h[:pid]} shard=#{h[:shard]} exit=#{exitstatus} success=#{ok}")
70
+ env_after = ENV.to_h.merge(
71
+ "POLYRUN_HOOK_ORCHESTRATOR" => "1",
72
+ "POLYRUN_SHARD_INDEX" => h[:shard].to_s,
73
+ "POLYRUN_SHARD_TOTAL" => workers.to_s,
74
+ "POLYRUN_WORKER_EXIT_STATUS" => exitstatus.to_s
75
+ )
76
+ rc = hook_cfg.run_phase_if_enabled(:after_shard, env_after)
77
+ after_hook_err = rc if rc != 0 && after_hook_err == 0
78
+ shard_results << {shard: h[:shard], exitstatus: exitstatus, success: ok}
79
+ end
80
+ rescue Interrupt
81
+ # Do not trap SIGINT: Process.wait raises Interrupt; a trap races and prints Interrupt + SystemExit traces.
82
+ run_shards_shutdown_on_signal!(pids, 130)
83
+ rescue SignalException => e
84
+ raise unless e.signm == "SIGTERM"
85
+
86
+ run_shards_shutdown_on_signal!(pids, 143)
87
+ end
88
+ [shard_results, after_hook_err]
89
+ end
90
+ end
91
+ end
92
+ end
@@ -2,12 +2,14 @@ require "shellwords"
2
2
  require "rbconfig"
3
3
 
4
4
  require_relative "run_shards_planning"
5
+ require_relative "run_shards_parallel_children"
5
6
 
6
7
  module Polyrun
7
8
  class CLI
8
9
  # Partition + spawn workers for `polyrun run-shards` (keeps {RunShardsCommand} file small).
9
10
  module RunShardsRun
10
11
  include RunShardsPlanning
12
+ include RunShardsParallelChildren
11
13
 
12
14
  private
13
15
 
@@ -18,57 +20,68 @@ module Polyrun
18
20
  run_shards_workers_and_merge(ctx)
19
21
  end
20
22
 
23
+ # rubocop:disable Metrics/AbcSize -- orchestration: hooks, merge, worker failures
21
24
  def run_shards_workers_and_merge(ctx)
22
- pids = run_shards_spawn_workers(ctx)
23
- return 1 if pids.empty?
24
-
25
- run_shards_warn_interleaved(ctx[:parallel], pids.size)
26
-
27
- shard_results = run_shards_wait_all_children(pids)
28
- failed = shard_results.reject { |r| r[:success] }.map { |r| r[:shard] }
29
-
30
- Polyrun::Debug.log(format(
31
- "run-shards: workers wall time since start: %.3fs",
32
- Process.clock_gettime(Process::CLOCK_MONOTONIC) - ctx[:run_t0]
33
- ))
25
+ hook_cfg = Polyrun::Hooks.from_config(ctx[:cfg])
26
+ suite_started = false
27
+ exit_code = 1
28
+
29
+ begin
30
+ env_suite = ENV.to_h.merge(
31
+ "POLYRUN_HOOK_ORCHESTRATOR" => "1",
32
+ "POLYRUN_SHARD_TOTAL" => ctx[:workers].to_s
33
+ )
34
+ code = hook_cfg.run_phase_if_enabled(:before_suite, env_suite)
35
+ return code if code != 0
36
+
37
+ suite_started = true
38
+
39
+ pids, spawn_err = run_shards_spawn_workers(ctx, hook_cfg)
40
+ if spawn_err
41
+ exit_code = spawn_err
42
+ return spawn_err
43
+ end
44
+ if pids.empty?
45
+ exit_code = 1
46
+ return 1
47
+ end
34
48
 
35
- if ctx[:parallel]
36
- Polyrun::Log.warn "polyrun run-shards: finished #{pids.size} worker(s)" + (failed.any? ? " (some failed)" : " (exit 0)")
37
- end
49
+ run_shards_warn_interleaved(ctx[:parallel], pids.size)
38
50
 
39
- if failed.any?
40
- run_shards_log_failed_reruns(failed, shard_results, ctx[:plan], ctx[:parallel], ctx[:workers], ctx[:cmd])
41
- return 1
42
- end
51
+ shard_results, wait_hook_err = run_shards_wait_all_children(pids, hook_cfg, ctx)
52
+ failed = shard_results.reject { |r| r[:success] }.map { |r| r[:shard] }
43
53
 
44
- run_shards_merge_or_hint_coverage(ctx)
45
- end
54
+ Polyrun::Debug.log(format(
55
+ "run-shards: workers wall time since start: %.3fs",
56
+ Process.clock_gettime(Process::CLOCK_MONOTONIC) - ctx[:run_t0]
57
+ ))
46
58
 
47
- def run_shards_spawn_workers(ctx)
48
- workers = ctx[:workers]
49
- cmd = ctx[:cmd]
50
- cfg = ctx[:cfg]
51
- plan = ctx[:plan]
52
- parallel = ctx[:parallel]
53
-
54
- pids = []
55
- workers.times do |shard|
56
- paths = plan.shard(shard)
57
- if paths.empty?
58
- Polyrun::Log.warn "polyrun run-shards: shard #{shard} skipped (no paths)" if @verbose || parallel
59
- next
59
+ if ctx[:parallel]
60
+ Polyrun::Log.warn "polyrun run-shards: finished #{pids.size} worker(s)" + (failed.any? ? " (some failed)" : " (exit 0)")
60
61
  end
61
62
 
62
- child_env = shard_child_env(cfg: cfg, workers: workers, shard: shard)
63
+ if failed.any?
64
+ run_shards_log_failed_reruns(failed, shard_results, ctx[:plan], ctx[:parallel], ctx[:workers], ctx[:cmd])
65
+ exit_code = 1
66
+ exit_code = 1 if wait_hook_err != 0
67
+ return exit_code
68
+ end
63
69
 
64
- Polyrun::Log.warn "polyrun run-shards: shard #{shard} → #{paths.size} file(s)" if @verbose
65
- pid = Process.spawn(child_env, *cmd, *paths)
66
- pids << {pid: pid, shard: shard}
67
- Polyrun::Debug.log("[parent pid=#{$$}] run-shards: Process.spawn shard=#{shard} child_pid=#{pid} spec_files=#{paths.size}")
68
- Polyrun::Log.warn "polyrun run-shards: started shard #{shard} pid=#{pid} (#{paths.size} file(s))" if parallel
70
+ exit_code = run_shards_merge_or_hint_coverage(ctx)
71
+ exit_code = 1 if wait_hook_err != 0 && exit_code == 0
72
+ exit_code
73
+ ensure
74
+ if suite_started
75
+ env_after = ENV.to_h.merge(
76
+ "POLYRUN_HOOK_ORCHESTRATOR" => "1",
77
+ "POLYRUN_SHARD_TOTAL" => ctx[:workers].to_s,
78
+ "POLYRUN_SUITE_EXIT_STATUS" => exit_code.to_s
79
+ )
80
+ hook_cfg.run_phase_if_enabled(:after_suite, env_after)
81
+ end
69
82
  end
70
- pids
71
83
  end
84
+ # rubocop:enable Metrics/AbcSize
72
85
 
73
86
  def run_shards_warn_interleaved(parallel, pid_count)
74
87
  return unless parallel && pid_count > 1
@@ -77,27 +90,6 @@ module Polyrun
77
90
  Polyrun::Log.warn "polyrun run-shards: each worker prints its own summary line; the last \"N examples\" line is not a total across shards."
78
91
  end
79
92
 
80
- def run_shards_wait_all_children(pids)
81
- shard_results = []
82
- Polyrun::Debug.time("Process.wait (#{pids.size} worker process(es))") do
83
- pids.each do |h|
84
- Process.wait(h[:pid])
85
- exitstatus = $?.exitstatus
86
- ok = $?.success?
87
- Polyrun::Debug.log("[parent pid=#{$$}] run-shards: Process.wait child_pid=#{h[:pid]} shard=#{h[:shard]} exit=#{exitstatus} success=#{ok}")
88
- shard_results << {shard: h[:shard], exitstatus: exitstatus, success: ok}
89
- end
90
- rescue Interrupt
91
- # Do not trap SIGINT: Process.wait raises Interrupt; a trap races and prints Interrupt + SystemExit traces.
92
- run_shards_shutdown_on_signal!(pids, 130)
93
- rescue SignalException => e
94
- raise unless e.signm == "SIGTERM"
95
-
96
- run_shards_shutdown_on_signal!(pids, 143)
97
- end
98
- shard_results
99
- end
100
-
101
93
  # Best-effort worker teardown then exit. Does not return.
102
94
  def run_shards_shutdown_on_signal!(pids, code)
103
95
  run_shards_terminate_children!(pids)
@@ -140,7 +132,7 @@ module Polyrun
140
132
 
141
133
  if ctx[:parallel]
142
134
  Polyrun::Log.warn <<~MSG
143
- polyrun run-shards: coverage — each worker writes coverage/polyrun-fragment-<shard>.json when Polyrun coverage is enabled (POLYRUN_SHARD_INDEX per process).
135
+ polyrun run-shards: coverage — each worker writes coverage/polyrun-fragment-worker<N>.json when Polyrun coverage is enabled (POLYRUN_SHARD_INDEX per process).
144
136
  polyrun run-shards: next step — merge with: polyrun merge-coverage -i 'coverage/polyrun-fragment-*.json' -o coverage/merged.json --format json,cobertura,console
145
137
  MSG
146
138
  end
data/lib/polyrun/cli.rb CHANGED
@@ -12,9 +12,11 @@ require_relative "cli/queue_command"
12
12
  require_relative "cli/timing_command"
13
13
  require_relative "cli/init_command"
14
14
  require_relative "cli/quick_command"
15
+ require_relative "cli/ci_shard_run_parse"
15
16
  require_relative "cli/ci_shard_run_command"
16
17
  require_relative "cli/config_command"
17
18
  require_relative "cli/default_run"
19
+ require_relative "cli/hooks_command"
18
20
  require_relative "cli/help"
19
21
 
20
22
  module Polyrun
@@ -28,7 +30,7 @@ module Polyrun
28
30
  DISPATCH_SUBCOMMAND_NAMES = %w[
29
31
  plan prepare merge-coverage report-coverage report-junit report-timing
30
32
  env config merge-timing db:setup-template db:setup-shard db:clone-shards
31
- run-shards parallel-rspec start build-paths init queue quick
33
+ run-shards parallel-rspec start build-paths init queue quick hook
32
34
  ].freeze
33
35
 
34
36
  # First argv token that is a normal subcommand (not a path); if argv[0] is not here but looks like paths, run implicit parallel.
@@ -48,9 +50,11 @@ module Polyrun
48
50
  include TimingCommand
49
51
  include InitCommand
50
52
  include QuickCommand
53
+ include CiShardRunParse
51
54
  include CiShardRunCommand
52
55
  include ConfigCommand
53
56
  include DefaultRun
57
+ include HooksCommand
54
58
  include Help
55
59
 
56
60
  def self.run(argv = ARGV)
@@ -175,6 +179,8 @@ module Polyrun
175
179
  cmd_queue(argv)
176
180
  when "quick"
177
181
  cmd_quick(argv)
182
+ when "hook"
183
+ cmd_hook(argv, config_path)
178
184
  else
179
185
  Polyrun::Log.warn "unknown command: #{command}"
180
186
  2
@@ -4,7 +4,7 @@ require_relative "resolver"
4
4
  module Polyrun
5
5
  class Config
6
6
  # Nested hash of values Polyrun uses: loaded YAML (string keys) with overlays for
7
- # merged +prepare.env+, resolved +partition.shard_index+ / +shard_total+ / +timing_granularity+,
7
+ # merged +prepare.env+, resolved +partition.shard_index+ / +shard_total+ / +shard_processes+ / +timing_granularity+,
8
8
  # and top-level +workers+ (+POLYRUN_WORKERS+ default).
9
9
  #
10
10
  # +build+ memoizes the last (cfg, env) in-process so repeated +dig+ calls on the same load do not
@@ -44,6 +44,7 @@ module Polyrun
44
44
  part = deep_stringify_keys(pc).merge(
45
45
  "shard_index" => r.resolve_shard_index(pc, env),
46
46
  "shard_total" => r.resolve_shard_total(pc, env),
47
+ "shard_processes" => r.resolve_shard_processes(pc, env),
47
48
  "timing_granularity" => r.resolve_partition_timing_granularity(pc, nil, env).to_s
48
49
  )
49
50
  base["partition"] = part
@@ -54,6 +54,14 @@ module Polyrun
54
54
  partition_int(pc, %w[shard_total total], 1)
55
55
  end
56
56
 
57
+ # Processes per CI matrix job for +ci-shard-run+ / +ci-shard-rspec+ (NxM: N jobs × M processes).
58
+ # +POLYRUN_SHARD_PROCESSES+ or +partition.shard_processes+; CLI +--shard-processes+ / +--workers+ overrides.
59
+ def resolve_shard_processes(pc, env = ENV)
60
+ return Integer(env["POLYRUN_SHARD_PROCESSES"]) if env["POLYRUN_SHARD_PROCESSES"] && !env["POLYRUN_SHARD_PROCESSES"].empty?
61
+
62
+ partition_int(pc, %w[shard_processes shard_workers workers_per_shard], 1)
63
+ end
64
+
57
65
  # +cli_val+ is an override (e.g. +run-shards --timing-granularity+); +nil+ uses YAML then +POLYRUN_TIMING_GRANULARITY+.
58
66
  def resolve_partition_timing_granularity(pc, cli_val, env = ENV)
59
67
  raw = cli_val
@@ -61,6 +61,11 @@ module Polyrun
61
61
  def version
62
62
  raw["version"] || raw[:version]
63
63
  end
64
+
65
+ # Optional +hooks:+ block for +run-shards+ / +parallel-rspec+ / +ci-shard-*+ (see {Hooks}).
66
+ def hooks
67
+ raw["hooks"] || raw[:hooks] || {}
68
+ end
64
69
  end
65
70
  end
66
71
 
@@ -8,6 +8,7 @@ require_relative "formatter"
8
8
  require_relative "merge"
9
9
  require_relative "result"
10
10
  require_relative "track_files"
11
+ require_relative "collector_fragment_meta"
11
12
  require_relative "../debug"
12
13
 
13
14
  module Polyrun
@@ -24,7 +25,7 @@ module Polyrun
24
25
 
25
26
  # @param root [String] project root (absolute or relative)
26
27
  # @param reject_patterns [Array<String>] path substrings to drop (like SimpleCov add_filter)
27
- # @param output_path [String, nil] default coverage/polyrun-fragment-<shard>.json
28
+ # @param output_path [String, nil] default see {.fragment_default_basename_from_env}
28
29
  # @param minimum_line_percent [Float, nil] exit 1 if below (when strict)
29
30
  # @param strict [Boolean] whether to exit non-zero on threshold failure (default true when minimum set)
30
31
  # @param track_under [Array<String>] when +track_files+ is nil, only keep coverage keys under these dirs relative to +root+. Default +["lib"]+.
@@ -34,18 +35,25 @@ module Polyrun
34
35
  # @param formatter [Object, nil] Object responding to +format(result, output_dir:, basename:)+ like SimpleCov formatters (e.g. {Formatter.multi} or {Formatter::MultiFormatter})
35
36
  # @param report_output_dir [String, nil] directory for +formatter+ outputs (default +coverage/+ under +root+)
36
37
  # @param report_basename [String] file prefix for formatter outputs (default +polyrun-coverage+)
38
+ # See {CollectorFragmentMeta.fragment_default_basename_from_env}.
39
+ def self.fragment_default_basename_from_env(env = ENV)
40
+ CollectorFragmentMeta.fragment_default_basename_from_env(env)
41
+ end
42
+
37
43
  def start!(root:, reject_patterns: [], track_under: ["lib"], track_files: nil, groups: nil, output_path: nil, minimum_line_percent: nil, strict: nil, meta: {}, formatter: nil, report_output_dir: nil, report_basename: "polyrun-coverage")
38
44
  return if disabled?
39
45
 
40
46
  root = File.expand_path(root)
41
- shard = ENV.fetch("POLYRUN_SHARD_INDEX", "0")
42
- output_path ||= File.join(root, "coverage", "polyrun-fragment-#{shard}.json")
47
+ basename = fragment_default_basename_from_env
48
+ output_path ||= File.join(root, "coverage", "polyrun-fragment-#{basename}.json")
43
49
  strict = if minimum_line_percent.nil?
44
50
  false
45
51
  else
46
52
  strict.nil? || strict
47
53
  end
48
54
 
55
+ fragment_meta = CollectorFragmentMeta.fragment_meta_from_env(basename)
56
+
49
57
  @config = {
50
58
  root: root,
51
59
  track_under: Array(track_under).map(&:to_s),
@@ -59,7 +67,8 @@ module Polyrun
59
67
  formatter: formatter,
60
68
  report_output_dir: report_output_dir,
61
69
  report_basename: report_basename,
62
- shard_total_at_start: ENV["POLYRUN_SHARD_TOTAL"].to_i
70
+ shard_total_at_start: ENV["POLYRUN_SHARD_TOTAL"].to_i,
71
+ fragment_meta: fragment_meta
63
72
  }
64
73
 
65
74
  unless ::Coverage.running?
@@ -110,11 +119,7 @@ module Polyrun
110
119
  end
111
120
 
112
121
  def self.finish_debug_time_label
113
- if ENV["POLYRUN_SHARD_TOTAL"].to_i > 1
114
- "worker pid=#{$$} shard=#{ENV.fetch("POLYRUN_SHARD_INDEX", "?")} Coverage::Collector.finish (write fragment)"
115
- else
116
- "Coverage::Collector.finish (write fragment)"
117
- end
122
+ CollectorFragmentMeta.finish_debug_time_label
118
123
  end
119
124
 
120
125
  def build_meta(cfg)
@@ -123,6 +128,7 @@ module Polyrun
123
128
  m["timestamp"] ||= Time.now.to_i
124
129
  m["command_name"] ||= "rspec"
125
130
  m["polyrun_coverage_root"] = cfg[:root].to_s
131
+ CollectorFragmentMeta.merge_fragment_meta!(m, cfg[:fragment_meta])
126
132
  if cfg[:groups]
127
133
  m["polyrun_coverage_groups"] = cfg[:groups].transform_keys(&:to_s).transform_values(&:to_s)
128
134
  end
@@ -12,6 +12,8 @@ module Polyrun
12
12
  collector_finish: "start",
13
13
  polyrun_shard_index: ENV["POLYRUN_SHARD_INDEX"],
14
14
  polyrun_shard_total: ENV["POLYRUN_SHARD_TOTAL"],
15
+ polyrun_shard_matrix_index: ENV["POLYRUN_SHARD_MATRIX_INDEX"],
16
+ polyrun_shard_matrix_total: ENV["POLYRUN_SHARD_MATRIX_TOTAL"],
15
17
  output_path: cfg[:output_path]
16
18
  )
17
19
  Polyrun::Debug.time(Collector.finish_debug_time_label) do
@@ -0,0 +1,57 @@
1
+ module Polyrun
2
+ module Coverage
3
+ # Shard / worker naming for coverage JSON fragments (N×M CI vs run-shards).
4
+ module CollectorFragmentMeta
5
+ module_function
6
+
7
+ # Default fragment basename (no extension) for +coverage/polyrun-fragment-<basename>.json+.
8
+ def fragment_default_basename_from_env(env = ENV)
9
+ local = env.fetch("POLYRUN_SHARD_INDEX", "0")
10
+ mt = env["POLYRUN_SHARD_MATRIX_TOTAL"].to_i
11
+ if mt > 1
12
+ mi = env.fetch("POLYRUN_SHARD_MATRIX_INDEX", "0")
13
+ "shard#{mi}-worker#{local}"
14
+ elsif env["POLYRUN_SHARD_TOTAL"].to_i > 1
15
+ "worker#{local}"
16
+ else
17
+ local
18
+ end
19
+ end
20
+
21
+ def finish_debug_time_label
22
+ mt = ENV["POLYRUN_SHARD_MATRIX_TOTAL"].to_i
23
+ if mt > 1
24
+ "worker pid=#{$$} shard(matrix)=#{ENV.fetch("POLYRUN_SHARD_MATRIX_INDEX", "?")} worker(local)=#{ENV.fetch("POLYRUN_SHARD_INDEX", "?")} Coverage::Collector.finish (write fragment)"
25
+ elsif ENV["POLYRUN_SHARD_TOTAL"].to_i > 1
26
+ "worker pid=#{$$} worker=#{ENV.fetch("POLYRUN_SHARD_INDEX", "?")} Coverage::Collector.finish (write fragment)"
27
+ else
28
+ "Coverage::Collector.finish (write fragment)"
29
+ end
30
+ end
31
+
32
+ def fragment_meta_from_env(basename)
33
+ mt = ENV["POLYRUN_SHARD_MATRIX_TOTAL"].to_i
34
+ {
35
+ basename: basename,
36
+ worker_index: ENV.fetch("POLYRUN_SHARD_INDEX", "0"),
37
+ shard_matrix_index: shard_matrix_index_value(mt)
38
+ }
39
+ end
40
+
41
+ def shard_matrix_index_value(matrix_total)
42
+ return nil if matrix_total <= 1
43
+
44
+ ENV.fetch("POLYRUN_SHARD_MATRIX_INDEX", "0")
45
+ end
46
+
47
+ def merge_fragment_meta!(m, fm)
48
+ return m if fm.nil?
49
+
50
+ m["polyrun_fragment_basename"] = fm[:basename].to_s if fm[:basename]
51
+ m["polyrun_worker_index"] = fm[:worker_index].to_s if fm[:worker_index]
52
+ m["polyrun_shard_matrix_index"] = fm[:shard_matrix_index].to_s if fm[:shard_matrix_index]
53
+ m
54
+ end
55
+ end
56
+ end
57
+ end