polyrun 1.4.2 → 1.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f37969f7a73313a98aa0fd343ae1ed871a32b6dc504da56dd5cb51f42ef2969a
-  data.tar.gz: 3e17734a4419ea15b243ad01261701106d99e3470e4408d2b545deb5876ddcf3
+  metadata.gz: 1d4fc5867eb97f45848d6da2b7ac8d0c3906de5cd0df849f02042aaaee1e9bbf
+  data.tar.gz: a10e216d02b76c722627ab5d0c73e5fe061c118d14f356bdc620655a7e83454a
 SHA512:
-  metadata.gz: 68b10554a23d62042db0d6d5307933ffdbb0fb58cc2452b78e1fca2e1a9b42095bc25e7e98b6b104f30a8be6cb0c4aad09212a57b1f4bbdbd2e2efad3501e3fc
-  data.tar.gz: 70594cb4f7cd5fb32be05aa0da60058e0fc9b50c9ca3ae4223e88a39293a5199e078eda52a2cb461f6782011aa22f05e3ca5efdc7b52cacc184f67a7326c13db
+  metadata.gz: fd37e0e3c6f3afccb8da9dba32b0b836d6b0ddf25181b0bbc2b3d20be561d93518af54ffb929cb62511259a788ad080f9f7080252231d3554aee5ab056c3c841
+  data.tar.gz: 92cb1e5ded19005ccca5975754525857ae5f1b9561947b9a6d1b736e62c098a61e377e3465dba1add9d265277f44672911be48e27097b9c3813ef188341f3a94

data/CHANGELOG.md

@@ -2,6 +2,18 @@
 
 ## Unreleased
 
+## 1.5.0 (2026-05-04)
+
+- Add `run-shards --worker-timeout SEC` and `POLYRUN_WORKER_TIMEOUT_SEC` (wall time per worker since spawn); stop stuck workers; record exit 124 for that shard.
+- Add `run-shards --worker-idle-timeout SEC` and `POLYRUN_WORKER_IDLE_TIMEOUT_SEC`; parent reads monotonic timestamps from `POLYRUN_WORKER_PING_FILE`; record exit 125 when the last ping is stale. Idle applies only after a valid positive ping (use wall timeout until the first ping).
+- Add `Polyrun::WorkerPing` (`ping!`, `ensure_interval_ping_thread!` when `POLYRUN_WORKER_PING_THREAD`). Add `Polyrun::RSpec.install_worker_ping!` and `Polyrun::Minitest.install_worker_ping!`; Polyrun Quick calls `WorkerPing.ping!` around each example. Parent creates ping paths under `tmp/polyrun/` and unlinks files after workers exit.
+- Poll every live shard worker together when timeouts are enabled so idle and wall limits apply to all children, not only the first waiter.
+- Split parallel worker teardown into `RunShardsParallelWait` and `RunShardsWorkerInterrupt`; keep spawn logic in `RunShardsParallelChildren`.
+- Add `Polyrun::Log.orchestration_warn`; when `POLYRUN_ORCHESTRATION_STDERR=1`, copy one line to process `$stderr` if `Log.stderr` is not the same object (custom/null sinks).
+- Wire `env_worker_timeout_sec` / `env_worker_idle_timeout_sec` into `ci-shard-run` plan context. Rescue `Interrupt` around `after_suite` in `run-shards` and `ci-shard` orchestration where suite hooks run.
+- In `Polyrun::Hooks#run_phase`, rescue `Interrupt` for Ruby DSL and shell hook phases (return 130).
+- Document worker timeout, idle ping, and `POLYRUN_ORCHESTRATION_STDERR` in `polyrun help`. Add `sig/polyrun/worker_ping.rbs` and extend `Polyrun::RSpec` / `Polyrun::Minitest` installer signatures.
+
 ## 1.4.2 (2026-04-24)
 
 - Add richer HTML coverage reports: summary cards, group coverage, sortable file tables, project-relative paths, and per-file source detail.

data/lib/polyrun/cli/ci_shard_hooks.rb

@@ -49,7 +49,11 @@ module Polyrun
               "POLYRUN_SHARD_TOTAL" => ctx[:workers].to_s,
               "POLYRUN_SUITE_EXIT_STATUS" => exit_code.to_s
             )
-            hook_cfg.run_phase_if_enabled(:after_suite, env_after)
+            begin
+              hook_cfg.run_phase_if_enabled(:after_suite, env_after)
+            rescue Interrupt
+              Polyrun::Log.warn "polyrun ci-shard: after_suite hook interrupted"
+            end
           end
         end
       end
@@ -109,9 +113,13 @@ module Polyrun
           exit_code
         ensure
           if suite_started
-            hook_cfg.run_phase_if_enabled(:after_suite, env_orch.merge(
-              "POLYRUN_SUITE_EXIT_STATUS" => exit_code.to_s
-            ))
+            begin
+              hook_cfg.run_phase_if_enabled(:after_suite, env_orch.merge(
+                "POLYRUN_SUITE_EXIT_STATUS" => exit_code.to_s
+              ))
+            rescue Interrupt
+              Polyrun::Log.warn "polyrun ci-shard: after_suite hook interrupted"
+            end
           end
         end
       end

data/lib/polyrun/cli/ci_shard_run_command.rb

@@ -66,7 +66,9 @@ module Polyrun
           merge_format: nil,
           config_path: config_path,
           matrix_shard_index: mx,
-          matrix_shard_total: mt
+          matrix_shard_total: mt,
+          worker_timeout_sec: env_worker_timeout_sec,
+          worker_idle_timeout_sec: env_worker_idle_timeout_sec
         }
       end
 

data/lib/polyrun/cli/help.rb

@@ -23,6 +23,9 @@ module Polyrun
           Warn if merge-coverage wall time exceeds N seconds (default 10): POLYRUN_MERGE_SLOW_WARN_SECONDS (0 disables)
           Failure fragments (run-shards --merge-failures): POLYRUN_MERGE_FAILURES=1; parent sets POLYRUN_FAILURE_FRAGMENTS=1 in workers; POLYRUN_FAILURE_FRAGMENT_DIR, POLYRUN_MERGED_FAILURES_OUT, POLYRUN_MERGED_FAILURES_FORMAT; after_suite sets POLYRUN_MERGED_FAILURES_PATH when merge ran
           Parallel RSpec workers: POLYRUN_WORKERS default 5, max 10 (run-shards / parallel-rspec / start); distinct from POLYRUN_SHARD_PROCESSES / ci-shard --shard-processes (local processes per CI matrix job)
+          Per-worker wall timeout: run-shards --worker-timeout SEC or POLYRUN_WORKER_TIMEOUT_SEC (max time since each worker spawn). Parent polls all live workers together. Exit 124; remaining workers stopped.
+          Per-worker idle timeout: --worker-idle-timeout SEC or POLYRUN_WORKER_IDLE_TIMEOUT_SEC counts only after a successful ping timestamp (positive float in POLYRUN_WORKER_PING_FILE); empty or unreadable pings do not satisfy idle enforcement—use wall timeout until the first ping. RSpec/Minitest/Quick installers call Polyrun::WorkerPing.ping! per example/suite. Ping files live under tmp/polyrun/ (gitignored via tmp/); parent unlinks each after its worker exits. Exit 125. Optional outer cap: --worker-timeout (exit 124). Optional periodic pings: POLYRUN_WORKER_PING_THREAD=1 (POLYRUN_WORKER_PING_INTERVAL_SEC); WorkerPing.ensure_interval_ping_thread! (installers invoke it—call yourself if wiring workers without install_worker_ping!).
+          If Polyrun::Log.stderr is null or redirected away, set POLYRUN_ORCHESTRATION_STDERR=1 to also print timeout/SIGINT summary lines to process stderr.
           Partition timing granularity (default file): POLYRUN_TIMING_GRANULARITY=file|example (experimental per-example; see partition.timing_granularity)
 
           commands:

data/lib/polyrun/cli/helpers.rb

@@ -11,6 +11,28 @@ module Polyrun
         Polyrun::Config::Resolver.env_int(name, fallback)
       end
 
+      # Per-worker wall clock (from spawn) for run-shards / ci-shard fan-out; unset or invalid means no limit.
+      def env_worker_timeout_sec
+        s = ENV["POLYRUN_WORKER_TIMEOUT_SEC"].to_s.strip
+        return nil if s.empty?
+
+        f = Float(s, exception: false)
+        return nil if f.nil? || f <= 0
+
+        f
+      end
+
+      # Max seconds without a new monotonic timestamp ping in the worker (see +polyrun/worker_ping+).
+      def env_worker_idle_timeout_sec
+        s = ENV["POLYRUN_WORKER_IDLE_TIMEOUT_SEC"].to_s.strip
+        return nil if s.empty?
+
+        f = Float(s, exception: false)
+        return nil if f.nil? || f <= 0
+
+        f
+      end
+
       def resolve_shard_index(pc)
         Polyrun::Config::Resolver.resolve_shard_index(pc)
       end

data/lib/polyrun/cli/run_shards_parallel_children.rb

@@ -1,7 +1,13 @@
+require "fileutils"
+
+require_relative "run_shards_parallel_wait"
+
 module Polyrun
   class CLI
-    # Spawns and waits on worker processes for +run-shards+ / +ci-shard-*+ fan-out.
+    # Spawns worker processes for +run-shards+ / +ci-shard-*+ fan-out. See {RunShardsParallelWait} for wait/timeout.
     module RunShardsParallelChildren
+      include RunShardsParallelWait
+
       private
 
       # @return [Array(Array, Integer, nil)] +[pids, spawn_error_code]+; +spawn_error_code+ is +nil+ when all spawns succeeded
@@ -45,9 +51,12 @@ module Polyrun
           child_env = child_env.merge("POLYRUN_HOOK_ORCHESTRATOR" => "0")
           child_env = hook_cfg.merge_worker_ruby_env(child_env)
 
+          ping_path = run_shards_prepare_worker_ping!(ctx, child_env, shard)
+
           Polyrun::Log.warn "polyrun run-shards: shard #{shard} → #{paths.size} file(s)" if @verbose
+          spawned_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
           pid = run_shards_spawn_one_worker(child_env, cmd, paths, hook_cfg)
-          pids << {pid: pid, shard: shard}
+          pids << {pid: pid, shard: shard, spawned_at: spawned_at, ping_path: ping_path}
           Polyrun::Debug.log("[parent pid=#{$$}] run-shards: Process.spawn shard=#{shard} child_pid=#{pid} spec_files=#{paths.size}")
           Polyrun::Log.warn "polyrun run-shards: started shard #{shard} pid=#{pid} (#{paths.size} file(s))" if parallel
         end
@@ -55,6 +64,21 @@ module Polyrun
       end
       # rubocop:enable Metrics/AbcSize
 
+      def run_shards_prepare_worker_ping!(ctx, child_env, shard)
+        idle_sec = ctx[:worker_idle_timeout_sec]
+        idle_sec = nil if idle_sec.is_a?(Numeric) && idle_sec <= 0
+        return nil unless idle_sec
+
+        dir = File.join(Dir.pwd, "tmp", "polyrun")
+        FileUtils.mkdir_p(dir)
+        path = File.expand_path("worker-ping-#{$$}-#{shard}.txt", dir)
+        File.binwrite(path, "")
+        child_env["POLYRUN_WORKER_PING_FILE"] = path
+        interval = ENV["POLYRUN_WORKER_PING_INTERVAL_SEC"].to_s.strip
+        child_env["POLYRUN_WORKER_PING_INTERVAL_SEC"] = interval.empty? ? "15" : interval
+        path
+      end
+
       def run_shards_spawn_one_worker(child_env, cmd, paths, hook_cfg)
         if hook_cfg.worker_hooks? && !Polyrun::Hooks.disabled?
           Process.spawn(child_env, "sh", "-c", hook_cfg.build_worker_shell_script(cmd, paths))
@@ -62,38 +86,6 @@ module Polyrun
           Process.spawn(child_env, *cmd, *paths)
         end
       end
-
-      # @return [Array(Array, Integer)] +[shard_results, after_shard_hook_error_code]+ (0 when all +after_shard+ hooks passed)
-      def run_shards_wait_all_children(pids, hook_cfg, ctx)
-        workers = ctx[:workers]
-        shard_results = []
-        after_hook_err = 0
-        Polyrun::Debug.time("Process.wait (#{pids.size} worker process(es))") do
-          pids.each do |h|
-            Process.wait(h[:pid])
-            exitstatus = $?.exitstatus
-            ok = $?.success?
-            Polyrun::Debug.log("[parent pid=#{$$}] run-shards: Process.wait child_pid=#{h[:pid]} shard=#{h[:shard]} exit=#{exitstatus} success=#{ok}")
-            env_after = ENV.to_h.merge(
-              "POLYRUN_HOOK_ORCHESTRATOR" => "1",
-              "POLYRUN_SHARD_INDEX" => h[:shard].to_s,
-              "POLYRUN_SHARD_TOTAL" => workers.to_s,
-              "POLYRUN_WORKER_EXIT_STATUS" => exitstatus.to_s
-            )
-            rc = hook_cfg.run_phase_if_enabled(:after_shard, env_after)
-            after_hook_err = rc if rc != 0 && after_hook_err == 0
-            shard_results << {shard: h[:shard], exitstatus: exitstatus, success: ok}
-          end
-        rescue Interrupt
-          # Do not trap SIGINT: Process.wait raises Interrupt; a trap races and prints Interrupt + SystemExit traces.
-          run_shards_shutdown_on_signal!(pids, 130)
-        rescue SignalException => e
-          raise unless e.signm == "SIGTERM"
-
-          run_shards_shutdown_on_signal!(pids, 143)
-        end
-        [shard_results, after_hook_err]
-      end
     end
   end
 end

data/lib/polyrun/cli/run_shards_parallel_wait.rb

@@ -0,0 +1,267 @@
+# rubocop:disable Polyrun/FileLength, Metrics/ModuleLength -- wait loop + idle/wall flush (kept out of spawn module)
+module Polyrun
+  class CLI
+    # Wait, wall/idle timeout, and +after_shard+ hooks for parallel workers (+run-shards+ / +ci-shard-*+).
+    module RunShardsParallelWait
+      WORKER_TIMEOUT_EXIT_STATUS = 124
+      WORKER_IDLE_TIMEOUT_EXIT_STATUS = 125
+
+      private
+
+      # @return [Array(Array, Integer)] +[shard_results, after_shard_hook_error_code]+ (0 when all +after_shard+ hooks passed)
+      # rubocop:disable Metrics/AbcSize -- wait loop + timeout flush
+      def run_shards_wait_all_children(pids, hook_cfg, ctx)
+        workers = ctx[:workers]
+        shard_results = []
+        after_hook_err = 0
+        timeout_sec = ctx[:worker_timeout_sec]
+        timeout_sec = nil if timeout_sec.is_a?(Numeric) && timeout_sec <= 0
+        idle_sec = ctx[:worker_idle_timeout_sec]
+        idle_sec = nil if idle_sec.is_a?(Numeric) && idle_sec <= 0
+
+        Polyrun::Debug.time("Process.wait (#{pids.size} worker process(es))") do
+          if timeout_sec || idle_sec
+            run_shards_wait_all_children_multiplex(
+              pids, hook_cfg, ctx, workers, timeout_sec, idle_sec, shard_results, after_hook_err
+            )
+          else
+            run_shards_wait_all_children_sequential(pids, hook_cfg, workers, shard_results, after_hook_err)
+          end
+        rescue Interrupt
+          run_shards_shutdown_on_signal!(pids, 130)
+        rescue SignalException => e
+          raise unless e.signm == "SIGTERM"
+
+          run_shards_shutdown_on_signal!(pids, 143)
+        end
+      end
+      # rubocop:enable Metrics/AbcSize
+
+      def run_shards_wait_all_children_sequential(pids, hook_cfg, workers, shard_results, after_hook_err)
+        pids.each do |h|
+          Process.wait(h[:pid])
+          st = $?
+          after_hook_err = run_shards_finalize_reaped_worker!(h, hook_cfg, workers, st, shard_results, after_hook_err)
+        end
+        [shard_results, after_hook_err]
+      end
+
+      # Poll every live PID each tick so wall and idle timeouts apply to all workers, not only the first in wait order.
+      def run_shards_wait_all_children_multiplex(pids, hook_cfg, ctx, workers, timeout_sec, idle_sec, shard_results, after_hook_err)
+        pending = pids.dup
+
+        loop do
+          pending.delete_if do |h|
+            wpid = Process.wait(h[:pid], Process::WNOHANG)
+            next false unless wpid == h[:pid]
+
+            st = $?
+            after_hook_err = run_shards_finalize_reaped_worker!(h, hook_cfg, workers, st, shard_results, after_hook_err)
+            true
+          end
+
+          return [shard_results, after_hook_err] if pending.empty?
+
+          now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+          violation = run_shards_timeout_violation(pids, pending, ctx, now, timeout_sec, idle_sec)
+          if violation
+            reason, timed_h = violation
+            others = pending.reject { |x| x[:pid] == timed_h[:pid] }
+            case reason
+            when :wall_timeout
+              return run_shards_wait_flush_after_worker_timeout!(
+                timed_h, others, hook_cfg, ctx, timeout_sec, workers, shard_results, after_hook_err
+              )
+            when :idle_timeout
+              return run_shards_wait_flush_after_worker_idle!(
+                timed_h, others, hook_cfg, ctx, idle_sec, workers, shard_results, after_hook_err
+              )
+            end
+          end
+
+          sleep(0.2)
+        end
+      end
+
+      # @return [(Symbol, Hash), nil] e.g. +[:wall_timeout, h]+ when a limit is exceeded
+      def run_shards_timeout_violation(pids_order, pending, ctx, now, timeout_sec, idle_sec)
+        pids_order.each do |h|
+          next unless pending.any? { |p| p[:pid] == h[:pid] }
+
+          if timeout_sec && timeout_sec > 0
+            spawned_at = h[:spawned_at] || ctx[:run_t0]
+            return [:wall_timeout, h] if now >= spawned_at + timeout_sec
+          end
+        end
+
+        pids_order.each do |h|
+          next unless pending.any? { |p| p[:pid] == h[:pid] }
+
+          next unless idle_sec && idle_sec > 0 && h[:ping_path]
+
+          t, = run_shards_read_worker_ping_payload(h[:ping_path])
+          return [:idle_timeout, h] if t && (now - t) > idle_sec
+        end
+
+        nil
+      end
+
+      def run_shards_finalize_reaped_worker!(h, hook_cfg, workers, st, shard_results, after_hook_err)
+        exitstatus = st.exitstatus
+        ok = st.success?
+        Polyrun::Debug.log("[parent pid=#{$$}] run-shards: Process.wait child_pid=#{h[:pid]} shard=#{h[:shard]} exit=#{exitstatus} success=#{ok}")
+        rc = run_shards_invoke_after_shard!(hook_cfg, h[:shard], workers, exitstatus)
+        after_hook_err = rc if rc != 0 && after_hook_err == 0
+        shard_results << {shard: h[:shard], exitstatus: exitstatus, success: ok}
+        run_shards_unlink_ping_path(h[:ping_path])
+        after_hook_err
+      end
+
+      def run_shards_unlink_ping_path(path)
+        s = path.to_s.strip
+        return if s.empty?
+
+        File.unlink(s) if File.file?(s)
+      rescue SystemCallError
+        # best-effort cleanup of tmp/polyrun/worker-ping-*.txt
+      end
+
+      def run_shards_read_worker_ping_time(path)
+        run_shards_read_worker_ping_payload(path)[0]
+      end
+
+      # @return [Array(Float?, String?)] monotonic time and optional location line (path:line)
+      def run_shards_read_worker_ping_payload(path)
+        return [nil, nil] unless path && File.file?(path)
+
+        s = File.binread(path)
+        return [nil, nil] if s.nil? || s.strip.empty?
+
+        time_line, rest = s.split("\n", 2)
+        first = time_line.to_s.strip
+        return [nil, nil] if first.empty?
+
+        f = first.to_f
+        t = f.positive? ? f : nil
+        loc = rest.to_s.strip
+        loc = nil if loc.empty?
+        [t, loc]
+      rescue SystemCallError
+        [nil, nil]
+      end
+
+      def run_shards_wait_flush_after_worker_idle!(timed_h, others, hook_cfg, ctx, idle_sec, workers, shard_results, after_hook_err)
+        run_shards_warn_worker_idle!(timed_h, ctx, idle_sec)
+        run_shards_force_stop_pid_status(timed_h[:pid])
+        run_shards_unlink_ping_path(timed_h[:ping_path])
+        rc = run_shards_invoke_after_shard!(hook_cfg, timed_h[:shard], workers, WORKER_IDLE_TIMEOUT_EXIT_STATUS)
+        after_hook_err = rc if rc != 0 && after_hook_err == 0
+        shard_results << {shard: timed_h[:shard], exitstatus: WORKER_IDLE_TIMEOUT_EXIT_STATUS, success: false}
+        others.each do |h2|
+          st2 = run_shards_wait_or_force_stop_status(h2[:pid])
+          exit2 = st2&.exitstatus
+          ok2 = st2 ? st2.success? : false
+          exit2 = WORKER_IDLE_TIMEOUT_EXIT_STATUS if exit2.nil?
+          run_shards_unlink_ping_path(h2[:ping_path])
+          rc2 = run_shards_invoke_after_shard!(hook_cfg, h2[:shard], workers, exit2)
+          after_hook_err = rc2 if rc2 != 0 && after_hook_err == 0
+          shard_results << {shard: h2[:shard], exitstatus: exit2, success: ok2}
+        end
+        [shard_results, after_hook_err]
+      end
+
+      def run_shards_wait_flush_after_worker_timeout!(timed_h, others, hook_cfg, ctx, timeout_sec, workers, shard_results, after_hook_err)
+        run_shards_warn_worker_timeout!(timed_h, ctx, timeout_sec)
+        run_shards_force_stop_pid_status(timed_h[:pid])
+        run_shards_unlink_ping_path(timed_h[:ping_path])
+        rc = run_shards_invoke_after_shard!(hook_cfg, timed_h[:shard], workers, WORKER_TIMEOUT_EXIT_STATUS)
+        after_hook_err = rc if rc != 0 && after_hook_err == 0
+        shard_results << {shard: timed_h[:shard], exitstatus: WORKER_TIMEOUT_EXIT_STATUS, success: false}
+        others.each do |h2|
+          st2 = run_shards_wait_or_force_stop_status(h2[:pid])
+          exit2 = st2&.exitstatus
+          ok2 = st2 ? st2.success? : false
+          exit2 = WORKER_TIMEOUT_EXIT_STATUS if exit2.nil?
+          run_shards_unlink_ping_path(h2[:ping_path])
+          rc2 = run_shards_invoke_after_shard!(hook_cfg, h2[:shard], workers, exit2)
+          after_hook_err = rc2 if rc2 != 0 && after_hook_err == 0
+          shard_results << {shard: h2[:shard], exitstatus: exit2, success: ok2}
+        end
+        [shard_results, after_hook_err]
+      end
+
+      def run_shards_invoke_after_shard!(hook_cfg, shard, workers, exitstatus)
+        env_after = ENV.to_h.merge(
+          "POLYRUN_HOOK_ORCHESTRATOR" => "1",
+          "POLYRUN_SHARD_INDEX" => shard.to_s,
+          "POLYRUN_SHARD_TOTAL" => workers.to_s,
+          "POLYRUN_WORKER_EXIT_STATUS" => exitstatus.to_s
+        )
+        hook_cfg.run_phase_if_enabled(:after_shard, env_after)
+      end
+
+      def run_shards_warn_worker_idle!(h, ctx, idle_sec)
+        paths = ctx[:plan].shard(h[:shard])
+        sample = paths.first(5).join(", ")
+        suffix =
+          if paths.empty?
+            " (no paths)"
+          elsif paths.size > 5
+            " (#{paths.size} files total)"
+          else
+            ""
+          end
+        _t, loc = run_shards_read_worker_ping_payload(h[:ping_path])
+        ping_suffix = (loc && !loc.to_s.strip.empty?) ? "; last ping #{loc.to_s.strip}" : ""
+        Polyrun::Log.orchestration_warn "polyrun run-shards: WORKER IDLE TIMEOUT after #{idle_sec}s since last per-example progress ping — shard #{h[:shard]} pid #{h[:pid]}#{ping_suffix}."
+        Polyrun::Log.warn "polyrun run-shards: idle shard file sample: #{sample}#{suffix}"
+        Polyrun::Log.warn "polyrun run-shards: use Polyrun::RSpec.install_worker_ping! / Polyrun::Minitest.install_worker_ping! (Polyrun Quick calls ping! each example); exit #{WORKER_IDLE_TIMEOUT_EXIT_STATUS}."
+      end
+
+      def run_shards_wait_or_force_stop_status(pid)
+        wpid = Process.wait(pid, Process::WNOHANG)
+        return $? if wpid == pid
+
+        run_shards_force_stop_pid_status(pid)
+      rescue Errno::ECHILD
+        nil
+      end
+
+      def run_shards_force_stop_pid_status(pid)
+        Process.kill(:KILL, pid)
+        st = nil
+        begin
+          Process.wait(pid)
+          st = $?
+        rescue Errno::ECHILD
+          # child already reaped
+        end
+        st
+      rescue Errno::ESRCH
+        begin
+          Process.wait(pid)
+          $?
+        rescue Errno::ECHILD
+          nil
+        end
+      end
+
+      def run_shards_warn_worker_timeout!(h, ctx, timeout_sec)
+        paths = ctx[:plan].shard(h[:shard])
+        sample = paths.first(5).join(", ")
+        suffix =
+          if paths.empty?
+            " (no paths)"
+          elsif paths.size > 5
+            " (#{paths.size} files total)"
+          else
+            ""
+          end
+        Polyrun::Log.orchestration_warn "polyrun run-shards: WORKER TIMEOUT after #{timeout_sec}s (wall time since worker spawn) — shard #{h[:shard]} pid #{h[:pid]}."
+        Polyrun::Log.warn "polyrun run-shards: timeout shard includes: #{sample}#{suffix}"
+        Polyrun::Log.warn "polyrun run-shards: override with --worker-timeout SEC or POLYRUN_WORKER_TIMEOUT_SEC; recorded exit #{WORKER_TIMEOUT_EXIT_STATUS} for this worker."
+      end
+    end
+  end
+end
+# rubocop:enable Polyrun/FileLength, Metrics/ModuleLength

data/lib/polyrun/cli/run_shards_plan_boot_phases.rb

@@ -19,6 +19,9 @@ module Polyrun
         err = run_shards_validate_cmd(cmd)
         return [:fail, err] if err
 
+        run_shards_normalize_worker_timeout_option!(o)
+        run_shards_normalize_worker_idle_timeout_option!(o)
+
         cmd = Shellwords.split(cmd.first) if cmd.size == 1 && cmd.first.include?(" ")
 
         [:ok, o, cmd]
@@ -118,9 +121,39 @@ module Polyrun
           merge_failures: run_shards_merge_failures_flag(o, cfg),
           merge_failures_output: run_shards_merge_failures_output_opt(o, cfg),
           merge_failures_format: run_shards_merge_failures_format_opt(o, cfg),
-          config_path: config_path
+          config_path: config_path,
+          worker_timeout_sec: run_shards_resolved_worker_timeout_sec(o),
+          worker_idle_timeout_sec: run_shards_resolved_worker_idle_timeout_sec(o)
         }
       end
+
+      def run_shards_normalize_worker_idle_timeout_option!(o)
+        v = o[:worker_idle_timeout_sec]
+        return if v.nil?
+
+        o[:worker_idle_timeout_sec] = nil if v <= 0
+      end
+
+      def run_shards_resolved_worker_idle_timeout_sec(o)
+        cli = o[:worker_idle_timeout_sec]
+        return cli.to_f if cli.is_a?(Numeric) && cli > 0
+
+        env_worker_idle_timeout_sec
+      end
+
+      def run_shards_normalize_worker_timeout_option!(o)
+        v = o[:worker_timeout_sec]
+        return if v.nil?
+
+        o[:worker_timeout_sec] = nil if v <= 0
+      end
+
+      def run_shards_resolved_worker_timeout_sec(o)
+        cli = o[:worker_timeout_sec]
+        return cli.to_f if cli.is_a?(Numeric) && cli > 0
+
+        env_worker_timeout_sec
+      end
     end
   end
 end

data/lib/polyrun/cli/run_shards_plan_options.rb

@@ -27,7 +27,9 @@ module Polyrun
           merge_format: nil,
           merge_failures: false,
           merge_failures_output: nil,
-          merge_failures_format: nil
+          merge_failures_format: nil,
+          worker_timeout_sec: nil,
+          worker_idle_timeout_sec: nil
         }
       end
 
@@ -39,8 +41,10 @@ module Polyrun
 
       # rubocop:disable Metrics/AbcSize -- one argv block for run-shards
       def run_shards_plan_options_register!(opts, st)
-        opts.banner = "usage: polyrun run-shards [--workers N] [--strategy NAME] [--paths-file P] [--timing P] [--timing-granularity VAL] [--constraints P] [--seed S] [--merge-coverage] [--merge-output P] [--merge-format LIST] [--merge-failures] [--merge-failures-output P] [--merge-failures-format jsonl|json] [--] <command> [args...]"
+        opts.banner = "usage: polyrun run-shards [--workers N] [--worker-timeout SEC] [--worker-idle-timeout SEC] [--strategy NAME] [--paths-file P] [--timing P] [--timing-granularity VAL] [--constraints P] [--seed S] [--merge-coverage] [--merge-output P] [--merge-format LIST] [--merge-failures] [--merge-failures-output P] [--merge-failures-format jsonl|json] [--] <command> [args...]"
         opts.on("--workers N", Integer) { |v| st[:workers] = v }
+        opts.on("--worker-timeout SEC", Float, "Max seconds per worker since spawn (also POLYRUN_WORKER_TIMEOUT_SEC); kills stuck workers (exit 124)") { |v| st[:worker_timeout_sec] = v }
+        opts.on("--worker-idle-timeout SEC", Float, "Max seconds since last valid WorkerPing timestamp in POLYRUN_WORKER_PING_FILE (needs prior ping); RSpec/Minitest: install_worker_ping!; Quick: automatic; exit 125") { |v| st[:worker_idle_timeout_sec] = v }
         opts.on("--strategy NAME", String) { |v| st[:strategy] = v }
         opts.on("--seed VAL") { |v| st[:seed] = v }
         opts.on("--paths-file PATH", String) { |v| st[:paths_file] = v }

data/lib/polyrun/cli/run_shards_run.rb

@@ -2,6 +2,7 @@ require "shellwords"
 require "rbconfig"
 
 require_relative "run_shards_planning"
+require_relative "run_shards_worker_interrupt"
 require_relative "run_shards_parallel_children"
 
 module Polyrun
@@ -9,6 +10,7 @@ module Polyrun
     # Partition + spawn workers for `polyrun run-shards` (keeps {RunShardsCommand} file small).
     module RunShardsRun
       include RunShardsPlanning
+      include RunShardsWorkerInterrupt
       include RunShardsParallelChildren
 
       private
@@ -93,7 +95,11 @@ module Polyrun
               "POLYRUN_SUITE_EXIT_STATUS" => exit_code.to_s,
               "POLYRUN_MERGED_FAILURES_PATH" => merged_failures_path.to_s
             )
-            hook_cfg.run_phase_if_enabled(:after_suite, env_after)
+            begin
+              hook_cfg.run_phase_if_enabled(:after_suite, env_after)
+            rescue Interrupt
+              Polyrun::Log.warn "polyrun run-shards: after_suite hook interrupted; workers are stopped or were not started"
+            end
           end
         end
       end
@@ -106,38 +112,6 @@ module Polyrun
         Polyrun::Log.warn "polyrun run-shards: each worker prints its own summary line; the last \"N examples\" line is not a total across shards."
       end
 
-      # Best-effort worker teardown then exit. Does not return.
-      def run_shards_shutdown_on_signal!(pids, code)
-        run_shards_terminate_children!(pids)
-        exit(code)
-      rescue Interrupt
-        pids.each do |h|
-          Process.kill(:KILL, h[:pid])
-        rescue Errno::ESRCH
-          # already reaped
-        end
-        pids.each do |h|
-          Process.wait(h[:pid])
-        rescue Errno::ESRCH, Errno::ECHILD, Interrupt
-          # already reaped or give up
-        end
-        exit(code)
-      end
-
-      # Send SIGTERM to each worker PID and wait so Ctrl+C / SIGTERM does not leave orphans.
-      def run_shards_terminate_children!(pids)
-        pids.each do |h|
-          Process.kill(:TERM, h[:pid])
-        rescue Errno::ESRCH
-          # already reaped
-        end
-        pids.each do |h|
-          Process.wait(h[:pid])
-        rescue Errno::ESRCH, Errno::ECHILD
-          # already reaped
-        end
-      end
-
       def run_shards_merge_or_hint_coverage(ctx)
         if ctx[:merge_coverage]
           mo = ctx[:merge_output] || "coverage/merged.json"

data/lib/polyrun/cli/run_shards_worker_interrupt.rb

@@ -0,0 +1,75 @@
+module Polyrun
+  class CLI
+    # SIGINT/SIGTERM handling and non-blocking reap for parallel worker PIDs (used by run-shards / ci-shard fan-out).
+    module RunShardsWorkerInterrupt
+      private
+
+      def run_shards_log_interrupt_workers(pids, _ctx)
+        parts = pids.map { |h| "shard=#{h[:shard]} pid=#{h[:pid]}" }
+        Polyrun::Log.orchestration_warn "polyrun run-shards: SIGINT/SIGTERM while waiting on workers — stopping: #{parts.join(", ")}"
+        Polyrun::Log.warn "polyrun run-shards: search this log for each shard's started … pid= line and RSpec output; repeat SIGINT during cleanup escalates to SIGKILL"
+      end
+
+      # Best-effort worker teardown then exit. Does not return.
+      def run_shards_shutdown_on_signal!(pids, code)
+        run_shards_log_interrupt_workers(pids, nil)
+        run_shards_terminate_children!(pids)
+        exit(code)
+      rescue Interrupt
+        run_shards_signal_workers_kill(pids)
+        run_shards_reap_worker_pids_interruptible(pids.map { |h| h[:pid] })
+        exit(code)
+      end
+
+      # Send SIGTERM to each worker PID and wait so Ctrl+C / SIGTERM does not leave orphans.
+      def run_shards_terminate_children!(pids)
+        run_shards_signal_workers_term(pids)
+        run_shards_reap_worker_pids_interruptible(pids.map { |h| h[:pid] })
+      end
+
+      def run_shards_signal_workers_term(pids)
+        pids.each do |h|
+          Process.kill(:TERM, h[:pid])
+        rescue Errno::ESRCH
+        end
+      end
+
+      def run_shards_signal_workers_kill(pids)
+        pids.each do |h|
+          Process.kill(:KILL, h[:pid])
+        rescue Errno::ESRCH
+        end
+      end
+
+      # Reap child PIDs without blocking uninterruptibly on one stuck zombie (avoids noisy stacks on repeat Ctrl+C).
+      def run_shards_reap_worker_pids_interruptible(pids)
+        pending = pids.compact.uniq
+        force_note = false
+        until pending.empty?
+          pending.reject! do |pid|
+            w = Process.wait(pid, Process::WNOHANG)
+            next true if w == pid
+
+            false
+          rescue Errno::ECHILD
+            true
+          end
+          break if pending.empty?
+
+          begin
+            sleep(0.05)
+          rescue Interrupt
+            unless force_note
+              force_note = true
+              Polyrun::Log.orchestration_warn "polyrun run-shards: repeated SIGINT during worker cleanup — SIGKILL to #{pending.size} process(es)"
+            end
+            pending.each do |pid|
+              Process.kill(:KILL, pid)
+            rescue Errno::ESRCH
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/polyrun/coverage/merge/formatters_html.rb

@@ -1,3 +1,4 @@
+# rubocop:disable Polyrun/FileLength -- HTML merge formatter + helpers in one file
 require "cgi"
 require "digest/sha1"
 require "erb"
@@ -9,6 +10,7 @@ module Polyrun
       module_function
 
       # Standalone HTML report with summary, file table, and per-file source details.
+      # rubocop:disable Metrics/AbcSize -- linear assembly of overview, file table, sections, asset reads
       def emit_html(coverage_blob, title: "Polyrun coverage", root: nil, groups: nil, generated_at: Time.now)
         files = coverage_blob.keys.sort.map { |path| html_file_payload(path, coverage_blob[path], root) }
         summary = html_summary(files)
@@ -31,6 +33,7 @@ module Polyrun
           javascript: File.read(html_javascript_path)
         )
       end
+      # rubocop:enable Metrics/AbcSize
 
       def html_asset_dir
         File.join(__dir__, "html")
@@ -197,3 +200,4 @@ module Polyrun
     end
   end
 end
+# rubocop:enable Polyrun/FileLength

data/lib/polyrun/hooks.rb

@@ -118,6 +118,9 @@ module Polyrun
       if reg&.any?(phase)
         begin
           reg.run(phase, merged)
+        rescue Interrupt
+          Polyrun::Log.warn "polyrun hooks: #{phase} ruby hook interrupted"
+          return 130
         rescue => e
           Polyrun::Log.warn "polyrun hooks: #{phase} ruby hook failed: #{e.class}: #{e.message}"
           return 1
@@ -125,7 +128,12 @@ module Polyrun
       end
 
       commands_for(phase).each do |cmd|
-        ok = system(merged, "sh", "-c", cmd)
+        ok = begin
+          system(merged, "sh", "-c", cmd)
+        rescue Interrupt
+          Polyrun::Log.warn "polyrun hooks: #{phase} shell hook interrupted"
+          return 130
+        end
         return $?.exitstatus unless ok
       end
       0

data/lib/polyrun/log.rb

@@ -6,6 +6,9 @@ module Polyrun
   #
   #   Polyrun::Log.stderr = Logger.new($stderr)
   #   Polyrun::Log.stdout = StringIO.new
+  #
+  # Orchestration (+orchestration_warn+): worker timeout and SIGINT lines use the same sink as +warn+ unless
+  # +POLYRUN_ORCHESTRATION_STDERR=1+ and stderr is not process +$stderr+ (then the summary is copied to +$stderr+).
   module Log
     class << self
       attr_writer :stderr
@@ -25,6 +28,19 @@ module Polyrun
         emit_line(stderr, msg)
       end
 
+      # Like {#warn}, and when +POLYRUN_ORCHESTRATION_STDERR=1+ and {#stderr} is not the process +$stderr+,
+      # also writes one line to +$stderr+ so timeout/interrupt attribution survives custom/null Log sinks.
+      def orchestration_warn(msg)
+        warn(msg)
+        return unless %w[1 true yes].include?(ENV["POLYRUN_ORCHESTRATION_STDERR"]&.downcase)
+        return if stderr.equal?($stderr)
+
+        # Intentionally the real stderr stream (+Kernel#warn+ routes through +Log.stderr+).
+        # rubocop:disable Style/StderrPuts
+        $stderr.puts(msg.to_s.chomp)
+        # rubocop:enable Style/StderrPuts
+      end
+
       def puts(msg = "")
         if msg.nil?
           stdout.write("\n")

data/lib/polyrun/minitest.rb

@@ -7,11 +7,45 @@ module Polyrun
   # after Rails / DB configuration (same timing as a direct call to
   # {Data::ParallelProvisioning.run_suite_hooks!}).
   module Minitest
+    module WorkerPingTestHook
+      def setup
+        Polyrun::WorkerPing.ping!(location: polyrun_minitest_location)
+        super
+      end
+
+      def teardown
+        super
+        Polyrun::WorkerPing.ping!(location: polyrun_minitest_location)
+      end
+
+      private
+
+      def polyrun_minitest_location
+        file, line = method(name).source_location
+        (file && line) ? "#{file}:#{line}" : nil
+      rescue NameError
+        nil
+      end
+    end
+
     module_function
 
     # Runs {Data::ParallelProvisioning.run_suite_hooks!} (serial vs shard worker hooks).
     def install_parallel_provisioning!
       Polyrun::Data::ParallelProvisioning.run_suite_hooks!
     end
+
+    # Same ping semantics as {RSpec.install_worker_ping!}: +ping!+ at test +setup+ and +teardown+.
+    # Requires +minitest+ to be loaded first (+Minitest::Test+ defined).
+    def install_worker_ping!
+      require_relative "worker_ping"
+      unless defined?(::Minitest::Test)
+        Polyrun::Log.warn "polyrun minitest: install_worker_ping! skipped (load minitest/autorun or minitest/test first)"
+        return
+      end
+
+      ::Minitest::Test.send(:prepend, WorkerPingTestHook)
+      Polyrun::WorkerPing.ensure_interval_ping_thread!
+    end
   end
 end

data/lib/polyrun/quick/example_runner.rb

@@ -1,7 +1,10 @@
+require_relative "../worker_ping"
 require_relative "assertions"
 require_relative "errors"
 require_relative "matchers"
 
+Polyrun::WorkerPing.ensure_interval_ping_thread!
+
 module Polyrun
   module Quick
     # Per-example execution: merged lets, hooks, assertions, optional Capybara::DSL.
@@ -20,6 +23,8 @@ module Polyrun
         define_let_methods!
         run_let_bangs_from_chain
         extend_capybara_if_enabled!
+        qloc = quick_example_location(block)
+        Polyrun::WorkerPing.ping!(location: qloc)
         begin
           run_before_hooks_from_chain(ancestor_chain)
           instance_eval(&block)
@@ -32,11 +37,17 @@ module Polyrun
           run_after_hooks_from_chain(ancestor_chain)
           reset_capybara_if_enabled!
           @_let_cache = {}
+          Polyrun::WorkerPing.ping!(location: qloc)
         end
       end
 
       private
 
+      def quick_example_location(block)
+        loc = block&.source_location
+        loc ? "#{loc[0]}:#{loc[1]}" : nil
+      end
+
       def merge_lets_from_chain(ancestor_chain)
         @merged_lets = {}
         ancestor_chain.each do |g|

data/lib/polyrun/rspec.rb

@@ -44,5 +44,23 @@ module Polyrun
         config.add_formatter Polyrun::Reporting::RspecFailureFragmentFormatter
       end
     end
+
+    # Writes {WorkerPing} after suite start, before/after each example (+location+ is file:line from metadata).
+    # Keeps +--worker-idle-timeout+ sensitive to example progress (not only a background thread).
+    def install_worker_ping!
+      require "rspec/core"
+      require_relative "worker_ping"
+      ::RSpec.configure do |config|
+        config.before(:suite) { Polyrun::WorkerPing.ping! }
+        config.before(:each) do |example|
+          Polyrun::WorkerPing.ping!(location: example.metadata[:location] || example.location)
+        end
+        config.after(:each) do |example|
+          Polyrun::WorkerPing.ping!(location: example.metadata[:location] || example.location)
+        end
+      end
+
+      Polyrun::WorkerPing.ensure_interval_ping_thread!
+    end
   end
 end

data/lib/polyrun/version.rb

@@ -1,3 +1,3 @@
 module Polyrun
-  VERSION = "1.4.2"
+  VERSION = "1.5.0"
 end

data/lib/polyrun/worker_ping.rb

@@ -0,0 +1,74 @@
+module Polyrun
+  # Writes a monotonic timestamp to +POLYRUN_WORKER_PING_FILE+ when the test process advances
+  # (typically once per example). When +location:+ is passed (path:line of the example), the file
+  # is two lines: timestamp, then that string. Parents use +--worker-idle-timeout+ to detect a worker with no
+  # progress *inside* a single example—unlike a background thread, +ping!+ does not run while Ruby
+  # is busy on the main thread, so a tight CPU loop or stuck native code leaves the timestamp stale.
+  #
+  # Prefer framework installs (call from helpers *after* loading the runner):
+  #
+  #   require "polyrun/rspec"
+  #   Polyrun::RSpec.install_worker_ping!
+  #
+  #   require "polyrun/minitest"
+  #   Polyrun::Minitest.install_worker_ping!
+  #
+  # Polyrun Quick runs +ping!+ automatically when requiring the Quick stack.
+  #
+  # Optional interval thread (+POLYRUN_WORKER_PING_THREAD=1+, +POLYRUN_WORKER_PING_INTERVAL_SEC+): call
+  # {#ensure_interval_ping_thread!} once at worker startup if you rely on periodic pings without per-example {#ping!};
+  # installers call this so the env toggle works out of the box.
+  module WorkerPing
+    class << self
+      def ping!(location: nil)
+        path = ping_file_path
+        return if path.empty?
+
+        t = Process.clock_gettime(Process::CLOCK_MONOTONIC).to_s
+        loc = location.to_s.strip
+        payload = loc.empty? ? t : "#{t}\n#{loc}"
+        File.binwrite(path, payload)
+      rescue SystemCallError
+        # best-effort
+      end
+
+      def ping_file_path
+        ENV["POLYRUN_WORKER_PING_FILE"].to_s.strip
+      end
+
+      # Starts a periodic +ping!+ thread when +POLYRUN_WORKER_PING_THREAD+ is truthy and +POLYRUN_WORKER_PING_FILE+ is set.
+      # Prefer per-example {#ping!}; safe to call more than once (idempotent).
+      # rubocop:disable ThreadSafety/ClassInstanceVariable -- idempotent once-per-process latch
+      def ensure_interval_ping_thread!
+        thread_flag = ENV["POLYRUN_WORKER_PING_THREAD"]
+        return unless %w[1 true yes].include?(thread_flag&.downcase)
+
+        path = ping_file_path
+        return if path.empty?
+
+        @interval_ping_mx ||= Mutex.new
+        @interval_ping_mx.synchronize do
+          return if @interval_ping_started
+
+          raw = ENV["POLYRUN_WORKER_PING_INTERVAL_SEC"].to_s.strip
+          interval = Float(raw.empty? ? "15" : raw, exception: false) || 15.0
+          interval = 1.0 if interval < 1.0
+
+          ping!
+          # rubocop:disable ThreadSafety/NewThread -- optional periodic ping alongside per-example ping!
+          Thread.new do
+            loop do
+              sleep(interval)
+              ping!
+            rescue SystemCallError, Interrupt
+              break
+            end
+          end
+          # rubocop:enable ThreadSafety/NewThread
+          @interval_ping_started = true
+        end
+      end
+      # rubocop:enable ThreadSafety/ClassInstanceVariable
+    end
+  end
+end

data/sig/polyrun/minitest.rbs

@@ -1,5 +1,7 @@
 module Polyrun
   module Minitest
     def self.install_parallel_provisioning!: () -> void
+
+    def self.install_worker_ping!: () -> void
   end
 end

data/sig/polyrun/rspec.rbs

@@ -3,5 +3,9 @@ module Polyrun
     def self.install_parallel_provisioning!: (untyped rspec_config) -> void
 
     def self.install_example_timing!: (?output_path: String? ) -> void
+
+    def self.install_failure_fragments!: (?only_if: untyped?) -> void
+
+    def self.install_worker_ping!: () -> void
   end
 end

data/sig/polyrun/worker_ping.rbs

@@ -0,0 +1,10 @@
+module Polyrun
+  module WorkerPing
+    def self.ping!: (?location: String?) -> void
+
+    def self.ping_file_path: () -> String
+
+    # Idempotent — installers call once; POLYRUN_WORKER_PING_THREAD gates the periodic thread.
+    def self.ensure_interval_ping_thread!: () -> void
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: polyrun
 version: !ruby/object:Gem::Version
-  version: 1.4.2
+  version: 1.5.0
 platform: ruby
 authors:
 - Andrei Makarov
@@ -188,10 +188,12 @@ files:
 - lib/polyrun/cli/report_commands.rb
 - lib/polyrun/cli/run_shards_command.rb
 - lib/polyrun/cli/run_shards_parallel_children.rb
+- lib/polyrun/cli/run_shards_parallel_wait.rb
 - lib/polyrun/cli/run_shards_plan_boot_phases.rb
 - lib/polyrun/cli/run_shards_plan_options.rb
 - lib/polyrun/cli/run_shards_planning.rb
 - lib/polyrun/cli/run_shards_run.rb
+- lib/polyrun/cli/run_shards_worker_interrupt.rb
 - lib/polyrun/cli/start_bootstrap.rb
 - lib/polyrun/cli/timing_command.rb
 - lib/polyrun/config.rb
@@ -280,6 +282,7 @@ files:
 - lib/polyrun/timing/rspec_example_formatter.rb
 - lib/polyrun/timing/summary.rb
 - lib/polyrun/version.rb
+- lib/polyrun/worker_ping.rb
 - polyrun.gemspec
 - sig/polyrun.rbs
 - sig/polyrun/cli.rbs
@@ -289,6 +292,7 @@ files:
 - sig/polyrun/minitest.rbs
 - sig/polyrun/quick.rbs
 - sig/polyrun/rspec.rbs
+- sig/polyrun/worker_ping.rbs
 homepage: https://github.com/amkisko/polyrun.rb
 licenses:
 - MIT