polyrun 1.2.0 → 1.4.0

data/lib/polyrun/hooks/dsl.rb

@@ -0,0 +1,128 @@
+require "monitor"
+
+require_relative "../log"
+
+module Polyrun
+  class Hooks
+    # Ruby DSL for +hooks.ruby+ / +hooks.ruby_file+ in +polyrun.yml+ (see README).
+    #
+    # Example file (+config/polyrun_hooks.rb+):
+    #
+    #   before(:suite) { |env| puts env["POLYRUN_HOOK_PHASE"] }
+    #   after(:each) { |env| }
+    #
+    # Blocks receive a hash with string keys (same env as shell hooks).
+    module Dsl
+      class Registry
+        def initialize
+          @phases = Hash.new { |h, k| h[k] = [] }
+        end
+
+        def add(phase, proc)
+          @phases[phase.to_sym] << proc
+        end
+
+        # @param env [Hash] string-keyed env
+        def run(phase, env)
+          @phases[phase.to_sym].each { |pr| pr.call(env) }
+        end
+
+        def any?(phase)
+          @phases[phase.to_sym].any?
+        end
+
+        def empty?
+          @phases.values.all?(&:empty?)
+        end
+
+        def worker_hooks?
+          any?(:before_worker) || any?(:after_worker)
+        end
+      end
+
+      # Evaluates a hook file with +before+ / +after+ (+before(:suite)+, etc.).
+      class FileContext
+        def initialize(path)
+          @path = path
+          @registry = Registry.new
+        end
+
+        attr_reader :registry
+
+        def load_file
+          instance_eval(File.read(@path), @path)
+          @registry
+        end
+
+        def before(sym, &block)
+          @registry.add(map_before(sym), block)
+        end
+
+        def after(sym, &block)
+          @registry.add(map_after(sym), block)
+        end
+
+        private
+
+        def map_before(sym)
+          case sym.to_sym
+          when :suite then :before_suite
+          when :all then :before_shard
+          when :each then :before_worker
+          else
+            raise ArgumentError, "hooks DSL: before(#{sym.inspect}) — use :suite, :all, or :each"
+          end
+        end
+
+        def map_after(sym)
+          case sym.to_sym
+          when :suite then :after_suite
+          when :all then :after_shard
+          when :each then :after_worker
+          else
+            raise ArgumentError, "hooks DSL: after(#{sym.inspect}) — use :suite, :all, or :each"
+          end
+        end
+      end
+
+      class << self
+        # @return [Registry, nil]
+        def load_registry(path)
+          return nil if path.nil? || path.to_s.strip.empty?
+
+          full = File.expand_path(path.to_s, Dir.pwd)
+          return nil unless File.file?(full)
+
+          mtime = File.mtime(full)
+          cache_mu.synchronize do
+            hit = registry_cache[full]
+            return hit[:registry] if hit && hit[:mtime] == mtime
+
+            reg = FileContext.new(full).load_file
+            registry_cache[full] = {mtime: mtime, registry: reg}
+            reg
+          end
+        rescue => e
+          Polyrun::Log.warn "polyrun hooks: failed to load #{full}: #{e.class}: #{e.message}"
+          nil
+        end
+
+        def clear_cache!
+          cache_mu.synchronize { registry_cache.clear }
+        end
+
+        private
+
+        # rubocop:disable ThreadSafety/ClassInstanceVariable -- single-threaded hook load; Monitor protects cache
+        def cache_mu
+          @cache_mu ||= Monitor.new
+        end
+
+        def registry_cache
+          @registry_cache ||= {}
+        end
+        # rubocop:enable ThreadSafety/ClassInstanceVariable
+      end
+    end
+  end
+end

data/lib/polyrun/hooks/worker_runner.rb

@@ -0,0 +1,27 @@
+require_relative "../log"
+
+module Polyrun
+  class Hooks
+    # Invoked by worker shell wrapper (+ruby -e+). Requires +POLYRUN_HOOKS_RUBY_FILE+.
+    module WorkerRunner
+      module_function
+
+      # @return [Integer] exit code (0 on success)
+      def run!(phase)
+        phase = phase.to_sym
+        path = ENV["POLYRUN_HOOKS_RUBY_FILE"]
+        return 0 if path.nil? || path.empty?
+
+        registry = Dsl.load_registry(path)
+        return 0 if registry.nil? || !registry.any?(phase)
+
+        env = ENV.to_h
+        registry.run(phase, env)
+        0
+      rescue => e
+        Polyrun::Log.warn "polyrun hooks worker #{phase}: #{e.class}: #{e.message}"
+        1
+      end
+    end
+  end
+end

data/lib/polyrun/hooks/worker_shell.rb

@@ -0,0 +1,50 @@
+require "shellwords"
+require "rbconfig"
+
+module Polyrun
+  class Hooks
+    # Builds +sh -c+ script for worker processes (shell + Ruby +before_worker+ / +after_worker+).
+    module WorkerShell
+      # @param cmd [Array<String>] argv before paths
+      # @param paths [Array<String>]
+      # @return [String] shell script body for +sh -c+ (worker process)
+      # rubocop:disable Metrics/AbcSize -- shell + ruby worker hook branches
+      def build_worker_shell_script(cmd, paths)
+        main = Shellwords.join(cmd + paths)
+        rb = RbConfig.ruby
+        bw_shell = commands_for(:before_worker)
+        aw_shell = commands_for(:after_worker)
+        bw_ruby = ruby_registry&.any?(:before_worker)
+        aw_ruby = ruby_registry&.any?(:after_worker)
+
+        lines = []
+        lines << "export POLYRUN_HOOK_PHASE=before_worker"
+        if bw_ruby || bw_shell.any?
+          lines << "set -e"
+          lines << worker_ruby_line(rb, :before_worker) if bw_ruby
+          bw_shell.each { |c| lines << c }
+        end
+        lines << "set +e"
+        lines << main
+        lines << "ec=$?"
+        lines << "export POLYRUN_HOOK_PHASE=after_worker"
+        if aw_ruby || aw_shell.any?
+          lines << "set +e"
+          aw_shell.each { |c| lines << "( #{c} ) || true" }
+          lines << worker_ruby_line(rb, :after_worker, wrap_allow_fail: true) if aw_ruby
+        end
+        lines << "exit $ec"
+        lines.join("\n")
+      end
+      # rubocop:enable Metrics/AbcSize
+
+      private
+
+      def worker_ruby_line(rb_exe, phase, wrap_allow_fail: false)
+        code = %(require "polyrun"; Polyrun::Hooks::WorkerRunner.run!(:#{phase}))
+        line = "#{rb_exe} -e #{Shellwords.escape(code)}"
+        wrap_allow_fail ? "( #{line} ) || true" : line
+      end
+    end
+  end
+end

data/lib/polyrun/hooks.rb

@@ -0,0 +1,185 @@
+require "shellwords"
+
+require_relative "log"
+require_relative "hooks/dsl"
+require_relative "hooks/worker_runner"
+require_relative "hooks/worker_shell"
+
+module Polyrun
+  # Shell and Ruby DSL hooks around parallel orchestration, named like RSpec lifecycle callbacks:
+  # +before_suite+ / +after_suite+ (+before(:suite)+ / +after(:suite)+),
+  # +before_shard+ / +after_shard+ (parent process, per partition index),
+  # +before_worker+ / +after_worker+ (inside each worker process, around the test command).
+  #
+  # Configure under +hooks:+ in +polyrun.yml+: shell strings, and/or +ruby:+ path to a Ruby DSL file.
+  # Run manually: +polyrun hook run <phase>+ (see {CLI}).
+  #
+  # Orchestration respects +POLYRUN_HOOKS_DISABLE=1+ (+run_phase_if_enabled+); +polyrun hook run+ always runs {#run_phase}.
+  class Hooks
+    include WorkerShell
+
+    PHASES = %i[
+      before_suite after_suite
+      before_shard after_shard
+      before_worker after_worker
+    ].freeze
+
+    attr_reader :ruby_file
+
+    def self.disabled?
+      v = ENV["POLYRUN_HOOKS_DISABLE"].to_s.downcase
+      %w[1 true yes].include?(v)
+    end
+
+    # When +POLYRUN_SHARD_TOTAL+ is greater than 1 (+ci-shard-run+ matrix), suite hooks are skipped by default; set
+    # +POLYRUN_HOOKS_SUITE_PER_MATRIX_JOB=1+ to run +before_suite+ / +after_suite+ on every matrix job (legacy).
+    def self.suite_per_matrix_job?
+      v = ENV["POLYRUN_HOOKS_SUITE_PER_MATRIX_JOB"].to_s.downcase
+      %w[1 true yes].include?(v)
+    end
+
+    def self.from_config(cfg)
+      raw = cfg.respond_to?(:hooks) ? cfg.hooks : {}
+      new(raw.is_a?(Hash) ? raw : {})
+    end
+
+    # Maps CLI or YAML-style names (+before_suite+, +"before(:suite)"+) to a phase symbol or +nil+.
+    def self.parse_phase(str)
+      new({}).send(:canonical_key, str)
+    end
+
+    # @param raw [Hash] +hooks+ block from YAML
+    def initialize(raw)
+      @ruby_file = extract_ruby_file(raw)
+      h = {}
+      raw.each do |k, v|
+        next if %w[ruby ruby_file].include?(k.to_s)
+
+        ck = canonical_key(k)
+        next if ck.nil?
+
+        h[ck] = v
+      end
+      @raw = h.freeze
+      @ruby_registry_loaded = false
+      @ruby_registry = nil
+    end
+
+    def empty?
+      no_shell = PHASES.all? { |p| commands_for(p).empty? }
+      return false unless no_shell
+
+      return true if @ruby_file.nil? || @ruby_file.to_s.strip.empty?
+      return true unless File.file?(File.expand_path(@ruby_file, Dir.pwd))
+
+      reg = ruby_registry
+      reg.nil? || reg.empty?
+    end
+
+    def worker_hooks?
+      return true if commands_for(:before_worker).any? || commands_for(:after_worker).any?
+
+      !!ruby_registry&.worker_hooks?
+    end
+
+    # Merges +POLYRUN_HOOKS_RUBY_FILE+ when a DSL file is configured (for worker +ruby -e+).
+    def merge_worker_ruby_env(env)
+      return env unless @ruby_file
+      abs = File.expand_path(@ruby_file, Dir.pwd)
+      return env unless File.file?(abs)
+
+      env.merge("POLYRUN_HOOKS_RUBY_FILE" => abs)
+    end
+
+    # @param phase [Symbol]
+    # @return [Array<String>]
+    def commands_for(phase)
+      v = @raw[phase.to_sym]
+      case v
+      when nil then []
+      when Array then v.map(&:to_s).map(&:strip).reject(&:empty?)
+      else
+        s = v.to_s.strip
+        s.empty? ? [] : [s]
+      end
+    end
+
+    # Runs Ruby DSL blocks (if any), then shell commands for +phase+.
+    # @return [Integer] exit code (0 if no commands)
+    def run_phase(phase, env)
+      return 0 unless PHASES.include?(phase.to_sym)
+
+      merged = stringify_env_for_hook(env).merge(
+        "POLYRUN_HOOK_PHASE" => phase.to_s,
+        "POLYRUN_HOOK" => "1"
+      )
+
+      reg = ruby_registry
+      if reg&.any?(phase)
+        begin
+          reg.run(phase, merged)
+        rescue => e
+          Polyrun::Log.warn "polyrun hooks: #{phase} ruby hook failed: #{e.class}: #{e.message}"
+          return 1
+        end
+      end
+
+      commands_for(phase).each do |cmd|
+        ok = system(merged, "sh", "-c", cmd)
+        return $?.exitstatus unless ok
+      end
+      0
+    end
+
+    # Like {#run_phase}, but no-ops when {disabled?} (+POLYRUN_HOOKS_DISABLE=1+). Used by run-shards / ci-shard orchestration.
+    def run_phase_if_enabled(phase, env)
+      return 0 if self.class.disabled?
+
+      run_phase(phase, env)
+    end
+
+    private
+
+    def extract_ruby_file(raw)
+      v = raw["ruby"] || raw[:ruby] || raw["ruby_file"] || raw[:ruby_file]
+      return nil if v.nil?
+
+      s = v.to_s.strip
+      s.empty? ? nil : s
+    end
+
+    def ruby_registry
+      return @ruby_registry if @ruby_registry_loaded
+
+      @ruby_registry_loaded = true
+      @ruby_registry = Dsl.load_registry(@ruby_file)
+    end
+
+    def stringify_env_for_hook(env)
+      h = {}
+      env.each { |k, v| h[k.to_s] = v }
+      h
+    end
+
+    # Accept RSpec-style quoted keys from YAML, e.g. +"before(:suite)"+.
+    def canonical_key(k)
+      s = k.to_s.strip
+      sym = if s.match?(/\Abefore\(\s*:suite\s*\)\z/i)
+        :before_suite
+      elsif s.match?(/\Aafter\(\s*:suite\s*\)\z/i)
+        :after_suite
+      elsif s.match?(/\Abefore\(\s*:all\s*\)\z/i)
+        :before_shard
+      elsif s.match?(/\Aafter\(\s*:all\s*\)\z/i)
+        :after_shard
+      elsif s.match?(/\Abefore\(\s*:each\s*\)\z/i)
+        :before_worker
+      elsif s.match?(/\Aafter\(\s*:each\s*\)\z/i)
+        :after_worker
+      else
+        s.downcase.tr("-", "_").to_sym
+      end
+      PHASES.include?(sym) ? sym : nil
+    end
+  end
+end

data/lib/polyrun/templates/ci_matrix.polyrun.yml

@@ -3,8 +3,9 @@
 #   bundle exec polyrun -c polyrun.yml ci-shard-run -- bundle exec rspec
 # (or ci-shard-rspec; or e.g. ci-shard-run -- bundle exec polyrun quick).
 # Equivalent to build-paths, plan --shard/--total, then run that command with this slice's paths.
-# A separate CI job downloads coverage/polyrun-fragment-*.json and runs merge-coverage.
-# Do not use parallel-rspec with multiple workers inside the same matrix row unless you intend nested parallelism.
+# A separate CI job downloads coverage/polyrun-fragment-*.json (e.g. shard<S>-worker<W>.json per N×M process) and runs merge-coverage.
+# For N×M (N matrix jobs × M processes per job): set shard_processes: M or POLYRUN_SHARD_PROCESSES,
+# or pass --shard-processes M to ci-shard-run / ci-shard-rspec (local split is round-robin).
 # See: docs/SETUP_PROFILE.md
 
 partition:

data/lib/polyrun/version.rb

@@ -1,3 +1,3 @@
 module Polyrun
-  VERSION = "1.2.0"
+  VERSION = "1.4.0"
 end

data/lib/polyrun.rb

@@ -25,6 +25,7 @@ require_relative "polyrun/database/shard"
 require_relative "polyrun/database/url_builder"
 require_relative "polyrun/database/provision"
 require_relative "polyrun/database/clone_shards"
+require_relative "polyrun/hooks"
 require_relative "polyrun/env/ci"
 require_relative "polyrun/timing/merge"
 require_relative "polyrun/timing/summary"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: polyrun
 version: !ruby/object:Gem::Version
-  version: 1.2.0
+  version: 1.4.0
 platform: ruby
 authors:
 - Andrei Makarov
@@ -166,7 +166,9 @@ files:
 - docs/SETUP_PROFILE.md
 - lib/polyrun.rb
 - lib/polyrun/cli.rb
+- lib/polyrun/cli/ci_shard_hooks.rb
 - lib/polyrun/cli/ci_shard_run_command.rb
+- lib/polyrun/cli/ci_shard_run_parse.rb
 - lib/polyrun/cli/config_command.rb
 - lib/polyrun/cli/coverage_commands.rb
 - lib/polyrun/cli/coverage_merge_io.rb
@@ -175,6 +177,7 @@ files:
 - lib/polyrun/cli/env_commands.rb
 - lib/polyrun/cli/help.rb
 - lib/polyrun/cli/helpers.rb
+- lib/polyrun/cli/hooks_command.rb
 - lib/polyrun/cli/init_command.rb
 - lib/polyrun/cli/plan_command.rb
 - lib/polyrun/cli/prepare_command.rb
@@ -183,6 +186,7 @@ files:
 - lib/polyrun/cli/quick_command.rb
 - lib/polyrun/cli/report_commands.rb
 - lib/polyrun/cli/run_shards_command.rb
+- lib/polyrun/cli/run_shards_parallel_children.rb
 - lib/polyrun/cli/run_shards_plan_boot_phases.rb
 - lib/polyrun/cli/run_shards_plan_options.rb
 - lib/polyrun/cli/run_shards_planning.rb
@@ -196,6 +200,7 @@ files:
 - lib/polyrun/coverage/cobertura_zero_lines.rb
 - lib/polyrun/coverage/collector.rb
 - lib/polyrun/coverage/collector_finish.rb
+- lib/polyrun/coverage/collector_fragment_meta.rb
 - lib/polyrun/coverage/filter.rb
 - lib/polyrun/coverage/formatter.rb
 - lib/polyrun/coverage/merge.rb
@@ -223,6 +228,10 @@ files:
 - lib/polyrun/database/url_builder/template_prepare.rb
 - lib/polyrun/debug.rb
 - lib/polyrun/env/ci.rb
+- lib/polyrun/hooks.rb
+- lib/polyrun/hooks/dsl.rb
+- lib/polyrun/hooks/worker_runner.rb
+- lib/polyrun/hooks/worker_shell.rb
 - lib/polyrun/log.rb
 - lib/polyrun/minitest.rb
 - lib/polyrun/partition/constraints.rb