polyrun 1.5.0 → 2.1.2

data/lib/polyrun/coverage/merge/formatters_html.rb

@@ -23,14 +23,14 @@ module Polyrun
         )
         file_list_html = render_html_partial("file_list", file_rows_html: files.map { |file| html_file_list_row(file) }.join("\n"))
         file_sections_html = files.map { |file| render_html_partial("file_section", file: file) }.join("\n")
-        ERB.new(File.read(html_template_path), trim_mode: "-").result_with_hash(
+        ERB.new(File.read(html_template_path, encoding: Encoding::UTF_8), trim_mode: "-").result_with_hash(
           title: CGI.escapeHTML(title.to_s),
           generated_label: html_generated_label(generated_at),
           overview_html: overview_html,
           file_list_html: file_list_html,
           file_sections_html: file_sections_html,
-          stylesheet: File.read(html_stylesheet_path),
-          javascript: File.read(html_javascript_path)
+          stylesheet: File.read(html_stylesheet_path, encoding: Encoding::UTF_8),
+          javascript: File.read(html_javascript_path, encoding: Encoding::UTF_8)
         )
       end
       # rubocop:enable Metrics/AbcSize
@@ -56,7 +56,7 @@ module Polyrun
       end
 
       def render_html_partial(name, locals = {})
-        ERB.new(File.read(html_partial_path(name)), trim_mode: "-").result_with_hash(locals)
+        ERB.new(File.read(html_partial_path(name), encoding: Encoding::UTF_8), trim_mode: "-").result_with_hash(locals)
       end
 
       def html_file_payload(path, file, root)
@@ -152,7 +152,7 @@ module Polyrun
       def html_source_lines(path, fallback_length)
         return Array.new(fallback_length, "") unless File.file?(path.to_s)
 
-        File.readlines(path.to_s, chomp: true)
+        File.readlines(path.to_s, chomp: true, encoding: Encoding::UTF_8)
       rescue Errno::ENOENT, Errno::EACCES, ArgumentError
         Array.new(fallback_length, "")
       end

data/lib/polyrun/data/factory_counts.rb

@@ -6,11 +6,19 @@ module Polyrun
       class << self
         def reset!
           @counts = Hash.new(0)
+          @example_counts = Hash.new(0)
+        end
+
+        def reset_example!
+          @example_counts = Hash.new(0)
         end
 
         def record(factory_name)
           @counts ||= Hash.new(0)
-          @counts[factory_name.to_s] += 1
+          @example_counts ||= Hash.new(0)
+          name = factory_name.to_s
+          @counts[name] += 1
+          @example_counts[name] += 1
         end
 
         def counts
@@ -18,6 +26,11 @@ module Polyrun
           @counts.dup
         end
 
+        def example_counts
+          @example_counts ||= Hash.new(0)
+          @example_counts.dup
+        end
+
         def summary_lines(top: 20)
           @counts ||= Hash.new(0)
           sorted = @counts.sort_by { |_, n| -n }

data/lib/polyrun/database/clone_shards.rb

@@ -57,6 +57,8 @@ module Polyrun
             end
 
             plan.each { |row| create_one_shard!(row, replace, force_drop, dry_run) }
+          rescue => e
+            raise Polyrun::Error, "CloneShards shard_index=#{shard_index}: #{e.message}"
           end
         end
         threads.each(&:join)

data/lib/polyrun/database/shard.rb

@@ -44,11 +44,12 @@ module Polyrun
 
         return u unless u.match?(%r{\A[a-z][a-z0-9+.-]*://}i)
 
-        if (m = u.match(%r{/([^/?]+)(\?|$)}))
+        if (m = u.match(%r{\A[a-z][a-z0-9+.-]*://[^/?#]+/([^/?]+)(\?|$)}i))
           base = m[1]
           suffixed = "#{base}_#{Integer(shard_index)}"
           u.sub(%r{/#{Regexp.escape(base)}(\?|$)}, "/#{suffixed}\\1")
         else
+          Polyrun::Log.warn "polyrun database: URL has no database segment; shard suffix skipped: #{u}"
           u
         end
       end

data/lib/polyrun/minitest.rb

@@ -47,5 +47,14 @@ module Polyrun
       ::Minitest::Test.send(:prepend, WorkerPingTestHook)
       Polyrun::WorkerPing.ensure_interval_ping_thread!
     end
+
+    # Per-test spec quality when +POLYRUN_SPEC_QUALITY=1+ (requires stdlib +Coverage+ for line deltas).
+    def install_spec_quality!(only_if: nil, root: nil, output_path: nil)
+      pred = only_if || -> { Polyrun::SpecQuality.enabled? }
+      return unless pred.call
+
+      require_relative "spec_quality/minitest_hook"
+      Polyrun::SpecQuality::MinitestHook.install!(only_if: pred, root: root, output_path: output_path)
+    end
   end
 end

data/lib/polyrun/partition/hrw.rb

@@ -8,15 +8,42 @@ module Polyrun
 
       # @return [Integer] shard index in 0...m
       def shard_for(path:, total_shards:, seed: "")
+        pick_shard(path: path, total_shards: total_shards, seed: seed) { |p, j, salt| score(p, j, salt) }
+      end
+
+      # Per-shard weights (heterogeneous nodes). Uniform weights match +shard_for+.
+      def weighted_shard_for(path:, total_shards:, seed: "", shard_weights: nil)
+        weights = normalize_shard_weights(shard_weights, total_shards)
+        pick_shard(path: path, total_shards: total_shards, seed: seed) do |p, j, salt|
+          base = score(p, j, salt).to_f
+          w = weights[j]
+          w.positive? ? base / w : base
+        end
+      end
+
+      def normalize_shard_weights(shard_weights, total_shards)
+        m = Integer(total_shards)
+        return Array.new(m, 1.0) if shard_weights.nil? || shard_weights.empty?
+
+        weights = shard_weights.map { |w| w.to_f }
+        if weights.size < m
+          weights += Array.new(m - weights.size, 1.0)
+        elsif weights.size > m
+          weights = weights[0, m]
+        end
+        weights
+      end
+
+      def pick_shard(path:, total_shards:, seed:)
         m = Integer(total_shards)
         raise Polyrun::Error, "total_shards must be >= 1" if m < 1
 
         best_j = 0
-        best = -1
+        best = -1.0
         salt = seed.to_s
         p = path.to_s
         m.times do |j|
-          h = score(p, j, salt)
+          h = yield(p, j, salt)
           if h > best
             best = h
             best_j = j
@@ -26,8 +53,18 @@ module Polyrun
       end
 
       def score(path, shard_index, salt)
-        Digest::SHA256.digest("#{salt}\n#{path}\n#{shard_index}").unpack1("H*").hex
+        digest = Digest::SHA256.digest("#{salt}\n#{path}\n#{shard_index}")
+        if fast_score?
+          digest.unpack1("Q>")
+        else
+          digest.unpack1("H*").hex
+        end
+      end
+
+      def fast_score?
+        %w[1 true yes].include?(ENV["POLYRUN_HRW_FAST_SCORE"]&.to_s&.downcase)
       end
+      private_class_method :fast_score?
     end
   end
 end

data/lib/polyrun/partition/paths_build.rb

@@ -53,7 +53,7 @@ module Polyrun
           st = stringify_keys(raw)
           taken =
             if st["glob"]
-              take_glob_paths(st, remaining, cwd)
+              take_glob_paths(st, remaining)
             elsif st["regex"]
               take_regex_paths(st, remaining)
             else
@@ -66,8 +66,9 @@ module Polyrun
         out
       end
 
-      def take_glob_paths(st, remaining, cwd)
-        taken = glob_under_cwd(st["glob"].to_s, cwd).select { |p| remaining.include?(p) }
+      def take_glob_paths(st, remaining)
+        pattern = st["glob"].to_s
+        taken = remaining.to_a.select { |p| path_matches_glob?(p, pattern) }
         if st["sort_by_substring_order"]
           subs = Array(st["sort_by_substring_order"]).map(&:to_s)
           def_prio = int_or(st["default_priority"], int_or(st["default_sort_key"], 99))
@@ -92,6 +93,10 @@ module Polyrun
         Dir.glob(File.join(root, pattern)).map { |p| normalize_rel(p, cwd) }
       end
 
+      def path_matches_glob?(rel_path, pattern)
+        File.fnmatch?(pattern, rel_path, File::FNM_PATHNAME | File::FNM_EXTGLOB)
+      end
+
       def normalize_rel(path, cwd)
         abs = File.expand_path(path, cwd)
         Pathname.new(abs).relative_path_from(Pathname.new(File.expand_path(cwd))).to_s.tr("\\", "/")

data/lib/polyrun/partition/plan.rb

@@ -1,3 +1,4 @@
+# rubocop:disable Polyrun/FileLength, Metrics/ClassLength -- partition strategies + constraints
 require_relative "timing_keys"
 require_relative "constraints"
 require_relative "hrw"
@@ -15,15 +16,22 @@ module Polyrun
     #   Default +timing_granularity+ is +file+ (one weight per spec file). Experimental +:example+
     #   uses +path:line+ locators and per-example weights in the timing JSON.
     # - +hrw+ (+rendezvous+) — rendezvous hashing for minimal remapping when m changes; optional constraints.
+    # - +weighted_hrw+ — rendezvous with per-shard weights (+shard_weights+); use +stable_cost_binpack+ for path costs.
+    # - +lazy_robin+ — sorted round-robin assignment with timing loaded for diagnostics and +shard_seconds+.
+    # - +preserve_order_round_robin+ — round-robin in paths-file order (no sort); membership from +paths_build+ only.
     class Plan
-      COST_STRATEGIES = %w[cost cost_binpack binpack timing].freeze
-      HRW_STRATEGIES = %w[hrw rendezvous].freeze
+      COST_STRATEGIES = %w[cost cost_binpack binpack timing stable_cost_binpack].freeze
+      HRW_STRATEGIES = %w[hrw rendezvous weighted_hrw].freeze
+      LAZY_ROBIN_STRATEGIES = %w[lazy_robin].freeze
+      MOD_STRATEGIES = %w[round_robin random_round_robin lazy_robin preserve_order_round_robin].freeze
 
-      attr_reader :items, :total_shards, :strategy, :seed, :constraints, :timing_granularity
+      attr_reader :items, :total_shards, :strategy, :seed, :constraints, :timing_granularity, :root
 
-      def initialize(items:, total_shards:, strategy: "round_robin", seed: nil, costs: nil, constraints: nil, root: nil, timing_granularity: :file)
+      def initialize(items:, total_shards:, strategy: "round_robin", seed: nil, costs: nil, constraints: nil, root: nil, timing_granularity: :file, stable_assignment: nil, stable_imbalance_threshold: 1.30, shard_weights: nil)
         @timing_granularity = TimingKeys.normalize_granularity(timing_granularity)
         @root = root ? File.expand_path(root) : Dir.pwd
+        @stable_assignment = normalize_stable_assignment(stable_assignment)
+        @stable_imbalance_threshold = stable_imbalance_threshold.to_f
         @items = items.map do |x|
           if @timing_granularity == :example
             TimingKeys.normalize_locator(x, @root, :example)
@@ -38,23 +46,30 @@ module Polyrun
         @seed = seed
         @constraints = normalize_constraints(constraints)
         @costs = normalize_costs(costs)
+        @shard_weights = shard_weights
 
         validate_constraints_strategy_combo!
         if cost_strategy? && (@costs.nil? || @costs.empty?)
           raise Polyrun::Error,
             "strategy #{@strategy} requires a timing map (path => seconds or path:line => seconds), e.g. merged polyrun_timing.json"
         end
+        if lazy_robin_strategy? && (@costs.nil? || @costs.empty?)
+          raise Polyrun::Error,
+            "strategy lazy_robin requires a timing map (path => seconds), e.g. merged polyrun_timing.json"
+        end
       end
 
       def ordered_items
         @ordered_items ||= case strategy
-        when "round_robin"
+        when "round_robin", "lazy_robin"
           items.sort
+        when "preserve_order_round_robin"
+          items.dup
         when "random_round_robin"
           StableShuffle.call(items.sort, random_seed)
         when "cost", "cost_binpack", "binpack", "timing"
           items.sort
-        when "hrw", "rendezvous"
+        when "hrw", "rendezvous", "weighted_hrw"
           items.sort
         else
           raise Polyrun::Error, "unknown partition strategy: #{strategy}"
@@ -79,11 +94,40 @@ module Polyrun
           cost_shards.map { |paths| paths.sum { |p| weight_for(p) } }
         elsif hrw_strategy?
           hrw_shards.map { |paths| paths.sum { |p| weight_for_optional(p) } }
+        elsif lazy_robin_strategy? && @costs&.any?
+          mod_shards.map { |paths| paths.sum { |p| weight_for(p) } }
         else
           []
         end
       end
 
+      def file_weight(path)
+        (lazy_robin_strategy? || cost_strategy?) ? weight_for(path) : weight_for_optional(path)
+      end
+
+      def shard_file_weights(shard_index)
+        shard(shard_index).map { |p| [p, file_weight(p)] }.sort_by { |(_, w)| [-w, p] }
+      end
+
+      def default_weight
+        vals = @costs&.values || []
+        if vals.empty?
+          1.0
+        else
+          vals.sum / vals.size
+        end
+      end
+
+      def stable_strategy?
+        strategy == "stable_cost_binpack"
+      end
+
+      attr_reader :stable_imbalance_threshold
+
+      def stable_assignment_map
+        @stable_assignment
+      end
+
       def manifest(shard_index)
         m = {
           "shard_index" => Integer(shard_index),
@@ -94,7 +138,7 @@ module Polyrun
         }
         m["timing_granularity"] = timing_granularity.to_s if timing_granularity == :example
         secs = shard_weight_totals
-        m["shard_seconds"] = secs if cost_strategy? || (hrw_strategy? && secs.any? { |x| x > 0 })
+        m["shard_seconds"] = secs if emit_shard_seconds?(secs)
         m
       end
 
@@ -110,6 +154,14 @@ module Polyrun
         HRW_STRATEGIES.include?(name.to_s)
       end
 
+      def self.lazy_robin_strategy?(name)
+        LAZY_ROBIN_STRATEGIES.include?(name.to_s)
+      end
+
+      def self.timing_load_strategy?(name)
+        cost_strategy?(name) || hrw_strategy?(name) || lazy_robin_strategy?(name)
+      end
+
       private
 
       def cost_strategy?
@@ -120,12 +172,38 @@ module Polyrun
         self.class.hrw_strategy?(strategy)
       end
 
+      def lazy_robin_strategy?
+        self.class.lazy_robin_strategy?(strategy)
+      end
+
+      def emit_shard_seconds?(secs)
+        return false if secs.empty?
+
+        cost_strategy? || lazy_robin_strategy? || (hrw_strategy? && secs.any? { |x| x > 0 })
+      end
+
       def normalize_constraints(c)
         return nil if c.nil?
 
         c.is_a?(Constraints) ? c : Constraints.from_hash(c, root: @root)
       end
 
+      def normalize_stable_assignment(map)
+        return nil if map.nil? || map.empty?
+
+        out = {}
+        map.each do |k, v|
+          key =
+            if @timing_granularity == :example
+              TimingKeys.normalize_locator(k.to_s, @root, :example)
+            else
+              File.expand_path(k.to_s, @root)
+            end
+          out[key] = Integer(v)
+        end
+        out
+      end
+
       def normalize_costs(costs)
         return nil if costs.nil?
 
@@ -150,18 +228,6 @@ module Polyrun
           "partition constraints require strategy cost_binpack (with --timing) or hrw/rendezvous"
       end
 
-      def default_weight
-        return @default_weight if defined?(@default_weight)
-
-        vals = @costs&.values || []
-        @default_weight =
-          if vals.empty?
-            1.0
-          else
-            vals.sum / vals.size
-          end
-      end
-
       def weight_for(path)
         key = cost_lookup_key(path.to_s)
         return @costs[key] if @costs&.key?(key)
@@ -197,3 +263,6 @@ end
 
 require_relative "plan_sharding"
 require_relative "plan_lpt"
+require_relative "timing_diagnostics"
+require_relative "reports"
+# rubocop:enable Polyrun/FileLength, Metrics/ClassLength

data/lib/polyrun/partition/plan_lpt.rb

@@ -9,20 +9,63 @@ module Polyrun
       end
 
       def build
+        if @plan.stable_strategy? && @plan.stable_assignment_map&.any?
+          stable = build_from_stable_map
+          return stable if imbalance_ratio(stable) <= @plan.stable_imbalance_threshold
+        end
+
         buckets = Array.new(@plan.total_shards) { [] }
         totals = Array.new(@plan.total_shards, 0.0)
-        lpt_fill_forced!(buckets, totals)
-        lpt_balance_free!(buckets, totals)
+        forced_pairs, free = partition_forced_and_free
+        lpt_apply_forced!(buckets, totals, forced_pairs)
+        lpt_balance_free!(buckets, totals, free)
+        buckets
+      end
+
+      def build_from_stable_map
+        buckets = Array.new(@plan.total_shards) { [] }
+        map = @plan.stable_assignment_map
+        @plan.items.each do |item|
+          key = @plan.send(:cost_lookup_key, item)
+          j = map[key]
+          j = Integer(j) if j
+          j = fallback_shard_for(item) unless j && j >= 0 && j < @plan.total_shards
+          buckets[j] << item
+        end
         buckets
       end
 
+      def fallback_shard_for(item)
+        Hrw.shard_for(path: item, total_shards: @plan.total_shards, seed: @plan.send(:hrw_salt))
+      end
+
+      def imbalance_ratio(buckets)
+        totals = buckets.map { |paths| paths.sum { |p| @plan.send(:weight_for, p) } }
+        return 1.0 if totals.empty?
+
+        avg = totals.sum / totals.size.to_f
+        return 1.0 unless avg.positive?
+
+        totals.max / avg
+      end
+
       private
 
-      def lpt_fill_forced!(buckets, totals)
+      def partition_forced_and_free
+        forced_pairs = []
+        free = []
         @plan.items.each do |item|
-          next unless @plan.constraints && (j = @plan.constraints.forced_shard_for(item))
+          if @plan.constraints && (j = @plan.constraints.forced_shard_for(item))
+            forced_pairs << [item, Integer(j)]
+          else
+            free << item
+          end
+        end
+        [forced_pairs, free]
+      end
 
-          j = Integer(j)
+      def lpt_apply_forced!(buckets, totals, forced_pairs)
+        forced_pairs.each do |item, j|
           raise Polyrun::Error, "constraint shard #{j} out of range" if j < 0 || j >= @plan.total_shards
 
           buckets[j] << item
@@ -30,8 +73,7 @@ module Polyrun
         end
       end
 
-      def lpt_balance_free!(buckets, totals)
-        free = @plan.items.reject { |item| @plan.constraints&.forced_shard_for(item) }
+      def lpt_balance_free!(buckets, totals, free)
         pairs = free.map { |p| [p, @plan.send(:weight_for, p)] }
         pairs.sort_by! { |(p, w)| [-w, p] }
 

data/lib/polyrun/partition/plan_sharding.rb

@@ -7,10 +7,18 @@ module Polyrun
         @hrw_shards ||= begin
           buckets = Array.new(total_shards) { [] }
           salt = hrw_salt
+          weighted = strategy == "weighted_hrw"
           items.each do |path|
             j =
               if @constraints && (fj = @constraints.forced_shard_for(path))
                 Integer(fj)
+              elsif weighted
+                Hrw.weighted_shard_for(
+                  path: path,
+                  total_shards: total_shards,
+                  seed: salt,
+                  shard_weights: @shard_weights
+                )
               else
                 Hrw.shard_for(path: path, total_shards: total_shards, seed: salt)
               end

data/lib/polyrun/partition/reports.rb

@@ -0,0 +1,139 @@
+module Polyrun
+  module Partition
+    # Imbalance and dominant-file reports from {Plan} shard weights.
+    module Reports
+      IMBALANCE_WARN = 1.20
+      IMBALANCE_ATTENTION = 1.50
+      DOMINANT_SHARD_FRACTION = 0.40
+
+      module_function
+
+      def emit_all!(plan)
+        totals = plan.shard_weight_totals
+        return if totals.empty? || totals.all?(&:zero?)
+
+        emit_imbalance!(plan, totals)
+        emit_dominant_files!(plan, totals)
+      end
+
+      def imbalance_metrics(totals)
+        return nil if totals.empty?
+
+        max = totals.max
+        min = totals.min
+        avg = totals.sum / totals.size.to_f
+        ratio = avg.positive? ? max / avg : 1.0
+        slowest = totals.each_with_index.max_by { |v, _| v }&.last
+        {
+          max_shard_seconds: max,
+          min_shard_seconds: min,
+          avg_shard_seconds: avg,
+          imbalance_ratio: ratio,
+          slowest_shard: slowest
+        }
+      end
+
+      # rubocop:disable Metrics/AbcSize -- imbalance summary lines
+      def emit_imbalance!(plan, totals = nil)
+        totals ||= plan.shard_weight_totals
+        m = imbalance_metrics(totals)
+        return unless m
+
+        lines = []
+        lines << "polyrun partition imbalance:"
+        lines << format(
+          "  max=%.2fs min=%.2fs avg=%.2fs imbalance_ratio=%.2f slowest_shard=%d",
+          m[:max_shard_seconds],
+          m[:min_shard_seconds],
+          m[:avg_shard_seconds],
+          m[:imbalance_ratio],
+          m[:slowest_shard]
+        )
+
+        slow_idx = m[:slowest_shard]
+        slow_paths = plan.shard(slow_idx)
+        slow_total = totals[slow_idx]
+        if slow_total.positive? && slow_paths.any?
+          top = plan.shard_file_weights(slow_idx).first
+          if top
+            _path, w = top
+            pct = (w / slow_total) * 100.0
+            lines << format("  largest_file_percent_of_shard=%.1f%%", pct)
+            if pct > DOMINANT_SHARD_FRACTION * 100.0
+              lines << "  hint: single file dominates slowest shard; try --timing-granularity example or split the file"
+            end
+          end
+        end
+
+        plan.total_shards.times do |i|
+          top5 = plan.shard_file_weights(i).first(5)
+          next if top5.empty?
+
+          lines << "  shard #{i} top files:"
+          top5.each do |path, w|
+            lines << format("    %.2fs  %s", w, path)
+          end
+        end
+
+        if m[:imbalance_ratio] > IMBALANCE_ATTENTION
+          lines << "  Attention required: slowest shard is #{format("%.2f", m[:imbalance_ratio])}x average"
+        elsif m[:imbalance_ratio] > IMBALANCE_WARN
+          lines << "  Warning: imbalance_ratio > #{IMBALANCE_WARN}"
+        end
+
+        lines.each { |ln| Polyrun::Log.warn ln }
+      end
+      # rubocop:enable Metrics/AbcSize
+
+      def dominant_candidates(plan, totals = nil)
+        totals ||= plan.shard_weight_totals
+        return [] if totals.empty?
+
+        suite_total = totals.sum
+        return [] if suite_total <= 0
+
+        target = suite_total / plan.total_shards.to_f
+        slow_idx = totals.each_with_index.max_by { |v, _| v }&.last
+        slow_total = slow_idx ? totals[slow_idx] : 0.0
+
+        weights = file_weights_aggregated(plan)
+        weights.filter_map do |path, w|
+          next if w <= target
+
+          mult = w / target
+          reasons = []
+          reasons << "#{format("%.1f", mult)}x target shard time" if mult > 1.0
+          reasons << "split candidate" if slow_total.positive? && w > DOMINANT_SHARD_FRACTION * slow_total
+          {path: path, seconds: w, target: target, multiple: mult, reasons: reasons}
+        end.sort_by { |h| -h[:seconds] }
+      end
+
+      def emit_dominant_files!(plan, totals = nil)
+        candidates = dominant_candidates(plan, totals)
+        return if candidates.empty?
+
+        Polyrun::Log.warn "Attention:"
+        candidates.first(10).each do |c|
+          Polyrun::Log.warn format("  %s: %.1fs", c[:path], c[:seconds])
+          Polyrun::Log.warn format("  This single file is %.1fx the target shard time.", c[:multiple])
+          Polyrun::Log.warn "  Try --timing-granularity example or split this file."
+        end
+      end
+
+      def file_weights_aggregated(plan)
+        by_file = Hash.new(0.0)
+        plan.items.each do |p|
+          w = plan.file_weight(p)
+          key =
+            if plan.timing_granularity == :example
+              TimingDiagnostics.file_from_locator(p.to_s)
+            else
+              p.to_s
+            end
+          by_file[key] += w
+        end
+        by_file
+      end
+    end
+  end
+end