polyrun 1.0.0

data/lib/polyrun/minitest.rb

@@ -0,0 +1,17 @@
+require_relative "../polyrun"
+
+module Polyrun
+  # Optional Minitest-oriented wiring (require +polyrun/minitest+ explicitly).
+  #
+  # Does not load the +minitest+ gem. Call {install_parallel_provisioning!} from +test/test_helper.rb+
+  # after Rails / DB configuration (same timing as a direct call to
+  # {Data::ParallelProvisioning.run_suite_hooks!}).
+  module Minitest
+    module_function
+
+    # Runs {Data::ParallelProvisioning.run_suite_hooks!} (serial vs shard worker hooks).
+    def install_parallel_provisioning!
+      Polyrun::Data::ParallelProvisioning.run_suite_hooks!
+    end
+  end
+end

data/lib/polyrun/partition/constraints.rb

@@ -0,0 +1,69 @@
+module Polyrun
+  module Partition
+    # Hard constraints for plan assignment (spec_queue.md): pins, serial globs.
+    # Pins win over serial_glob. First matching pin glob wins.
+    class Constraints
+      attr_reader :pin_map, :serial_globs, :serial_shard
+
+      # @param pin_map [Hash{String=>Integer}] glob or exact path => shard index
+      # @param serial_globs [Array<String>] fnmatch patterns forced to +serial_shard+ (default 0) unless pinned
+      # @param root [String] project root for expanding relative paths
+      def initialize(pin_map: {}, serial_globs: [], serial_shard: 0, root: nil)
+        @pin_map = pin_map.transform_keys(&:to_s).transform_values { |v| Integer(v) }
+        @serial_globs = Array(serial_globs).map(&:to_s)
+        @serial_shard = Integer(serial_shard)
+        @root = root ? File.expand_path(root) : Dir.pwd
+      end
+
+      def self.from_hash(h, root: nil)
+        h = h.transform_keys(&:to_s) if h.is_a?(Hash)
+        return new(root: root) unless h.is_a?(Hash)
+
+        pins = h["pin"] || h["pins"] || {}
+        serial = h["serial_glob"] || h["serial_globs"] || []
+        serial_shard = h["serial_shard"] || 0
+        new(
+          pin_map: pins.is_a?(Hash) ? pins : {},
+          serial_globs: serial.is_a?(Array) ? serial : [],
+          serial_shard: serial_shard,
+          root: root
+        )
+      end
+
+      # Returns Integer shard index if constrained, or nil if free to place by LPT/HRW.
+      def forced_shard_for(path)
+        rel = path.to_s
+        abs = File.expand_path(rel, @root)
+
+        @pin_map.each do |pattern, shard|
+          next if pattern.to_s.empty?
+
+          if match_pattern?(pattern.to_s, rel, abs)
+            return shard
+          end
+        end
+
+        @serial_globs.each do |g|
+          if match_pattern?(g, rel, abs)
+            return @serial_shard
+          end
+        end
+
+        nil
+      end
+
+      def any?
+        @pin_map.any? || @serial_globs.any?
+      end
+
+      private
+
+      def match_pattern?(pattern, rel, abs)
+        p = pattern.to_s
+        File.fnmatch?(p, rel, File::FNM_PATHNAME | File::FNM_EXTGLOB) ||
+          File.fnmatch?(p, abs, File::FNM_PATHNAME | File::FNM_EXTGLOB) ||
+          p == rel || p == abs
+      end
+    end
+  end
+end

data/lib/polyrun/partition/hrw.rb

@@ -0,0 +1,33 @@
+require "digest"
+
+module Polyrun
+  module Partition
+    # Rendezvous / highest-hash shard assignment (spec_queue.md): stateless, stable when m changes.
+    module Hrw
+      module_function
+
+      # @return [Integer] shard index in 0...m
+      def shard_for(path:, total_shards:, seed: "")
+        m = Integer(total_shards)
+        raise Polyrun::Error, "total_shards must be >= 1" if m < 1
+
+        best_j = 0
+        best = -1
+        salt = seed.to_s
+        p = path.to_s
+        m.times do |j|
+          h = score(p, j, salt)
+          if h > best
+            best = h
+            best_j = j
+          end
+        end
+        best_j
+      end
+
+      def score(path, shard_index, salt)
+        Digest::SHA256.digest("#{salt}\n#{path}\n#{shard_index}").unpack1("H*").hex
+      end
+    end
+  end
+end

data/lib/polyrun/partition/min_heap.rb

@@ -0,0 +1,64 @@
+module Polyrun
+  module Partition
+    # Binary min-heap of [load, shard_index] for LPT placement (tie-break: lower shard index).
+    class MinHeap
+      def initialize
+        @a = []
+      end
+
+      def push(load, shard_index)
+        @a << [load.to_f, Integer(shard_index)]
+        sift_up(@a.size - 1)
+      end
+
+      def pop_min
+        return nil if @a.empty?
+
+        min = @a[0]
+        last = @a.pop
+        @a[0] = last if @a.any?
+        sift_down(0) if @a.size > 1
+        min
+      end
+
+      def empty?
+        @a.empty?
+      end
+
+      private
+
+      def sift_up(i)
+        while i.positive?
+          p = (i - 1) / 2
+          break unless less(@a[i], @a[p])
+
+          @a[i], @a[p] = @a[p], @a[i]
+          i = p
+        end
+      end
+
+      def sift_down(i)
+        n = @a.size
+        loop do
+          l = 2 * i + 1
+          r = l + 1
+          smallest = i
+          smallest = l if l < n && less(@a[l], @a[smallest])
+          smallest = r if r < n && less(@a[r], @a[smallest])
+          break if smallest == i
+
+          @a[i], @a[smallest] = @a[smallest], @a[i]
+          i = smallest
+        end
+      end
+
+      # Compare [load, j]: lower load wins; tie -> lower j
+      def less(x, y)
+        return true if x[0] < y[0]
+        return false if x[0] > y[0]
+
+        x[1] < y[1]
+      end
+    end
+  end
+end

data/lib/polyrun/partition/paths.rb

@@ -0,0 +1,28 @@
+module Polyrun
+  module Partition
+    # Shared spec path listing for run-shards, queue init, and plan helpers.
+    module Paths
+      module_function
+
+      def read_lines(path)
+        File.read(File.expand_path(path.to_s, Dir.pwd)).split("\n").map(&:strip).reject(&:empty?)
+      end
+
+      # When +paths_file+ is set but missing, returns +{ error: "..." }+.
+      # Otherwise returns +{ items:, source: }+ (human-readable source label).
+      def resolve_run_shard_items(paths_file: nil, cwd: Dir.pwd)
+        if paths_file
+          abs = File.expand_path(paths_file.to_s, cwd)
+          unless File.file?(abs)
+            return {error: "paths file not found: #{abs}"}
+          end
+          {items: read_lines(abs), source: paths_file.to_s}
+        elsif File.file?(File.join(cwd, "spec", "spec_paths.txt"))
+          {items: read_lines(File.join(cwd, "spec", "spec_paths.txt")), source: "spec/spec_paths.txt"}
+        else
+          {items: Dir.glob(File.join(cwd, "spec/**/*_spec.rb")).sort, source: "spec/**/*_spec.rb glob"}
+        end
+      end
+    end
+  end
+end

data/lib/polyrun/partition/paths_build.rb

@@ -0,0 +1,128 @@
+require "fileutils"
+require "pathname"
+require "set"
+
+module Polyrun
+  module Partition
+    # Writes +partition.paths_file+ from +partition.paths_build+
+    module PathsBuild
+      module_function
+
+      # @return [Integer] 0 on success or skip, 2 on configuration error
+      def apply!(partition:, cwd: Dir.pwd)
+        return 0 if skip_paths_build?
+
+        pb = partition["paths_build"] || partition[:paths_build]
+        return 0 unless pb.is_a?(Hash) && !pb.empty?
+
+        paths_file = (partition["paths_file"] || partition[:paths_file] || "spec/spec_paths.txt").to_s
+        out_abs = File.expand_path(paths_file, cwd)
+        lines = build_ordered_paths(pb, cwd)
+        FileUtils.mkdir_p(File.dirname(out_abs))
+        File.write(out_abs, lines.join("\n") + "\n")
+        Polyrun::Log.warn "polyrun paths-build: wrote #{lines.size} path(s) → #{paths_file}"
+        0
+      rescue Polyrun::Error => e
+        Polyrun::Log.warn "polyrun paths-build: #{e.message}"
+        2
+      end
+
+      def skip_paths_build?
+        v = ENV["POLYRUN_SKIP_PATHS_BUILD"].to_s.downcase
+        %w[1 true yes].include?(v)
+      end
+
+      # Builds ordered path strings relative to +cwd+ (forward slashes).
+      def build_ordered_paths(pb, cwd)
+        pb = stringify_keys(pb)
+        all_glob = pb["all_glob"].to_s
+        all_glob = "spec/**/*_spec.rb" if all_glob.empty?
+
+        pool = glob_under_cwd(all_glob, cwd)
+        pool.uniq!
+        stages = Array(pb["stages"])
+        return sort_paths(pool) if stages.empty?
+
+        apply_stages_to_pool(stages, pool, cwd)
+      end
+
+      def apply_stages_to_pool(stages, pool, cwd)
+        remaining = Set.new(pool)
+        out = []
+        stages.each do |raw|
+          st = stringify_keys(raw)
+          taken =
+            if st["glob"]
+              take_glob_paths(st, remaining, cwd)
+            elsif st["regex"]
+              take_regex_paths(st, remaining)
+            else
+              raise Polyrun::Error, 'paths_build stage needs "glob" or "regex"'
+            end
+          out.concat(taken)
+          remaining.subtract(taken)
+        end
+        out.concat(sort_paths(remaining.to_a))
+        out
+      end
+
+      def take_glob_paths(st, remaining, cwd)
+        taken = glob_under_cwd(st["glob"].to_s, cwd).select { |p| remaining.include?(p) }
+        if st["sort_by_substring_order"]
+          subs = Array(st["sort_by_substring_order"]).map(&:to_s)
+          def_prio = int_or(st["default_priority"], int_or(st["default_sort_key"], 99))
+          taken.sort_by! { |p| [substring_priority(p, subs, def_prio), p] }
+        else
+          sort_paths!(taken)
+        end
+        taken
+      end
+
+      def take_regex_paths(st, remaining)
+        ic = st["ignore_case"]
+        ignore_case = ic == true || %w[1 true yes].include?(ic.to_s.downcase)
+        rx = Regexp.new(st["regex"].to_s, ignore_case ? Regexp::IGNORECASE : 0)
+        taken = remaining.to_a.select { |p| rx.match?(p) || rx.match?(File.basename(p)) }
+        sort_paths!(taken)
+        taken
+      end
+
+      def glob_under_cwd(pattern, cwd)
+        root = File.expand_path(cwd)
+        Dir.glob(File.join(root, pattern)).map { |p| normalize_rel(p, cwd) }
+      end
+
+      def normalize_rel(path, cwd)
+        abs = File.expand_path(path, cwd)
+        Pathname.new(abs).relative_path_from(Pathname.new(File.expand_path(cwd))).to_s.tr("\\", "/")
+      end
+
+      def sort_paths(paths)
+        paths.sort
+      end
+
+      def sort_paths!(paths)
+        paths.sort!
+      end
+
+      def substring_priority(path, substrings, default)
+        substrings.each_with_index do |s, i|
+          return i if path.include?(s)
+        end
+        default
+      end
+
+      def stringify_keys(h)
+        return {} unless h.is_a?(Hash)
+
+        h.each_with_object({}) { |(k, v), o| o[k.to_s] = v }
+      end
+
+      def int_or(v, fallback)
+        Integer(v)
+      rescue ArgumentError, TypeError
+        fallback
+      end
+    end
+  end
+end

data/lib/polyrun/partition/plan.rb

@@ -0,0 +1,189 @@
+require "json"
+
+require_relative "constraints"
+require_relative "hrw"
+require_relative "min_heap"
+require_relative "stable_shuffle"
+
+module Polyrun
+  module Partition
+    # Assigns discrete items (e.g. spec paths) to shards (spec_queue.md).
+    #
+    # Strategies:
+    # - +round_robin+ — sorted paths, assign by index mod +total_shards+.
+    # - +random_round_robin+ — Fisher–Yates shuffle (optional +seed+), then same mod assignment.
+    # - +cost_binpack+ (+cost+, +binpack+, +timing+) — LPT greedy binpack using per-path weights;
+    #   optional {Constraints} for pins / serial globs before LPT on the rest.
+    # - +hrw+ (+rendezvous+) — rendezvous hashing for minimal remapping when m changes; optional constraints.
+    class Plan
+      COST_STRATEGIES = %w[cost cost_binpack binpack timing].freeze
+      HRW_STRATEGIES = %w[hrw rendezvous].freeze
+
+      attr_reader :items, :total_shards, :strategy, :seed, :constraints
+
+      def initialize(items:, total_shards:, strategy: "round_robin", seed: nil, costs: nil, constraints: nil, root: nil)
+        @items = items.map(&:to_s).freeze
+        @total_shards = Integer(total_shards)
+        raise Polyrun::Error, "total_shards must be >= 1" if @total_shards < 1
+
+        @strategy = strategy.to_s
+        @seed = seed
+        @root = root ? File.expand_path(root) : Dir.pwd
+        @constraints = normalize_constraints(constraints)
+        @costs = normalize_costs(costs)
+
+        validate_constraints_strategy_combo!
+        if cost_strategy? && (@costs.nil? || @costs.empty?)
+          raise Polyrun::Error,
+            "strategy #{@strategy} requires a timing map (path => seconds), e.g. merged polyrun_timing.json"
+        end
+      end
+
+      def ordered_items
+        @ordered_items ||= case strategy
+        when "round_robin"
+          items.sort
+        when "random_round_robin"
+          StableShuffle.call(items.sort, random_seed)
+        when "cost", "cost_binpack", "binpack", "timing"
+          items.sort
+        when "hrw", "rendezvous"
+          items.sort
+        else
+          raise Polyrun::Error, "unknown partition strategy: #{strategy}"
+        end
+      end
+
+      def shard(shard_index)
+        idx = Integer(shard_index)
+        raise Polyrun::Error, "shard_index out of range" if idx < 0 || idx >= total_shards
+
+        if cost_strategy?
+          cost_shards[idx]
+        elsif hrw_strategy?
+          hrw_shards[idx]
+        else
+          mod_shards[idx]
+        end
+      end
+
+      def shard_weight_totals
+        if cost_strategy?
+          cost_shards.map { |paths| paths.sum { |p| weight_for(p) } }
+        elsif hrw_strategy?
+          hrw_shards.map { |paths| paths.sum { |p| weight_for_optional(p) } }
+        else
+          []
+        end
+      end
+
+      def manifest(shard_index)
+        m = {
+          "shard_index" => Integer(shard_index),
+          "shard_total" => total_shards,
+          "strategy" => strategy,
+          "seed" => seed,
+          "paths" => shard(shard_index)
+        }
+        secs = shard_weight_totals
+        m["shard_seconds"] = secs if cost_strategy? || (hrw_strategy? && secs.any? { |x| x > 0 })
+        m
+      end
+
+      def self.load_timing_costs(path)
+        abs = File.expand_path(path.to_s, Dir.pwd)
+        return {} unless File.file?(abs)
+
+        data = JSON.parse(File.read(abs))
+        return {} unless data.is_a?(Hash)
+
+        out = {}
+        data.each do |k, v|
+          key = File.expand_path(k.to_s, Dir.pwd)
+          out[key] = v.to_f
+        end
+        out
+      end
+
+      def self.cost_strategy?(name)
+        COST_STRATEGIES.include?(name.to_s)
+      end
+
+      def self.hrw_strategy?(name)
+        HRW_STRATEGIES.include?(name.to_s)
+      end
+
+      private
+
+      def cost_strategy?
+        self.class.cost_strategy?(strategy)
+      end
+
+      def hrw_strategy?
+        self.class.hrw_strategy?(strategy)
+      end
+
+      def normalize_constraints(c)
+        return nil if c.nil?
+
+        c.is_a?(Constraints) ? c : Constraints.from_hash(c, root: @root)
+      end
+
+      def normalize_costs(costs)
+        return nil if costs.nil?
+
+        c = {}
+        costs.each do |k, v|
+          key = File.expand_path(k.to_s, @root)
+          c[key] = v.to_f
+        end
+        c
+      end
+
+      def validate_constraints_strategy_combo!
+        return unless @constraints&.any?
+        return if cost_strategy? || hrw_strategy?
+
+        raise Polyrun::Error,
+          "partition constraints require strategy cost_binpack (with --timing) or hrw/rendezvous"
+      end
+
+      def default_weight
+        return @default_weight if defined?(@default_weight)
+
+        vals = @costs&.values || []
+        @default_weight =
+          if vals.empty?
+            1.0
+          else
+            vals.sum / vals.size
+          end
+      end
+
+      def weight_for(path)
+        abs = File.expand_path(path.to_s, @root)
+        return @costs[abs] if @costs&.key?(abs)
+
+        default_weight
+      end
+
+      def weight_for_optional(path)
+        abs = File.expand_path(path.to_s, @root)
+        return @costs[abs] if @costs&.key?(abs)
+
+        0.0
+      end
+
+      def cost_shards
+        @cost_shards ||= build_lpt_buckets
+      end
+
+      def build_lpt_buckets
+        PlanLptBuckets.new(self).build
+      end
+    end
+  end
+end
+
+require_relative "plan_sharding"
+require_relative "plan_lpt"

data/lib/polyrun/partition/plan_lpt.rb

@@ -0,0 +1,49 @@
+require_relative "min_heap"
+
+module Polyrun
+  module Partition
+    # LPT greedy binpack for cost strategies (extracted from {Plan} for size limits).
+    class PlanLptBuckets
+      def initialize(plan)
+        @plan = plan
+      end
+
+      def build
+        buckets = Array.new(@plan.total_shards) { [] }
+        totals = Array.new(@plan.total_shards, 0.0)
+        lpt_fill_forced!(buckets, totals)
+        lpt_balance_free!(buckets, totals)
+        buckets
+      end
+
+      private
+
+      def lpt_fill_forced!(buckets, totals)
+        @plan.items.each do |item|
+          next unless @plan.constraints && (j = @plan.constraints.forced_shard_for(item))
+
+          j = Integer(j)
+          raise Polyrun::Error, "constraint shard #{j} out of range" if j < 0 || j >= @plan.total_shards
+
+          buckets[j] << item
+          totals[j] += @plan.send(:weight_for, item)
+        end
+      end
+
+      def lpt_balance_free!(buckets, totals)
+        free = @plan.items.reject { |item| @plan.constraints&.forced_shard_for(item) }
+        pairs = free.map { |p| [p, @plan.send(:weight_for, p)] }
+        pairs.sort_by! { |(p, w)| [-w, p] }
+
+        heap = MinHeap.new
+        @plan.total_shards.times { |j| heap.push(totals[j], j) }
+
+        pairs.each do |path, w|
+          load, j = heap.pop_min
+          buckets[j] << path
+          heap.push(load + w, j)
+        end
+      end
+    end
+  end
+end

data/lib/polyrun/partition/plan_sharding.rb

@@ -0,0 +1,48 @@
+require_relative "hrw"
+
+module Polyrun
+  module Partition
+    class Plan
+      def hrw_shards
+        @hrw_shards ||= begin
+          buckets = Array.new(total_shards) { [] }
+          salt = hrw_salt
+          items.each do |path|
+            j =
+              if @constraints && (fj = @constraints.forced_shard_for(path))
+                Integer(fj)
+              else
+                Hrw.shard_for(path: path, total_shards: total_shards, seed: salt)
+              end
+            raise Polyrun::Error, "constraint shard out of range" if j < 0 || j >= total_shards
+
+            buckets[j] << path
+          end
+          buckets
+        end
+      end
+
+      # One pass over +ordered_items+ (round_robin / random_round_robin); avoids O(workers × n) rescans in +shard+.
+      def mod_shards
+        @mod_shards ||= begin
+          list = ordered_items
+          buckets = Array.new(total_shards) { [] }
+          list.each_with_index { |path, i| buckets[i % total_shards] << path }
+          buckets
+        end
+      end
+
+      def hrw_salt
+        s = seed
+        (s.nil? || s.to_s.empty?) ? "polyrun-hrw" : s.to_s
+      end
+
+      def random_seed
+        s = seed
+        return Integer(s) if s && s != ""
+
+        0
+      end
+    end
+  end
+end

data/lib/polyrun/partition/stable_shuffle.rb

@@ -0,0 +1,18 @@
+module Polyrun
+  module Partition
+    # Deterministic Fisher–Yates shuffle (spec_queue.md).
+    module StableShuffle
+      module_function
+
+      def call(items, seed)
+        rng = Random.new(Integer(seed))
+        a = items.dup
+        (a.size - 1).downto(1) do |i|
+          j = rng.rand(i + 1)
+          a[i], a[j] = a[j], a[i]
+        end
+        a
+      end
+    end
+  end
+end

data/lib/polyrun/prepare/artifacts.rb

@@ -0,0 +1,40 @@
+require "digest/sha2"
+require "json"
+
+module Polyrun
+  module Prepare
+    # Writes +polyrun-artifacts.json+ (spec2 §3) for cache keys and CI upload lists.
+    module Artifacts
+      VERSION = 1
+
+      module_function
+
+      # +entries+ is array of hashes: { "path" => ..., "kind" => "file"|"directory", optional "digest", "size" }
+      def write!(root:, recipe:, entries:, dry_run: false)
+        list = entries.map { |e| normalize_entry(e) }
+        doc = {
+          "version" => VERSION,
+          "recipe" => recipe,
+          "dry_run" => dry_run,
+          "generated_at" => Time.now.utc.iso8601,
+          "artifacts" => list
+        }
+        path = File.join(root, "polyrun-artifacts.json")
+        File.write(path, JSON.pretty_generate(doc))
+        path
+      end
+
+      def normalize_entry(e)
+        h = e.transform_keys(&:to_s)
+        p = h["path"].to_s
+        kind = h["kind"] || (File.directory?(p) ? "directory" : "file")
+        out = {"path" => p, "kind" => kind}
+        if File.exist?(p)
+          out["size"] = File.size(p) if File.file?(p)
+          out["digest"] = h["digest"] || (File.file?(p) ? "sha256:#{Digest::SHA256.file(p).hexdigest}" : nil)
+        end
+        out.compact
+      end
+    end
+  end
+end