polyrun 1.5.0 → 2.1.2

data/lib/polyrun/spec_quality.rb

@@ -0,0 +1,205 @@
+# rubocop:disable Polyrun/FileLength -- per-example recorder API
+require "json"
+require "fileutils"
+
+require_relative "coverage/example_diff"
+require_relative "partition/timing_keys"
+require_relative "spec_quality/config"
+require_relative "spec_quality/profile"
+require_relative "spec_quality/fragment"
+require_relative "spec_quality/sql_counter"
+require_relative "spec_quality/merge"
+require_relative "spec_quality/plan_loader"
+require_relative "spec_quality/report"
+require_relative "spec_quality/rspec_hook"
+require_relative "spec_quality/minitest_hook"
+
+module Polyrun
+  # Per-example spec quality: coverage line deltas, resource use, optional SQL counts.
+  # Opt-in with +POLYRUN_SPEC_QUALITY=1+ or +Polyrun::SpecQuality.start!+.
+  module SpecQuality
+    class << self
+      def start!(root: Dir.pwd, config_path: nil, output_path: nil, **overrides)
+        return if Config.disabled?
+
+        @config = Config.load(root: root, config_path: config_path, **overrides)
+        @output_path = output_path || Fragment.default_fragment_path
+        @pause_depth = 0
+        @current = nil
+        @rng = Random.new(Process.pid ^ Time.now.to_i)
+
+        Fragment.truncate_fragment!(@output_path) unless fragment_append_mode?
+        SqlCounter.install! if @config["sql_counter"]
+        self
+      end
+
+      def started?
+        instance_variable_defined?(:@config) && @config
+      end
+
+      def enabled?
+        Config.enabled? && !Config.disabled?
+      end
+
+      attr_reader :config
+
+      attr_reader :output_path
+
+      def recording?
+        started? && @current && @pause_depth.zero?
+      end
+
+      def start_example!(location:, wall_start: nil)
+        return unless started?
+        return if paused?
+        return if Config.ignored_example?(location, @config["ignore_examples"])
+        return unless sample_example?
+
+        @current = {
+          location: normalize_location(location),
+          wall_start: wall_start || Process.clock_gettime(Process::CLOCK_MONOTONIC),
+          coverage_before: Coverage::ExampleDiff.peek_blob,
+          profile_before: Profile.snapshot,
+          sql_count: 0,
+          sql_fingerprints: Hash.new(0),
+          factory_counts: {}
+        }
+        Polyrun::Data::FactoryCounts.reset_example! if defined?(Polyrun::Data::FactoryCounts)
+        nil
+      end
+
+      def finish_example!(location: nil, pending: false)
+        return unless started?
+        cur = @current
+        @current = nil
+        return if cur.nil? || paused?
+        return if pending
+
+        loc = location || cur[:location]
+        return if loc.nil? || loc.to_s.empty?
+
+        after_cov = Coverage::ExampleDiff.peek_blob
+        delta = Coverage::ExampleDiff.diff(cur[:coverage_before], after_cov)
+        delta = Coverage::ExampleDiff.apply_track_under(
+          delta,
+          root: @config["root"],
+          track_under: @config["track_under"],
+          ignore_paths: @config["ignore_paths"]
+        )
+
+        profile_after = Profile.snapshot
+        profile_delta = Profile.diff(cur[:profile_before], profile_after)
+        wall = Process.clock_gettime(Process::CLOCK_MONOTONIC) - cur[:wall_start]
+        profile_delta["wall"] = wall
+
+        factory_counts =
+          if defined?(Polyrun::Data::FactoryCounts)
+            Polyrun::Data::FactoryCounts.example_counts
+          else
+            {}
+          end
+
+        row = build_row(cur, loc, delta, profile_delta, factory_counts)
+        Fragment.append_row!(@output_path, row)
+        row
+      end
+
+      def pause
+        @pause_depth += 1
+        if block_given?
+          begin
+            yield
+          ensure
+            resume
+          end
+        end
+      end
+
+      def resume
+        @pause_depth -= 1 if @pause_depth.positive?
+      end
+
+      def paused?
+        @pause_depth.positive?
+      end
+
+      def record_sql!(sql)
+        return unless recording?
+
+        @current[:sql_count] += 1
+        fp = normalize_sql(sql)
+        @current[:sql_fingerprints][fp] += 1
+      end
+
+      def spec_quality_requested_for_quick?(root = Dir.pwd)
+        return false if Config.disabled?
+        return true if %w[1 true yes].include?(ENV["POLYRUN_SPEC_QUALITY"]&.to_s&.downcase)
+        return true if %w[1 true yes].include?(ENV["POLYRUN_SPEC_QUALITY_FRAGMENTS"]&.to_s&.downcase)
+
+        path = File.join(File.expand_path(root), Config::DEFAULT_CONFIG_RELATIVE)
+        return false unless File.file?(path)
+
+        %w[1 true yes].include?(ENV["POLYRUN_QUICK_SPEC_QUALITY"]&.to_s&.downcase)
+      end
+
+      private
+
+      def fragment_append_mode?
+        truthy?(ENV["POLYRUN_SPEC_QUALITY_FRAGMENT_APPEND"])
+      end
+
+      def sample_example?
+        rate = @config["sample"].to_f
+        return true if rate >= 1.0
+        return false if rate <= 0.0
+
+        @rng.rand < rate
+      end
+
+      def normalize_location(location)
+        s = location.to_s.strip
+        return s if s.empty?
+
+        root = @config["root"]
+        if (m = s.match(/\A(.+):(\d+)\z/)) && m[2].match?(/\A\d+\z/)
+          fp = Polyrun::Partition::TimingKeys.canonical_file_path(File.expand_path(m[1], root))
+          return "#{fp}:#{m[2]}"
+        end
+
+        Polyrun::Partition::TimingKeys.canonical_file_path(File.expand_path(s, root))
+      end
+
+      def build_row(cur, location, delta, profile_delta, factory_counts)
+        profile = Profile.slice_profile(profile_delta, @config["profile"])
+        repeated_sql = cur[:sql_fingerprints].select { |_sql, n| n >= min_query_count }.transform_keys(&:to_s)
+
+        {
+          "example" => location.to_s,
+          "unique_lines" => delta[:unique_lines],
+          "line_churn" => delta[:line_churn],
+          "max_line_churn" => delta[:max_line_churn],
+          "lines" => delta[:lines],
+          "profile" => profile,
+          "sql_count" => cur[:sql_count],
+          "repeated_sql" => repeated_sql,
+          "factory_counts" => factory_counts.transform_keys(&:to_s),
+          "polyrun_shard_index" => ENV["POLYRUN_SHARD_INDEX"],
+          "polyrun_shard_total" => ENV["POLYRUN_SHARD_TOTAL"]
+        }.compact
+      end
+
+      def min_query_count
+        @config["min_query_count"].to_i
+      end
+
+      def normalize_sql(sql)
+        sql.to_s.gsub(/\s+/, " ").strip[0, 500]
+      end
+
+      def truthy?(value)
+        Config.send(:truthy?, value)
+      end
+    end
+  end
+end
+# rubocop:enable Polyrun/FileLength

data/lib/polyrun/templates/POLYRUN.md

@@ -40,6 +40,12 @@ Do not combine Model A and Model B in one workflow without a documented reason (
 - `spec/spec_helper.rb`: `require "polyrun"` and collector or Rails helper as appropriate.
 - Fragments: `coverage/polyrun-fragment-<shard>.json` → `polyrun merge-coverage` → `polyrun report-coverage` / `report-junit` for CI.
 
+## Spec quality (optional)
+
+- `POLYRUN_SPEC_QUALITY=1` and `Polyrun::RSpec.install_spec_quality!` in `spec_helper` (requires coverage enabled for line deltas).
+- `config/polyrun_spec_quality.yml` — thresholds and CI gates (`polyrun init --profile spec-quality`).
+- Parallel: `run-shards --merge-spec-quality` → `coverage/polyrun-spec-quality.json` → `report-spec-quality`.
+
 ## Further reading
 
 - Polyrun README: partition, prepare, databases, merge-coverage, `Polyrun::Env::Ci`

data/lib/polyrun/templates/ci_matrix.polyrun.yml

@@ -16,3 +16,7 @@ partition:
   paths_build:
     all_glob: spec/**/*_spec.rb
     stages: []
+
+# Optional reporting (see docs/SPEC_QUALITY.md):
+# reporting:
+#   merge_spec_quality: true

data/lib/polyrun/templates/polyrun_hooks_spec_quality.rb

@@ -0,0 +1,12 @@
+# Optional Polyrun hooks — copy to config/polyrun_hooks.rb and reference from polyrun.yml:
+#   hooks:
+#     ruby_file: config/polyrun_hooks.rb
+#
+# Worker-phase hooks run in each parallel test child (around bundle exec rspec).
+
+before(:each) do |env|
+  next unless env["POLYRUN_HOOK_ORCHESTRATOR"] == "0"
+  next unless defined?(Polyrun::SpecQuality) && Polyrun::SpecQuality.enabled?
+
+  Polyrun::SpecQuality::RspecHook.ensure_started!
+end

data/lib/polyrun/templates/polyrun_spec_quality.yml

@@ -0,0 +1,20 @@
+# Opt-in per-example spec quality (coverage deltas, resource use). See docs/SPEC_QUALITY.md.
+track_under:
+  - lib
+  - app
+min_line_churn: 50
+min_query_count: 20
+hot_line_example_overlap: 10
+strict: false
+sample: 1.0
+ignore_examples: []
+ignore_paths: []
+ignore_query_patterns: []
+profile:
+  - cpu
+  - mem
+sql_counter: false
+# CI gates (used with POLYRUN_SPEC_QUALITY_STRICT=1 or strict: true):
+# minimum_unique_lines_per_example: 1
+# max_zero_hit_examples: 0
+# max_hot_line_overlap: 100

data/lib/polyrun/templates/rails_prepare.polyrun.yml

@@ -17,6 +17,11 @@ prepare:
   rails_root: .
   command: bundle exec ruby bin/test_prepare.rb
 
+# Optional reporting (see docs/SPEC_QUALITY.md):
+# reporting:
+#   merge_spec_quality: true
+#   merge_spec_quality_output: coverage/polyrun-spec-quality.json
+
 # Uncomment and edit when using Postgres template + per-shard DB names from polyrun env:
 # databases:
 #   template_db: my_app_template

data/lib/polyrun/timing/merge.rb

@@ -1,12 +1,11 @@
 require "json"
 
 require_relative "../debug"
+require_relative "stats"
+require_relative "variance_report"
 
 module Polyrun
   module Timing
-    # Merges per-shard timing JSON files (spec2 §2.4): path => wall seconds (float), or (experimental)
-    # +absolute_path:line+ => seconds for per-example timing.
-    # Disjoint suites: values merged by taking the maximum per path when duplicates appear.
     module Merge
       module_function
 
@@ -18,8 +17,8 @@ module Polyrun
 
           data.each do |file, sec|
             f = file.to_s
-            t = sec.to_f
-            merged[f] = merged.key?(f) ? [merged[f], t].max : t
+            entry = sec
+            merged[f] = merged.key?(f) ? Stats.merge_entries(merged[f], entry) : Stats.normalize_entry(entry)
           end
         end
         merged
@@ -29,6 +28,7 @@ module Polyrun
         Polyrun::Debug.log_kv(merge_timing: "merge_and_write", input_count: paths.size, output_path: output_path)
         merged = Polyrun::Debug.time("Timing::Merge.merge_files") { merge_files(paths) }
         Polyrun::Debug.time("Timing::Merge.write JSON") { File.write(output_path, JSON.pretty_generate(merged)) }
+        Timing::VarianceReport.emit_warnings!(merged)
         merged
       end
     end

data/lib/polyrun/timing/rspec_example_formatter.rb

@@ -1,6 +1,6 @@
 require "json"
 
-require "rspec/core/formatters/base_formatter"
+require "rspec/core/formatters"
 
 module Polyrun
   module Timing
@@ -13,23 +13,23 @@ module Polyrun
     # Or {Polyrun::RSpec.install_example_timing!} (+output_path:+ avoids touching +ENV+).
     #
     # Default output path: +ENV["POLYRUN_EXAMPLE_TIMING_OUT"]+ if set, else +polyrun_timing_examples.json+.
-    class RSpecExampleFormatter < RSpec::Core::Formatters::BaseFormatter
-      RSpec::Core::Formatters.register self, :example_finished, :close
+    class RSpecExampleFormatter
+      ::RSpec::Core::Formatters.register self, :example_finished, :close
 
       def initialize(output)
-        super
+        @output = output
         @times = {}
       end
 
       def example_finished(notification)
         ex = notification.example
-        result = ex.execution_result
-        return if result.pending?
+        return if ex.pending?
 
+        result = ex.execution_result
         t = result.run_time
         return unless t
 
-        path = ex.metadata[:absolute_path]
+        path = example_absolute_path(ex)
         return unless path
 
         line = ex.metadata[:line_number]
@@ -48,6 +48,13 @@ module Polyrun
       def timing_output_path
         ENV["POLYRUN_EXAMPLE_TIMING_OUT"] || "polyrun_timing_examples.json"
       end
+
+      private
+
+      def example_absolute_path(example)
+        metadata = example.metadata
+        metadata[:absolute_file_path] || metadata[:file_path] || metadata[:absolute_path]
+      end
     end
   end
 end

data/lib/polyrun/timing/stats.rb

@@ -0,0 +1,76 @@
+module Polyrun
+  module Timing
+    # Normalizes scalar or object timing entries for merge and binpack weight lookup.
+    module Stats
+      STAT_KEYS = %w[last_seconds min max mean p95 runs failures timeouts].freeze
+
+      module_function
+
+      def normalize_entry(value)
+        case value
+        when Hash
+          normalize_hash(value)
+        else
+          sec = value.to_f
+          {
+            "last_seconds" => sec,
+            "min" => sec,
+            "max" => sec,
+            "mean" => sec,
+            "p95" => sec,
+            "runs" => 1,
+            "failures" => 0,
+            "timeouts" => 0
+          }
+        end
+      end
+
+      # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity -- timing hash key coercion
+      def normalize_hash(h)
+        out = {}
+        sec = h["last_seconds"] || h[:last_seconds] || h["seconds"] || h[:seconds]
+        sec = sec.to_f if sec
+        mean = (h["mean"] || h[:mean] || sec)&.to_f
+        out["last_seconds"] = (sec || mean || 0.0).to_f
+        out["min"] = (h["min"] || h[:min] || out["last_seconds"]).to_f
+        out["max"] = (h["max"] || h[:max] || out["last_seconds"]).to_f
+        out["mean"] = (mean || out["last_seconds"]).to_f
+        out["p95"] = (h["p95"] || h[:p95] || out["max"]).to_f
+        out["runs"] = Integer(h["runs"] || h[:runs] || 1)
+        out["failures"] = Integer(h["failures"] || h[:failures] || 0)
+        out["timeouts"] = Integer(h["timeouts"] || h[:timeouts] || 0)
+        out
+      end
+      # rubocop:enable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+
+      def binpack_weight(entry)
+        h = normalize_entry(entry)
+        h["last_seconds"].positive? ? h["last_seconds"] : h["mean"]
+      end
+
+      # rubocop:disable Metrics/AbcSize -- weighted mean merge
+      def merge_entries(a, b)
+        ha = normalize_entry(a)
+        hb = normalize_entry(b)
+        runs = ha["runs"] + hb["runs"]
+        mean =
+          if runs.positive?
+            ((ha["mean"] * ha["runs"]) + (hb["mean"] * hb["runs"])) / runs.to_f
+          else
+            0.0
+          end
+        {
+          "last_seconds" => [ha["last_seconds"], hb["last_seconds"]].max,
+          "min" => [ha["min"], hb["min"]].min,
+          "max" => [ha["max"], hb["max"]].max,
+          "mean" => mean,
+          "p95" => [ha["p95"], hb["p95"]].max,
+          "runs" => runs,
+          "failures" => ha["failures"] + hb["failures"],
+          "timeouts" => ha["timeouts"] + hb["timeouts"]
+        }
+      end
+      # rubocop:enable Metrics/AbcSize
+    end
+  end
+end

data/lib/polyrun/timing/summary.rb

@@ -1,14 +1,17 @@
+require_relative "stats"
+
 module Polyrun
   module Timing
     # Human-readable slow-file list from merged timing JSON (per-file cost).
     module Summary
       module_function
 
-      # +merged+ is path (String) => seconds (Float), as produced by +Timing::Merge.merge_files+.
+      # +merged+ is path (String) => seconds (Float) or stats Hash, as produced by +Timing::Merge.merge_files+.
       def format_slow_files(merged, top: 30, title: "Polyrun slowest files (by wall time, seconds)")
         return "#{title}\n  (no data)\n" if merged.nil? || merged.empty?
 
-        pairs = merged.sort_by { |_, sec| -sec.to_f }.first(Integer(top))
+        pairs = merged.map { |path, sec| [path, Stats.binpack_weight(sec)] }
+          .sort_by { |(_, sec)| -sec.to_f }.first(Integer(top))
         lines = [title, ""]
         pairs.each_with_index do |(path, sec), i|
           lines << format("  %2d. %s  %.4f", i + 1, path, sec.to_f)

data/lib/polyrun/timing/variance_report.rb

@@ -0,0 +1,51 @@
+module Polyrun
+  module Timing
+    # Flags high-variance, flaky, and regression timing entries.
+    module VarianceReport
+      module_function
+
+      # rubocop:disable Metrics/AbcSize -- variance flag scan
+      def analyze(merged_stats)
+        flags = []
+        merged_stats.each do |path, entry|
+          h = Stats.normalize_entry(entry)
+          next if h["runs"] < 2
+
+          median = h["mean"]
+          if median.positive? && (h["p95"] / median) > 2.0
+            flags << {path: path, kind: "high_variance", detail: "p95/mean=#{format("%.2f", h["p95"] / median)}"}
+          end
+
+          if h["runs"] >= 3 && (h["failures"].to_f / h["runs"]) > 0.3
+            flags << {path: path, kind: "often_failed", detail: "failures=#{h["failures"]}/#{h["runs"]}"}
+          end
+
+          if h["timeouts"].to_i >= 2
+            flags << {path: path, kind: "timeout_cluster", detail: "timeouts=#{h["timeouts"]}"}
+          end
+
+          if h["mean"].positive? && h["last_seconds"] > (2.0 * h["mean"])
+            flags << {path: path, kind: "runtime_regression", detail: "last=#{h["last_seconds"]} mean=#{h["mean"]}"}
+          end
+        end
+        flags
+      end
+      # rubocop:enable Metrics/AbcSize
+
+      def emit_warnings!(merged_stats)
+        analyze(merged_stats).each do |f|
+          Polyrun::Log.warn "polyrun timing #{f[:kind]}: #{f[:path]} (#{f[:detail]})"
+        end
+      end
+
+      def format_report(merged_stats)
+        lines = ["Polyrun timing variance report", ""]
+        analyze(merged_stats).each do |f|
+          lines << "  [#{f[:kind]}] #{f[:path]} — #{f[:detail]}"
+        end
+        lines << "  (none)" if lines.size == 2
+        lines.join("\n") + "\n"
+      end
+    end
+  end
+end

data/lib/polyrun/version.rb

@@ -1,3 +1,3 @@
 module Polyrun
-  VERSION = "1.5.0"
+  VERSION = "2.1.2"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: polyrun
 version: !ruby/object:Gem::Version
-  version: 1.5.0
+  version: 2.1.2
 platform: ruby
 authors:
 - Andrei Makarov
@@ -180,12 +180,14 @@ files:
 - lib/polyrun/cli/helpers.rb
 - lib/polyrun/cli/hooks_command.rb
 - lib/polyrun/cli/init_command.rb
+- lib/polyrun/cli/partition_diagnostics.rb
 - lib/polyrun/cli/plan_command.rb
 - lib/polyrun/cli/prepare_command.rb
 - lib/polyrun/cli/prepare_recipe.rb
 - lib/polyrun/cli/queue_command.rb
 - lib/polyrun/cli/quick_command.rb
 - lib/polyrun/cli/report_commands.rb
+- lib/polyrun/cli/run_queue_command.rb
 - lib/polyrun/cli/run_shards_command.rb
 - lib/polyrun/cli/run_shards_parallel_children.rb
 - lib/polyrun/cli/run_shards_parallel_wait.rb
@@ -194,6 +196,7 @@ files:
 - lib/polyrun/cli/run_shards_planning.rb
 - lib/polyrun/cli/run_shards_run.rb
 - lib/polyrun/cli/run_shards_worker_interrupt.rb
+- lib/polyrun/cli/spec_quality_commands.rb
 - lib/polyrun/cli/start_bootstrap.rb
 - lib/polyrun/cli/timing_command.rb
 - lib/polyrun/config.rb
@@ -204,6 +207,7 @@ files:
 - lib/polyrun/coverage/collector.rb
 - lib/polyrun/coverage/collector_finish.rb
 - lib/polyrun/coverage/collector_fragment_meta.rb
+- lib/polyrun/coverage/example_diff.rb
 - lib/polyrun/coverage/filter.rb
 - lib/polyrun/coverage/formatter.rb
 - lib/polyrun/coverage/merge.rb
@@ -252,11 +256,14 @@ files:
 - lib/polyrun/partition/plan.rb
 - lib/polyrun/partition/plan_lpt.rb
 - lib/polyrun/partition/plan_sharding.rb
+- lib/polyrun/partition/reports.rb
 - lib/polyrun/partition/stable_shuffle.rb
+- lib/polyrun/partition/timing_diagnostics.rb
 - lib/polyrun/partition/timing_keys.rb
 - lib/polyrun/prepare/artifacts.rb
 - lib/polyrun/prepare/assets.rb
 - lib/polyrun/process_stdio.rb
+- lib/polyrun/queue/duration.rb
 - lib/polyrun/queue/file_store.rb
 - lib/polyrun/queue/file_store_pending.rb
 - lib/polyrun/quick.rb
@@ -274,13 +281,27 @@ files:
 - lib/polyrun/reporting/rspec_failure_fragment_formatter.rb
 - lib/polyrun/reporting/rspec_junit.rb
 - lib/polyrun/rspec.rb
+- lib/polyrun/spec_quality.rb
+- lib/polyrun/spec_quality/config.rb
+- lib/polyrun/spec_quality/fragment.rb
+- lib/polyrun/spec_quality/merge.rb
+- lib/polyrun/spec_quality/minitest_hook.rb
+- lib/polyrun/spec_quality/plan_loader.rb
+- lib/polyrun/spec_quality/profile.rb
+- lib/polyrun/spec_quality/report.rb
+- lib/polyrun/spec_quality/rspec_hook.rb
+- lib/polyrun/spec_quality/sql_counter.rb
 - lib/polyrun/templates/POLYRUN.md
 - lib/polyrun/templates/ci_matrix.polyrun.yml
 - lib/polyrun/templates/minimal_gem.polyrun.yml
+- lib/polyrun/templates/polyrun_hooks_spec_quality.rb
+- lib/polyrun/templates/polyrun_spec_quality.yml
 - lib/polyrun/templates/rails_prepare.polyrun.yml
 - lib/polyrun/timing/merge.rb
 - lib/polyrun/timing/rspec_example_formatter.rb
+- lib/polyrun/timing/stats.rb
 - lib/polyrun/timing/summary.rb
+- lib/polyrun/timing/variance_report.rb
 - lib/polyrun/version.rb
 - lib/polyrun/worker_ping.rb
 - polyrun.gemspec