polyrun 1.5.0 → 2.1.0

data/lib/polyrun/spec_quality/report.rb

@@ -0,0 +1,261 @@
+# rubocop:disable Polyrun/FileLength, Metrics/ModuleLength -- report analysis + text formatting
+module Polyrun
+  module SpecQuality
+    # Human-readable spec quality report from merged JSON.
+    module Report
+      module_function
+
+      # rubocop:disable Metrics/AbcSize -- merged example analysis
+      def analyze(merged, cfg = {}, plan_shards: nil)
+        cfg = default_cfg(cfg)
+        examples = merged["examples"] || {}
+        hot_lines = merged["hot_lines"] || {}
+        shard_summary = merged["shard_summary"] || Merge.shard_summary(examples)
+
+        zero_hit = examples.select { |_loc, row| row["unique_lines"].to_i.zero? }
+        churn = examples.select { |_loc, row| row["line_churn"].to_i >= cfg["min_line_churn"] }
+          .sort_by { |_loc, row| -row["line_churn"].to_i }
+        hot = hot_lines.select { |_line, h| h["example_count"].to_i >= cfg["hot_line_example_overlap"] }
+          .sort_by { |_line, h| [-h["example_count"].to_i, -h["total_hits"].to_i] }
+
+        outliers = build_outliers(examples, cfg)
+        partition_hints = partition_hints_for(hot, examples, plan_shards) if plan_shards && !plan_shards.empty?
+
+        {
+          zero_hit: zero_hit,
+          line_churn: churn,
+          hot_lines: hot,
+          outliers: outliers,
+          shard_summary: shard_summary,
+          partition_hints: partition_hints,
+          config: cfg
+        }
+      end
+      # rubocop:enable Metrics/AbcSize
+
+      def format_report(merged, cfg: {}, top: 30, profile: nil, plan_shards: nil)
+        analysis = analyze(merged, cfg, plan_shards: plan_shards)
+        lines = ["Polyrun spec quality report", ""]
+
+        lines.concat(format_shard_summary_section(analysis[:shard_summary]))
+        lines << ""
+        lines.concat(format_zero_hit_section(analysis[:zero_hit], top))
+        lines << ""
+        lines.concat(format_hot_lines_section(analysis[:hot_lines], top))
+        lines << ""
+        lines.concat(format_churn_section(analysis[:line_churn], top))
+        hints_section = format_partition_hints_section(analysis[:partition_hints], top)
+        unless hints_section.empty?
+          lines << ""
+          lines.concat(hints_section)
+        end
+        lines << ""
+        lines.concat(format_outliers_section(analysis[:outliers], top, profile))
+
+        lines.join("\n") + "\n"
+      end
+
+      def gate_violations(merged, cfg = {})
+        cfg = default_cfg(cfg)
+        analysis = analyze(merged, cfg)
+        violations = []
+
+        max_zero = cfg["max_zero_hit_examples"]
+        if max_zero && analysis[:zero_hit].size > max_zero.to_i
+          violations << "zero_hit_examples=#{analysis[:zero_hit].size} max=#{max_zero}"
+        end
+
+        min_unique = cfg["minimum_unique_lines_per_example"]
+        if min_unique
+          bad = analysis[:zero_hit].keys
+          if bad.size.positive?
+            violations << "examples_below_minimum_unique_lines=#{bad.size} minimum=#{min_unique}"
+          end
+        end
+
+        max_hot = cfg["max_hot_line_overlap"]
+        if max_hot
+          over = analysis[:hot_lines].count { |_k, h| h["example_count"].to_i > max_hot.to_i }
+          violations << "hot_line_overlap_count=#{over} max=#{max_hot}" if over.positive?
+        end
+
+        violations
+      end
+
+      def emit_warnings!(merged, cfg = {})
+        analyze(merged, cfg).fetch(:line_churn, {}).each do |loc, row|
+          Polyrun::Log.warn "polyrun spec-quality line_churn: #{loc} churn=#{row["line_churn"]}"
+        end
+      end
+
+      def default_cfg(cfg)
+        h = cfg.is_a?(Hash) ? cfg.transform_keys(&:to_s) : {}
+        SpecQuality::Config::DEFAULTS.merge(h)
+      end
+
+      def format_shard_summary_section(shard_summary)
+        lines = ["Shard attribution:"]
+        if shard_summary.nil? || shard_summary.empty?
+          lines << "  (none)"
+          return lines
+        end
+
+        shard_summary.sort_by { |k, _| k.to_s }.each do |shard, stats|
+          lines << format(
+            "  shard %s — examples=%d zero_hit=%d line_churn=%d",
+            shard,
+            stats["examples"],
+            stats["zero_hit"],
+            stats["line_churn"]
+          )
+        end
+        lines
+      end
+
+      def format_partition_hints_section(hints, top)
+        return [] if hints.nil? || hints.empty?
+
+        lines = ["Partition hints (hot lines × shard):"]
+        hints.first(top).each do |h|
+          lines << format("  %s — shard %s (%d examples)", h[:line], h[:shard], h[:example_count])
+        end
+        lines << "  …" if hints.size > top
+        lines
+      end
+
+      def partition_hints_for(hot_lines, examples, plan_shards)
+        hot_lines.filter_map do |line, h|
+          example_locs = h["examples"] || []
+          shard_counts = Hash.new(0)
+          example_locs.each do |loc|
+            s = PlanLoader.shard_for_example(loc, plan_shards) || examples.dig(loc, "polyrun_shard_index")&.to_s
+            shard_counts[s] += 1 if s
+          end
+          next if shard_counts.empty?
+
+          shard, count = shard_counts.max_by { |_s, n| n }
+          {line: line, shard: shard, example_count: count, total_hits: h["total_hits"]}
+        end.sort_by { |h| [-h[:example_count], -h[:total_hits]] }
+      end
+
+      def format_zero_hit_section(zero_hit, top)
+        lines = ["Zero production lines (#{zero_hit.size} examples):"]
+        if zero_hit.empty?
+          lines << "  (none)"
+          return lines
+        end
+
+        zero_hit.keys.sort.first(top).each { |loc| lines << "  #{loc}" }
+        lines << "  …" if zero_hit.size > top
+        lines
+      end
+
+      def format_hot_lines_section(hot_lines, top)
+        lines = ["Hot lines (shared across examples):"]
+        if hot_lines.empty?
+          lines << "  (none)"
+          return lines
+        end
+
+        hot_lines.first(top).each do |line, h|
+          lines << format("  %s — %d examples, %d cumulative hits", line, h["example_count"], h["total_hits"])
+        end
+        lines << "  …" if hot_lines.size > top
+        lines
+      end
+
+      def format_churn_section(churn_rows, top)
+        lines = ["Per-example line churn (top #{[top, churn_rows.size].min}):"]
+        if churn_rows.empty?
+          lines << "  (none)"
+          return lines
+        end
+
+        churn_rows.first(top).each do |loc, row|
+          lines << format("  %s — churn=%d max_line=%d", loc, row["line_churn"], row["max_line_churn"])
+        end
+        lines
+      end
+
+      # rubocop:disable Metrics/AbcSize -- outlier row filter
+      def build_outliers(examples, cfg)
+        examples.filter_map do |loc, row|
+          prof = row["profile"] || {}
+          unique = row["unique_lines"].to_i
+          wall = prof["wall"].to_f
+          alloc = prof["gc_allocated"].to_i
+          cpu = prof["cpu_user"].to_f + prof["cpu_system"].to_f
+          sql = row["sql_count"].to_i
+          factories = (row["factory_counts"] || {}).values.sum
+
+          score = 0
+          reasons = []
+          if unique.zero?
+            score += 10
+            reasons << "zero_lines"
+          end
+          if wall > 1.0 && unique < 3
+            score += 5
+            reasons << "slow_low_coverage"
+          end
+          if alloc > 50_000 && wall > 0.5
+            score += 3
+            reasons << "high_alloc"
+          end
+          if cpu > 0.5 && unique < 3
+            score += 3
+            reasons << "high_cpu_low_coverage"
+          end
+          if sql >= cfg["min_query_count"]
+            score += 4
+            reasons << "high_sql_count"
+          end
+          if factories >= 10
+            score += 2
+            reasons << "many_factories"
+          end
+          next if score.zero?
+
+          {location: loc, score: score, reasons: reasons, row: row}
+        end.sort_by { |h| -h[:score] }
+      end
+      # rubocop:enable Metrics/AbcSize
+
+      # rubocop:disable Metrics/AbcSize -- outlier text formatting
+      def format_outliers_section(outliers, top, profile)
+        lines = ["Correlated outliers (slow / empty / heavy):"]
+        if outliers.empty?
+          lines << "  (none)"
+          return lines
+        end
+
+        dims = profile ? profile.to_s.split(",").map(&:strip) : nil
+        outliers.first(top).each do |o|
+          row = o[:row]
+          prof = row["profile"] || {}
+          detail = o[:reasons].join(", ")
+          if dims.nil? || dims.empty?
+            lines << format("  %s — score=%d (%s)", o[:location], o[:score], detail)
+          else
+            prof_bits = dims.filter_map do |d|
+              case d
+              when "wall" then "wall=#{format("%.2f", prof["wall"])}" if prof["wall"]
+              when "cpu" then "cpu=#{format("%.2f", prof["cpu_user"].to_f + prof["cpu_system"].to_f)}"
+              when "mem" then "alloc=#{prof["gc_allocated"]}" if prof["gc_allocated"]
+              when "io"
+                r = prof["io_read_bytes"]
+                w = prof["io_write_bytes"]
+                "io=#{r}/#{w}" if r || w
+              end
+            end
+            lines << format("  %s — score=%d (%s) %s", o[:location], o[:score], detail, prof_bits.join(" "))
+          end
+        end
+        lines << "  …" if outliers.size > top
+        lines
+      end
+      # rubocop:enable Metrics/AbcSize
+    end
+  end
+end
+# rubocop:enable Polyrun/FileLength, Metrics/ModuleLength

data/lib/polyrun/spec_quality/rspec_hook.rb

@@ -0,0 +1,55 @@
+module Polyrun
+  module SpecQuality
+    # RSpec hooks for per-example spec quality recording.
+    module RspecHook
+      module_function
+
+      def install!(only_if: nil, root: nil, output_path: nil)
+        pred = only_if || -> { Polyrun::SpecQuality.enabled? }
+        return unless pred.call
+
+        require "rspec/core"
+        ensure_started!(root: root, output_path: output_path)
+
+        ::RSpec.configure do |config|
+          config.before(:each) do |example|
+            next if example.pending?
+
+            Polyrun::SpecQuality.start_example!(
+              location: example.metadata[:location] || example.location
+            )
+          end
+
+          config.after(:each) do |example|
+            Polyrun::SpecQuality.finish_example!(
+              location: example.metadata[:location] || example.location,
+              pending: example.pending?
+            )
+          end
+        end
+      end
+
+      def ensure_started!(root: nil, output_path: nil)
+        return if Polyrun::SpecQuality.started?
+
+        Polyrun::SpecQuality.start!(
+          root: root || infer_root,
+          output_path: output_path
+        )
+      end
+
+      def infer_root
+        if defined?(::Rails) && ::Rails.respond_to?(:root) && ::Rails.root
+          return ::Rails.root.to_s
+        end
+
+        caller_locations.each do |loc|
+          inferred = Polyrun::Coverage::Rails.infer_root_from_path(loc.path)
+          return inferred if inferred
+        end
+
+        Dir.pwd
+      end
+    end
+  end
+end

data/lib/polyrun/spec_quality/sql_counter.rb

@@ -0,0 +1,34 @@
+module Polyrun
+  module SpecQuality
+    # Optional per-example SQL query counting via ActiveSupport::Notifications.
+    module SqlCounter
+      class << self
+        def install!
+          return false unless notifications_available?
+          return true if @installed
+
+          @installed = true
+          @subscriber = ActiveSupport::Notifications.subscribe("sql.active_record") do |*args|
+            next unless Polyrun::SpecQuality.recording?
+
+            event = ActiveSupport::Notifications::Event.new(*args)
+            Polyrun::SpecQuality.record_sql!(event.payload[:sql].to_s)
+          end
+          true
+        end
+
+        def uninstall!
+          return unless @installed && @subscriber
+
+          ActiveSupport::Notifications.unsubscribe(@subscriber)
+          @subscriber = nil
+          @installed = false
+        end
+
+        def notifications_available?
+          defined?(ActiveSupport::Notifications)
+        end
+      end
+    end
+  end
+end

data/lib/polyrun/spec_quality.rb

@@ -0,0 +1,205 @@
+# rubocop:disable Polyrun/FileLength -- per-example recorder API
+require "json"
+require "fileutils"
+
+require_relative "coverage/example_diff"
+require_relative "partition/timing_keys"
+require_relative "spec_quality/config"
+require_relative "spec_quality/profile"
+require_relative "spec_quality/fragment"
+require_relative "spec_quality/sql_counter"
+require_relative "spec_quality/merge"
+require_relative "spec_quality/plan_loader"
+require_relative "spec_quality/report"
+require_relative "spec_quality/rspec_hook"
+require_relative "spec_quality/minitest_hook"
+
+module Polyrun
+  # Per-example spec quality: coverage line deltas, resource use, optional SQL counts.
+  # Opt-in with +POLYRUN_SPEC_QUALITY=1+ or +Polyrun::SpecQuality.start!+.
+  module SpecQuality
+    class << self
+      def start!(root: Dir.pwd, config_path: nil, output_path: nil, **overrides)
+        return if Config.disabled?
+
+        @config = Config.load(root: root, config_path: config_path, **overrides)
+        @output_path = output_path || Fragment.default_fragment_path
+        @pause_depth = 0
+        @current = nil
+        @rng = Random.new(Process.pid ^ Time.now.to_i)
+
+        Fragment.truncate_fragment!(@output_path) unless fragment_append_mode?
+        SqlCounter.install! if @config["sql_counter"]
+        self
+      end
+
+      def started?
+        instance_variable_defined?(:@config) && @config
+      end
+
+      def enabled?
+        Config.enabled? && !Config.disabled?
+      end
+
+      attr_reader :config
+
+      attr_reader :output_path
+
+      def recording?
+        started? && @current && @pause_depth.zero?
+      end
+
+      def start_example!(location:, wall_start: nil)
+        return unless started?
+        return if paused?
+        return if Config.ignored_example?(location, @config["ignore_examples"])
+        return unless sample_example?
+
+        @current = {
+          location: normalize_location(location),
+          wall_start: wall_start || Process.clock_gettime(Process::CLOCK_MONOTONIC),
+          coverage_before: Coverage::ExampleDiff.peek_blob,
+          profile_before: Profile.snapshot,
+          sql_count: 0,
+          sql_fingerprints: Hash.new(0),
+          factory_counts: {}
+        }
+        Polyrun::Data::FactoryCounts.reset_example! if defined?(Polyrun::Data::FactoryCounts)
+        nil
+      end
+
+      def finish_example!(location: nil, pending: false)
+        return unless started?
+        cur = @current
+        @current = nil
+        return if cur.nil? || paused?
+        return if pending
+
+        loc = location || cur[:location]
+        return if loc.nil? || loc.to_s.empty?
+
+        after_cov = Coverage::ExampleDiff.peek_blob
+        delta = Coverage::ExampleDiff.diff(cur[:coverage_before], after_cov)
+        delta = Coverage::ExampleDiff.apply_track_under(
+          delta,
+          root: @config["root"],
+          track_under: @config["track_under"],
+          ignore_paths: @config["ignore_paths"]
+        )
+
+        profile_after = Profile.snapshot
+        profile_delta = Profile.diff(cur[:profile_before], profile_after)
+        wall = Process.clock_gettime(Process::CLOCK_MONOTONIC) - cur[:wall_start]
+        profile_delta["wall"] = wall
+
+        factory_counts =
+          if defined?(Polyrun::Data::FactoryCounts)
+            Polyrun::Data::FactoryCounts.example_counts
+          else
+            {}
+          end
+
+        row = build_row(cur, loc, delta, profile_delta, factory_counts)
+        Fragment.append_row!(@output_path, row)
+        row
+      end
+
+      def pause
+        @pause_depth += 1
+        if block_given?
+          begin
+            yield
+          ensure
+            resume
+          end
+        end
+      end
+
+      def resume
+        @pause_depth -= 1 if @pause_depth.positive?
+      end
+
+      def paused?
+        @pause_depth.positive?
+      end
+
+      def record_sql!(sql)
+        return unless recording?
+
+        @current[:sql_count] += 1
+        fp = normalize_sql(sql)
+        @current[:sql_fingerprints][fp] += 1
+      end
+
+      def spec_quality_requested_for_quick?(root = Dir.pwd)
+        return false if Config.disabled?
+        return true if %w[1 true yes].include?(ENV["POLYRUN_SPEC_QUALITY"]&.to_s&.downcase)
+        return true if %w[1 true yes].include?(ENV["POLYRUN_SPEC_QUALITY_FRAGMENTS"]&.to_s&.downcase)
+
+        path = File.join(File.expand_path(root), Config::DEFAULT_CONFIG_RELATIVE)
+        return false unless File.file?(path)
+
+        %w[1 true yes].include?(ENV["POLYRUN_QUICK_SPEC_QUALITY"]&.to_s&.downcase)
+      end
+
+      private
+
+      def fragment_append_mode?
+        truthy?(ENV["POLYRUN_SPEC_QUALITY_FRAGMENT_APPEND"])
+      end
+
+      def sample_example?
+        rate = @config["sample"].to_f
+        return true if rate >= 1.0
+        return false if rate <= 0.0
+
+        @rng.rand < rate
+      end
+
+      def normalize_location(location)
+        s = location.to_s.strip
+        return s if s.empty?
+
+        root = @config["root"]
+        if (m = s.match(/\A(.+):(\d+)\z/)) && m[2].match?(/\A\d+\z/)
+          fp = Polyrun::Partition::TimingKeys.canonical_file_path(File.expand_path(m[1], root))
+          return "#{fp}:#{m[2]}"
+        end
+
+        Polyrun::Partition::TimingKeys.canonical_file_path(File.expand_path(s, root))
+      end
+
+      def build_row(cur, location, delta, profile_delta, factory_counts)
+        profile = Profile.slice_profile(profile_delta, @config["profile"])
+        repeated_sql = cur[:sql_fingerprints].select { |_sql, n| n >= min_query_count }.transform_keys(&:to_s)
+
+        {
+          "example" => location.to_s,
+          "unique_lines" => delta[:unique_lines],
+          "line_churn" => delta[:line_churn],
+          "max_line_churn" => delta[:max_line_churn],
+          "lines" => delta[:lines],
+          "profile" => profile,
+          "sql_count" => cur[:sql_count],
+          "repeated_sql" => repeated_sql,
+          "factory_counts" => factory_counts.transform_keys(&:to_s),
+          "polyrun_shard_index" => ENV["POLYRUN_SHARD_INDEX"],
+          "polyrun_shard_total" => ENV["POLYRUN_SHARD_TOTAL"]
+        }.compact
+      end
+
+      def min_query_count
+        @config["min_query_count"].to_i
+      end
+
+      def normalize_sql(sql)
+        sql.to_s.gsub(/\s+/, " ").strip[0, 500]
+      end
+
+      def truthy?(value)
+        Config.send(:truthy?, value)
+      end
+    end
+  end
+end
+# rubocop:enable Polyrun/FileLength

data/lib/polyrun/templates/POLYRUN.md

@@ -40,6 +40,12 @@ Do not combine Model A and Model B in one workflow without a documented reason (
 - `spec/spec_helper.rb`: `require "polyrun"` and collector or Rails helper as appropriate.
 - Fragments: `coverage/polyrun-fragment-<shard>.json` → `polyrun merge-coverage` → `polyrun report-coverage` / `report-junit` for CI.
 
+## Spec quality (optional)
+
+- `POLYRUN_SPEC_QUALITY=1` and `Polyrun::RSpec.install_spec_quality!` in `spec_helper` (requires coverage enabled for line deltas).
+- `config/polyrun_spec_quality.yml` — thresholds and CI gates (`polyrun init --profile spec-quality`).
+- Parallel: `run-shards --merge-spec-quality` → `coverage/polyrun-spec-quality.json` → `report-spec-quality`.
+
 ## Further reading
 
 - Polyrun README: partition, prepare, databases, merge-coverage, `Polyrun::Env::Ci`

data/lib/polyrun/templates/ci_matrix.polyrun.yml

@@ -16,3 +16,7 @@ partition:
   paths_build:
     all_glob: spec/**/*_spec.rb
     stages: []
+
+# Optional reporting (see docs/SPEC_QUALITY.md):
+# reporting:
+#   merge_spec_quality: true

data/lib/polyrun/templates/polyrun_hooks_spec_quality.rb

@@ -0,0 +1,12 @@
+# Optional Polyrun hooks — copy to config/polyrun_hooks.rb and reference from polyrun.yml:
+#   hooks:
+#     ruby_file: config/polyrun_hooks.rb
+#
+# Worker-phase hooks run in each parallel test child (around bundle exec rspec).
+
+before(:each) do |env|
+  next unless env["POLYRUN_HOOK_ORCHESTRATOR"] == "0"
+  next unless defined?(Polyrun::SpecQuality) && Polyrun::SpecQuality.enabled?
+
+  Polyrun::SpecQuality::RspecHook.ensure_started!
+end

data/lib/polyrun/templates/polyrun_spec_quality.yml

@@ -0,0 +1,20 @@
+# Opt-in per-example spec quality (coverage deltas, resource use). See docs/SPEC_QUALITY.md.
+track_under:
+  - lib
+  - app
+min_line_churn: 50
+min_query_count: 20
+hot_line_example_overlap: 10
+strict: false
+sample: 1.0
+ignore_examples: []
+ignore_paths: []
+ignore_query_patterns: []
+profile:
+  - cpu
+  - mem
+sql_counter: false
+# CI gates (used with POLYRUN_SPEC_QUALITY_STRICT=1 or strict: true):
+# minimum_unique_lines_per_example: 1
+# max_zero_hit_examples: 0
+# max_hot_line_overlap: 100

data/lib/polyrun/templates/rails_prepare.polyrun.yml

@@ -17,6 +17,11 @@ prepare:
   rails_root: .
   command: bundle exec ruby bin/test_prepare.rb
 
+# Optional reporting (see docs/SPEC_QUALITY.md):
+# reporting:
+#   merge_spec_quality: true
+#   merge_spec_quality_output: coverage/polyrun-spec-quality.json
+
 # Uncomment and edit when using Postgres template + per-shard DB names from polyrun env:
 # databases:
 #   template_db: my_app_template

data/lib/polyrun/timing/merge.rb

@@ -1,12 +1,11 @@
 require "json"
 
 require_relative "../debug"
+require_relative "stats"
+require_relative "variance_report"
 
 module Polyrun
   module Timing
-    # Merges per-shard timing JSON files (spec2 §2.4): path => wall seconds (float), or (experimental)
-    # +absolute_path:line+ => seconds for per-example timing.
-    # Disjoint suites: values merged by taking the maximum per path when duplicates appear.
     module Merge
       module_function
 
@@ -18,8 +17,8 @@ module Polyrun
 
           data.each do |file, sec|
             f = file.to_s
-            t = sec.to_f
-            merged[f] = merged.key?(f) ? [merged[f], t].max : t
+            entry = sec
+            merged[f] = merged.key?(f) ? Stats.merge_entries(merged[f], entry) : Stats.normalize_entry(entry)
           end
         end
         merged
@@ -29,6 +28,7 @@ module Polyrun
         Polyrun::Debug.log_kv(merge_timing: "merge_and_write", input_count: paths.size, output_path: output_path)
         merged = Polyrun::Debug.time("Timing::Merge.merge_files") { merge_files(paths) }
         Polyrun::Debug.time("Timing::Merge.write JSON") { File.write(output_path, JSON.pretty_generate(merged)) }
+        Timing::VarianceReport.emit_warnings!(merged)
         merged
       end
     end