polyrun 2.1.2 → 2.2.0

data/lib/polyrun/coverage/formatter.rb

@@ -3,6 +3,7 @@ require "json"
 
 require_relative "merge"
 require_relative "result"
+require_relative "file_stats_report"
 
 module Polyrun
   module Coverage
@@ -26,8 +27,11 @@ module Polyrun
         when :cobertura then CoberturaFormatter
         when :console then ConsoleFormatter
         when :html then HtmlFormatter
+        when :csv then CsvFormatter
+        when :markdown then MarkdownFormatter
         else
-          raise ArgumentError, "unknown coverage format: #{name.inspect} (expected :json, :lcov, :cobertura, :console, :html)"
+          raise ArgumentError,
+            "unknown coverage format: #{name.inspect} (expected :json, :lcov, :cobertura, :console, :html, :csv, :markdown)"
         end
       end
 
@@ -111,6 +115,23 @@ module Polyrun
           {html: path}
         end
       end
+
+      class CsvFormatter < Base
+        def write_files(result, output_dir, basename)
+          path = File.join(output_dir, "#{basename}.csv")
+          File.write(path, FileStatsReport.emit_csv(result.coverage_blob))
+          {csv: path}
+        end
+      end
+
+      class MarkdownFormatter < Base
+        def write_files(result, output_dir, basename)
+          path = File.join(output_dir, "#{basename}.md")
+          title = (result.meta && result.meta["title"]) || (result.meta && result.meta[:title]) || "Polyrun coverage report"
+          File.write(path, FileStatsReport.emit_markdown(result.coverage_blob, title: title))
+          {markdown: path}
+        end
+      end
     end
   end
 end

data/lib/polyrun/coverage/merge/formatters.rb

@@ -153,16 +153,24 @@ module Polyrun
       # Per-file line stats for HTML and other formatters.
       # Integer line counts for one file entry (for O(files x groups) group aggregation).
       def line_counts(file_entry)
+        if native_acceleration?
+          MergeNative.line_counts(file_entry)
+        else
+          line_counts_ruby(file_entry)
+        end
+      end
+
+      def line_counts_ruby(file_entry)
         line_arr = line_array_from_file_entry(file_entry)
         return {relevant: 0, covered: 0} unless line_arr.is_a?(Array)
 
         relevant = 0
         covered = 0
-        line_arr.each do |h|
-          next if h.nil? || h == "ignored"
+        line_arr.each do |hit|
+          next if hit.nil? || hit == "ignored"
 
           relevant += 1
-          covered += 1 if h.to_i > 0
+          covered += 1 if hit.to_i > 0
         end
         {relevant: relevant, covered: covered}
       end

data/lib/polyrun/coverage/merge_merge_two.rb

@@ -4,114 +4,163 @@ module Polyrun
       module_function
 
       def merge_two(a, b)
-        keys = a.keys | b.keys
+        if native_acceleration?
+          MergeNative.merge_two(a, b)
+        else
+          merge_two_ruby(a, b)
+        end
+      end
+
+      def merge_two_ruby(a, b)
+        a = {} if a.nil?
+        b = {} if b.nil?
+        if a.size >= b.size
+          merge_two_by_keys(a, b)
+        else
+          merge_two_by_keys(b, a)
+        end
+      end
+
+      def merge_two_by_keys(primary, secondary)
         out = {}
-        keys.each do |path|
-          out[path] = merge_file_entry(a[path], b[path])
+        primary.each do |path, entry|
+          out[path] = merge_file_entry(entry, secondary[path])
+        end
+        secondary.each do |path, entry|
+          out[path] = entry unless out.key?(path)
         end
         out
       end
 
-      def normalize_file_entry(v)
-        return nil if v.nil?
-        return {"lines" => v} if v.is_a?(Array)
+      def native_merge_line_arrays?
+        native_acceleration?
+      end
 
-        v
+      def native_acceleration?
+        MergeNative.available?
+      end
+
+      def normalize_file_entry(value)
+        return nil if value.nil?
+        return {"lines" => value} if value.is_a?(Array)
+
+        value
       end
 
       def line_array_from_file_entry(file)
-        h = normalize_file_entry(file)
-        return nil unless h.is_a?(Hash)
+        hash = normalize_file_entry(file)
+        return nil unless hash.is_a?(Hash)
 
-        h["lines"] || h[:lines]
+        hash["lines"] || hash[:lines]
       end
 
-      def merge_file_entry(x, y)
-        x = normalize_file_entry(x)
-        y = normalize_file_entry(y)
-        return y if x.nil?
-        return x if y.nil?
+      def merge_file_entry(left, right)
+        left = normalize_file_entry(left)
+        right = normalize_file_entry(right)
+        return right if left.nil?
+        return left if right.nil?
 
-        lines = merge_line_arrays(x["lines"] || x[:lines], y["lines"] || y[:lines])
+        lines = merge_line_arrays(left["lines"] || left[:lines], right["lines"] || right[:lines])
         entry = {"lines" => lines}
-        bx = x["branches"] || x[:branches]
-        by = y["branches"] || y[:branches]
-        br = merge_branch_arrays(bx, by)
-        entry["branches"] = br if br
+        left_branches = left["branches"] || left[:branches]
+        right_branches = right["branches"] || right[:branches]
+        branches = merge_branch_arrays(left_branches, right_branches)
+        entry["branches"] = branches if branches
         entry
       end
 
-      def merge_line_arrays(a, b)
-        a ||= []
-        b ||= []
-        na = a.size
-        nb = b.size
-        max_len = (na > nb) ? na : nb
+      def merge_line_arrays(left, right)
+        if native_acceleration?
+          MergeNative.merge_line_arrays(left, right)
+        else
+          merge_line_arrays_ruby(left, right)
+        end
+      end
+
+      def merge_line_arrays_ruby(left, right)
+        left ||= []
+        right ||= []
+        left_size = left.size
+        right_size = right.size
+        max_len = (left_size > right_size) ? left_size : right_size
         out = Array.new(max_len)
-        i = 0
-        while i < max_len
-          out[i] = merge_line_hits(a[i], b[i])
-          i += 1
+        index = 0
+        while index < max_len
+          out[index] = merge_line_hits(left[index], right[index])
+          index += 1
         end
         out
       end
 
-      def merge_line_hits(x, y)
-        return y if x.nil?
-        return x if y.nil?
-        return "ignored" if x == "ignored" || y == "ignored"
+      def merge_line_hits(left, right)
+        return right if left.nil?
+        return left if right.nil?
+        return "ignored" if left == "ignored" || right == "ignored"
 
-        xi = line_hit_to_i(x)
-        yi = line_hit_to_i(y)
-        return xi + yi if xi && yi
+        left_i = line_hit_to_i(left)
+        right_i = line_hit_to_i(right)
+        return left_i + right_i if left_i && right_i
 
-        return yi if xi.nil? && yi
-        return xi if yi.nil? && xi
+        return right_i if left_i.nil? && right_i
+        return left_i if right_i.nil? && left_i
 
-        x
+        left
       end
 
-      def line_hit_to_i(v)
-        case v
-        when Integer then v
+      def line_hit_to_i(value)
+        case value
+        when Integer then value
         when nil then nil
         else
-          Integer(v, exception: false)
+          Integer(value, exception: false)
         end
       end
 
-      def merge_branch_arrays(a, b)
-        return nil if a.nil? && b.nil?
-        return (a || b).dup if a.nil? || b.nil?
+      def merge_branch_arrays(left, right)
+        return nil if left.nil? && right.nil?
+        return (left || right).dup if left.nil? || right.nil?
 
         index = {}
-        [a, b].each do |arr|
-          arr.each do |br|
-            k = branch_key(br)
-            existing = index[k]
-            index[k] =
+        [left, right].each do |array|
+          array.each do |branch|
+            next unless branch.is_a?(Hash)
+
+            key = branch_key(branch)
+            existing = index[key]
+            index[key] =
               if existing
-                merge_branch_entries(existing, br)
+                merge_branch_entries(existing, branch)
               else
-                br.dup
+                branch.dup
               end
           end
         end
-        index.values.sort_by { |br| branch_key(br) }
+        index.values.sort_by { |branch| branch_key(branch) }
       end
 
-      def branch_key(br)
-        h = br.is_a?(Hash) ? br : {}
-        [h["type"] || h[:type], h["start_line"] || h[:start_line], h["end_line"] || h[:end_line]]
+      def branch_key(branch)
+        hash = branch.is_a?(Hash) ? branch : {}
+        [hash["type"] || hash[:type], hash["start_line"] || hash[:start_line], hash["end_line"] || hash[:end_line]]
       end
 
-      def merge_branch_entries(x, y)
-        out = x.is_a?(Hash) ? x.dup : {}
-        xc = (x["coverage"] || x[:coverage]).to_i
-        yc = (y["coverage"] || y[:coverage]).to_i
-        out["coverage"] = xc + yc
+      def merge_branch_arrays_for_native(left, right)
+        merge_branch_arrays(left, right)
+      end
+
+      def sort_branches_for_native(branches)
+        branches.sort_by { |branch| branch_key(branch) }
+      end
+
+      def merge_branch_entries(left, right)
+        out = left.is_a?(Hash) ? left.dup : {}
+        left_count = (left["coverage"] || left[:coverage]).to_i
+        right_count = (right["coverage"] || right[:coverage]).to_i
+        out["coverage"] = left_count + right_count
         out
       end
     end
   end
 end
+
+require_relative "merge_native"
+Polyrun::Coverage::MergeNative.load!

data/lib/polyrun/coverage/merge_native.rb

@@ -0,0 +1,37 @@
+module Polyrun
+  module Coverage
+    module MergeNative
+      module_function
+
+      def load!
+        return true if available?
+
+        extension_dir = File.expand_path("../../../ext/polyrun_coverage_merge", __dir__)
+        if File.directory?(extension_dir)
+          $LOAD_PATH.unshift(extension_dir) unless $LOAD_PATH.include?(extension_dir)
+        end
+
+        require "polyrun_coverage_merge"
+        true
+      rescue LoadError
+        false
+      end
+
+      def available?
+        ::Object.const_defined?(:PolyrunCoverageMerge)
+      end
+
+      def merge_line_arrays(left, right)
+        ::PolyrunCoverageMerge.merge_line_arrays(left, right)
+      end
+
+      def merge_two(left, right)
+        ::PolyrunCoverageMerge.merge_two(left, right)
+      end
+
+      def line_counts(file_entry)
+        ::PolyrunCoverageMerge.line_counts(file_entry)
+      end
+    end
+  end
+end

data/lib/polyrun/coverage/reporting.rb

@@ -9,7 +9,7 @@ module Polyrun
     # Ready-to-use multi-format output (SimpleCov-compatible result blob), no extra gems.
     # Pass +formatter:+ for multi-formatter composition ({Formatter::MultiFormatter}, custom classes).
     module Reporting
-      DEFAULT_FORMATS = %w[json lcov cobertura console html].freeze
+      DEFAULT_FORMATS = %w[json lcov cobertura console html csv markdown].freeze
 
       # Comma list for +merge-coverage+ / +run-shards --merge-coverage+ defaults (Codecov, Jenkins, HTML, etc.).
       DEFAULT_MERGE_FORMAT_LIST = DEFAULT_FORMATS.join(",").freeze

data/lib/polyrun/export/csv.rb

@@ -0,0 +1,25 @@
+module Polyrun
+  module Export
+    # RFC-style CSV field escaping for report exports.
+    module Csv
+      module_function
+
+      def escape_field(value)
+        string = value.nil? ? "" : value.to_s
+        if string.match?(/[",\r\n]/)
+          "\"#{string.gsub('"', '""')}\""
+        else
+          string
+        end
+      end
+
+      def generate(headers, rows)
+        lines = [headers.map { |header| escape_field(header) }.join(",")]
+        rows.each do |row|
+          lines << Array(row).map { |cell| escape_field(cell) }.join(",")
+        end
+        lines.join("\n") + "\n"
+      end
+    end
+  end
+end

data/lib/polyrun/export/markdown.rb

@@ -0,0 +1,47 @@
+module Polyrun
+  module Export
+    # GitHub-flavored markdown tables for report exports.
+    module Markdown
+      module_function
+
+      def escape_cell(value)
+        value.nil? ? "" : value.to_s.gsub("|", "\\|").tr("\n", " ")
+      end
+
+      def heading(level, text)
+        level = level.to_i.clamp(1, 6)
+        "#{"#" * level} #{text}\n"
+      end
+
+      def table(headers, rows)
+        return "" if headers.nil? || headers.empty?
+
+        header_line = "| #{headers.map { |header| escape_cell(header) }.join(" | ")} |"
+        separator_line = "| #{headers.map { "---" }.join(" | ")} |"
+        body_lines = rows.map do |row|
+          "| #{Array(row).map { |cell| escape_cell(cell) }.join(" | ")} |"
+        end
+        ([header_line, separator_line] + body_lines).join("\n") + "\n"
+      end
+
+      def document(title, sections)
+        lines = [heading(1, title), ""]
+        sections.each do |section|
+          heading_text = section[:heading] || section["heading"]
+          lines << heading(2, heading_text) if heading_text && !heading_text.to_s.empty?
+
+          body = section[:body] || section["body"]
+          lines << body if body && !body.to_s.empty?
+
+          table_headers = section[:headers] || section["headers"]
+          table_rows = section[:rows] || section["rows"]
+          if table_headers && !table_headers.empty?
+            lines << table(table_headers, table_rows || [])
+          end
+          lines << ""
+        end
+        lines.join("\n")
+      end
+    end
+  end
+end

data/lib/polyrun/hooks/shell_runner.rb

@@ -0,0 +1,33 @@
+require "open3"
+
+module Polyrun
+  class Hooks
+    # Runs shell hook commands; suppresses stdout/stderr on success unless verbose.
+    module ShellRunner
+      private
+
+      def run_shell_hook(cmd, merged)
+        if hook_shell_output_verbose?
+          ok = system(merged, "sh", "-c", cmd)
+          return [ok, $?.exitstatus]
+        end
+
+        stdout, stderr, status = Open3.capture3(merged, "sh", "-c", cmd)
+        unless status.success?
+          $stdout.print(stdout) unless stdout.empty?
+          $stderr.print(stderr) unless stderr.empty?
+        end
+        [status.success?, status.exitstatus]
+      rescue Interrupt
+        Polyrun::Log.warn "polyrun hooks: shell hook interrupted"
+        [false, 130]
+      end
+
+      def hook_shell_output_verbose?
+        Polyrun::Debug.enabled? ||
+          %w[1 true yes].include?(ENV["POLYRUN_VERBOSE"]&.to_s&.downcase) ||
+          %w[1 true yes].include?(ENV["POLYRUN_HOOKS_VERBOSE"]&.to_s&.downcase)
+      end
+    end
+  end
+end

data/lib/polyrun/hooks.rb

@@ -2,6 +2,7 @@ require "shellwords"
 
 require_relative "log"
 require_relative "hooks/dsl"
+require_relative "hooks/shell_runner"
 require_relative "hooks/worker_runner"
 require_relative "hooks/worker_shell"
 
@@ -17,6 +18,7 @@ module Polyrun
   # Orchestration respects +POLYRUN_HOOKS_DISABLE=1+ (+run_phase_if_enabled+); +polyrun hook run+ always runs {#run_phase}.
   class Hooks
     include WorkerShell
+    include ShellRunner
 
     PHASES = %i[
       before_suite after_suite
@@ -128,13 +130,8 @@ module Polyrun
       end
 
       commands_for(phase).each do |cmd|
-        ok = begin
-          system(merged, "sh", "-c", cmd)
-        rescue Interrupt
-          Polyrun::Log.warn "polyrun hooks: #{phase} shell hook interrupted"
-          return 130
-        end
-        return $?.exitstatus unless ok
+        ok, exitstatus = run_shell_hook(cmd, merged)
+        return exitstatus unless ok
       end
       0
     end

data/lib/polyrun/partition/paths_build.rb

@@ -40,12 +40,20 @@ module Polyrun
 
         pool = glob_under_cwd(all_glob, cwd)
         pool.uniq!
+        pool = reject_exclude_prefixes(pool, pb)
         stages = Array(pb["stages"])
         return sort_paths(pool) if stages.empty?
 
         apply_stages_to_pool(stages, pool, cwd)
       end
 
+      def reject_exclude_prefixes(pool, pb)
+        prefixes = Array(pb["exclude_prefixes"]).map(&:to_s).reject(&:empty?)
+        return pool if prefixes.empty?
+
+        pool.reject { |path| prefixes.any? { |prefix| path.start_with?(prefix) } }
+      end
+
       def apply_stages_to_pool(stages, pool, cwd)
         remaining = Set.new(pool)
         out = []

data/lib/polyrun/queue/worker_loop.rb

@@ -0,0 +1,39 @@
+module Polyrun
+  module Queue
+    # Claim / run / ack loop for +polyrun run-queue+ workers (forked or direct).
+    module WorkerLoop
+      module_function
+
+      def run(store:, worker_id:, batch:, cmd:, on_failure:)
+        batches_ok = 0
+        batches_fail = 0
+        loop do
+          claim = store.claim!(worker_id: worker_id, batch_size: batch)
+          paths = claim["paths"] || []
+          break if paths.empty?
+
+          code = run_batch(cmd, paths)
+          if code == 0
+            store.ack!(lease_id: claim["lease_id"], worker_id: worker_id)
+            batches_ok += 1
+          elsif on_failure.to_s == "requeue"
+            store.reclaim_lease!(claim["lease_id"])
+            batches_fail += 1
+            return {ok: batches_ok, fail: batches_fail, exit_code: 1}
+          else
+            batches_fail += 1
+            return {ok: batches_ok, fail: batches_fail, exit_code: code.zero? ? 1 : code}
+          end
+        end
+        {ok: batches_ok, fail: batches_fail, exit_code: 0}
+      rescue Polyrun::Error => e
+        Polyrun::Log.warn "polyrun run-queue worker #{worker_id}: #{e.message}"
+        {ok: batches_ok, fail: batches_fail, exit_code: 2}
+      end
+
+      def run_batch(cmd, paths)
+        system(*cmd, *paths) ? 0 : ($?.exitstatus || 1)
+      end
+    end
+  end
+end

data/lib/polyrun/reporting/failure_merge.rb

@@ -1,6 +1,9 @@
 require "json"
 require "fileutils"
 
+require_relative "../export/csv"
+require_relative "../export/markdown"
+
 module Polyrun
   module Reporting
     # Merge per-worker / per-shard failure fragments (JSONL or RSpec JSON) into one report.
@@ -8,6 +11,10 @@ module Polyrun
     module FailureMerge
       DEFAULT_FRAGMENT_DIR = "tmp/polyrun_failures".freeze
       FRAGMENT_GLOB = "polyrun-failure-fragment-*.jsonl".freeze
+      CSV_HEADERS = %w[
+        id full_description location file_path line_number message exception_class
+        polyrun_shard_index polyrun_shard_total source
+      ].freeze
 
       module_function
 
@@ -24,7 +31,7 @@ module Polyrun
       end
 
       # @param paths [Array<String>] fragment paths (.jsonl and/or RSpec --format json outputs)
-      # @param format [String] "jsonl" or "json"
+      # @param format [String] "jsonl", "json", "csv", or "markdown"
       # @param output [String] destination path
       # @return [Integer] count of failure rows merged
       def merge_files!(paths, output:, format: "jsonl")
@@ -34,23 +41,52 @@ module Polyrun
         FileUtils.mkdir_p(File.dirname(out_abs))
         case fmt
         when "json"
-          doc = {
-            "meta" => {
-              "polyrun_merge" => true,
-              "inputs" => paths.map { |p| File.expand_path(p) },
-              "failure_count" => rows.size
-            },
-            "failures" => rows
-          }
-          File.write(out_abs, JSON.generate(doc))
+          write_json_output!(out_abs, paths, rows)
         when "jsonl"
-          File.write(out_abs, rows.map { |h| JSON.generate(h) }.join("\n") + (rows.empty? ? "" : "\n"))
+          write_jsonl_output!(out_abs, rows)
+        when "csv"
+          write_csv_output!(out_abs, rows)
+        when "markdown", "md"
+          write_markdown_output!(out_abs, rows)
         else
-          raise Polyrun::Error, "merge-failures: unknown format #{fmt.inspect} (use jsonl or json)"
+          raise Polyrun::Error, "merge-failures: unknown format #{fmt.inspect} (use jsonl, json, csv, or markdown)"
         end
         rows.size
       end
 
+      def write_json_output!(out_abs, paths, rows)
+        doc = {
+          "meta" => {
+            "polyrun_merge" => true,
+            "inputs" => paths.map { |p| File.expand_path(p) },
+            "failure_count" => rows.size
+          },
+          "failures" => rows
+        }
+        File.write(out_abs, JSON.generate(doc))
+      end
+
+      def write_jsonl_output!(out_abs, rows)
+        File.write(out_abs, rows.map { |h| JSON.generate(h) }.join("\n") + (rows.empty? ? "" : "\n"))
+      end
+
+      def write_csv_output!(out_abs, rows)
+        csv_rows = rows.map { |row| CSV_HEADERS.map { |key| row[key] } }
+        File.write(out_abs, Export::Csv.generate(CSV_HEADERS, csv_rows))
+      end
+
+      def write_markdown_output!(out_abs, rows)
+        sections = rows.map.with_index(1) do |row, index|
+          {
+            heading: "Failure #{index}: #{row["full_description"] || row["location"] || row["id"]}",
+            headers: %w[field value],
+            rows: row.sort_by { |key, _| key.to_s }.map { |key, value| [key, value] }
+          }
+        end
+        sections = [{heading: "Failures", body: "(none)"}] if sections.empty?
+        File.write(out_abs, Export::Markdown.document("Polyrun failure report", sections))
+      end
+
       def collect_rows(paths)
         rows = []
         paths.each do |p|