polyrun 2.1.2 → 2.2.0

data/lib/polyrun/timing/summary.rb

@@ -1,26 +1,76 @@
 require_relative "stats"
+require_relative "../export/csv"
+require_relative "../export/markdown"
 
 module Polyrun
   module Timing
     # Human-readable slow-file list from merged timing JSON (per-file cost).
     module Summary
+      CSV_HEADERS = %w[rank path last_seconds min max mean p95 runs failures timeouts].freeze
+      MARKDOWN_HEADERS = %w[rank path seconds].freeze
+
       module_function
 
+      def ranked_entries(merged, top: 30)
+        return [] if merged.nil? || merged.empty?
+
+        merged.map { |path, entry| [path, Stats.normalize_entry(entry)] }
+          .sort_by { |(_, entry)| -Stats.binpack_weight(entry) }
+          .first(Integer(top))
+      end
+
       # +merged+ is path (String) => seconds (Float) or stats Hash, as produced by +Timing::Merge.merge_files+.
       def format_slow_files(merged, top: 30, title: "Polyrun slowest files (by wall time, seconds)")
         return "#{title}\n  (no data)\n" if merged.nil? || merged.empty?
 
-        pairs = merged.map { |path, sec| [path, Stats.binpack_weight(sec)] }
-          .sort_by { |(_, sec)| -sec.to_f }.first(Integer(top))
         lines = [title, ""]
-        pairs.each_with_index do |(path, sec), i|
-          lines << format("  %2d. %s  %.4f", i + 1, path, sec.to_f)
+        ranked_entries(merged, top: top).each_with_index do |(path, entry), index|
+          lines << format("  %2d. %s  %.4f", index + 1, path, Stats.binpack_weight(entry).to_f)
         end
         lines.join("\n") + "\n"
       end
 
+      def format_csv(merged, top: 30)
+        rows = ranked_entries(merged, top: top).each_with_index.map do |(path, entry), index|
+          [
+            index + 1,
+            path,
+            entry["last_seconds"],
+            entry["min"],
+            entry["max"],
+            entry["mean"],
+            entry["p95"],
+            entry["runs"],
+            entry["failures"],
+            entry["timeouts"]
+          ]
+        end
+        Export::Csv.generate(CSV_HEADERS, rows)
+      end
+
+      def format_markdown(merged, top: 30, title: "Polyrun slowest files")
+        rows = ranked_entries(merged, top: top).each_with_index.map do |(path, entry), index|
+          [index + 1, path, format("%.4f", Stats.binpack_weight(entry).to_f)]
+        end
+        Export::Markdown.document(
+          title,
+          [{heading: "Slow files", headers: MARKDOWN_HEADERS, rows: rows}]
+        )
+      end
+
+      def render(merged, format: "text", **kwargs)
+        case format.to_s.downcase
+        when "text", "console", "txt" then format_slow_files(merged, **kwargs)
+        when "csv" then format_csv(merged, **kwargs)
+        when "markdown", "md" then format_markdown(merged, **kwargs)
+        else
+          raise Polyrun::Error, "report-timing: unknown format #{format.inspect} (use text, csv, or markdown)"
+        end
+      end
+
       def write_file(merged, path, **kwargs)
-        File.write(path, format_slow_files(merged, **kwargs))
+        format = kwargs.delete(:format) || "text"
+        File.write(path, render(merged, format: format, **kwargs))
         path
       end
     end

data/lib/polyrun/version.rb

@@ -1,3 +1,3 @@
 module Polyrun
-  VERSION = "2.1.2"
+  VERSION = "2.2.0"
 end

data/lib/polyrun/worker_output.rb

@@ -0,0 +1,159 @@
+require "fileutils"
+require_relative "worker_output_forwarders"
+
+module Polyrun
+  # Routes parallel worker stdout/stderr through per-shard log files with optional prefixed TTY echo.
+  #
+  # Opt-in: +POLYRUN_WORKER_OUTPUT_ROUTING=1+ (or set +POLYRUN_WORKER_LOG_DIR+).
+  # +POLYRUN_WORKER_OUTPUT_ROUTING=0+ keeps inherited stdio (default polyrun behavior).
+  # +POLYRUN_WORKER_OUTPUT_PREFIX=0+ writes logs only (no live prefixed echo).
+  module WorkerOutput
+    module_function
+
+    def routing_enabled?
+      return false if disabled_by_env?
+
+      truthy?(ENV["POLYRUN_WORKER_OUTPUT_ROUTING"]) || !log_directory.nil?
+    end
+
+    def log_directory
+      value = ENV["POLYRUN_WORKER_LOG_DIR"]
+      return nil if value.nil?
+
+      stripped = value.to_s.strip
+      stripped.empty? ? nil : stripped
+    end
+
+    def prefix_live?
+      return true if ENV["POLYRUN_WORKER_OUTPUT_PREFIX"].nil?
+
+      truthy?(ENV["POLYRUN_WORKER_OUTPUT_PREFIX"])
+    end
+
+    def log_path_for(shard)
+      File.expand_path("shard-#{shard}.log", log_directory || default_log_directory)
+    end
+
+    def worker_log_directory_label
+      log_directory || default_log_directory
+    end
+
+    def prepare_log_dir!
+      directory = log_directory || default_log_directory
+      FileUtils.mkdir_p(directory)
+      Dir.glob(File.join(directory, "shard-*.log")).each { |path| File.delete(path) }
+    end
+
+    def warn_shard_log(shard)
+      return unless routing_enabled?
+
+      Polyrun::Log.warn "polyrun run-shards: shard #{shard} worker log → #{log_path_for(shard)}"
+    end
+
+    def spawn_worker(child_env, cmd, paths, hook_cfg)
+      shard = child_env.fetch("POLYRUN_SHARD_INDEX", "?")
+      out_read, out_write = IO.pipe
+      err_read, err_write = IO.pipe
+
+      pid =
+        if hook_cfg.worker_hooks? && !Polyrun::Hooks.disabled?
+          Process.spawn(
+            child_env,
+            "sh",
+            "-c",
+            hook_cfg.build_worker_shell_script(cmd, paths),
+            out: out_write,
+            err: err_write
+          )
+        else
+          Process.spawn(child_env, *cmd, *paths, out: out_write, err: err_write)
+        end
+
+      out_write.close
+      err_write.close
+
+      start_forwarders(
+        pid: pid,
+        shard: shard,
+        stdout_io: out_read,
+        stderr_io: err_read,
+        log_path: log_path_for(shard)
+      )
+      pid
+    end
+
+    def start_forwarders(pid:, shard:, stdout_io:, stderr_io:, log_path:)
+      forwarder = WorkerForwarder.new(
+        shard: shard,
+        pid: pid,
+        log_path: log_path,
+        prefix_live: prefix_live?
+      )
+      threads = [
+        forwarder_thread(stdout_io, forwarder, :stdout),
+        forwarder_thread(stderr_io, forwarder, :stderr)
+      ]
+      registry[pid] = {threads: threads, ios: [stdout_io, stderr_io], forwarder: forwarder}
+    end
+
+    def finish_worker(pid)
+      entry = registry.delete(pid)
+      return unless entry
+
+      drain_forwarders(entry)
+    end
+
+    def shutdown_all!
+      registry.each_value { |entry| drain_forwarders(entry) }
+      registry.clear
+    end
+
+    def default_log_directory
+      "tmp/polyrun/workers"
+    end
+
+    def disabled_by_env?
+      %w[0 false no off].include?(ENV["POLYRUN_WORKER_OUTPUT_ROUTING"].to_s.strip.downcase)
+    end
+
+    def truthy?(value)
+      return false if value.nil?
+
+      %w[1 true yes on].include?(value.to_s.strip.downcase)
+    end
+    private_class_method :truthy?
+
+    # rubocop:disable ThreadSafety/ClassInstanceVariable -- per-parent-process worker forwarder registry
+    def registry
+      @registry ||= {}
+    end
+    private_class_method :registry
+    # rubocop:enable ThreadSafety/ClassInstanceVariable
+
+    def forwarder_thread(io, forwarder, stream)
+      # rubocop:disable ThreadSafety/NewThread -- one reader thread per worker pipe
+      Thread.new do
+        loop do
+          forwarder.consume(stream, io.readpartial(4096))
+        end
+      rescue IOError, Errno::EPIPE
+        # worker closed the pipe
+      ensure
+        io.close unless io.closed?
+      end
+      # rubocop:enable ThreadSafety/NewThread
+    end
+    private_class_method :forwarder_thread
+
+    def drain_forwarders(entry)
+      entry[:ios].each do |io|
+        io.close unless io.closed?
+      rescue IOError
+        nil
+      end
+      entry[:threads].each(&:join)
+      entry[:forwarder].close
+    end
+    private_class_method :drain_forwarders
+  end
+end

data/lib/polyrun/worker_output_forwarders.rb

@@ -0,0 +1,90 @@
+require "time"
+
+module Polyrun
+  module WorkerOutput
+    class LineForwarder
+      def initialize(shard:, pid:, log_io:, prefix_live:, tty_io:)
+        @shard = shard
+        @pid = pid
+        @log_io = log_io
+        @prefix_live = prefix_live
+        @tty_io = tty_io
+        @buffer = +""
+        @mutex = Mutex.new
+      end
+
+      def consume(chunk)
+        return if chunk.nil? || chunk.empty?
+
+        @mutex.synchronize do
+          if progress_chunk?(chunk)
+            write_tty(chunk)
+            @log_io&.write(chunk)
+            return
+          end
+
+          @buffer << chunk
+          while (newline_index = @buffer.index("\n"))
+            emit_line(@buffer.slice!(0, newline_index + 1))
+          end
+        end
+      end
+
+      def flush
+        @mutex.synchronize do
+          next if @buffer.empty?
+
+          emit_line(@buffer)
+          @buffer.clear
+        end
+      end
+
+      private
+
+      def progress_chunk?(chunk)
+        chunk.include?("\r") && !chunk.include?("\n")
+      end
+
+      def emit_line(line)
+        prefixed = "#{prefix}#{line}"
+        @log_io&.write(prefixed)
+        write_tty(prefixed) if @prefix_live
+      end
+
+      def prefix
+        "[#{Time.now.strftime("%Y-%m-%dT%H:%M:%S")} shard=#{@shard} pid=#{@pid}] "
+      end
+
+      def write_tty(text)
+        @tty_io.write(text)
+        @tty_io.flush
+      end
+    end
+
+    class WorkerForwarder
+      def initialize(shard:, pid:, log_path:, prefix_live:)
+        @shard = shard
+        @pid = pid
+        @log_io = File.open(log_path, "wb")
+        @stdout = LineForwarder.new(shard: shard, pid: pid, log_io: @log_io, prefix_live: prefix_live, tty_io: Polyrun::Log.stdout)
+        @stderr = LineForwarder.new(shard: shard, pid: pid, log_io: @log_io, prefix_live: prefix_live, tty_io: Polyrun::Log.stderr)
+        @write_mutex = Mutex.new
+      end
+
+      def consume(stream, chunk)
+        @write_mutex.synchronize do
+          case stream
+          when :stdout then @stdout.consume(chunk)
+          when :stderr then @stderr.consume(chunk)
+          end
+        end
+      end
+
+      def close
+        @stdout.flush
+        @stderr.flush
+        @log_io.close
+      end
+    end
+  end
+end

data/polyrun.gemspec

@@ -10,10 +10,14 @@ Gem::Specification.new do |spec|
   spec.license = "MIT"
   spec.required_ruby_version = ">= 3.1.0"
 
-  spec.files = Dir["lib/**/*", "sig/**/*.rbs", "bin/polyrun", "README.md", "CHANGELOG.md", "docs/SETUP_PROFILE.md", "LICENSE", "CONTRIBUTING.md", "CODE_OF_CONDUCT.md", "SECURITY.md", "polyrun.gemspec"].reject { |f| File.directory?(f) }
+  spec.files = (
+    Dir["lib/**/*", "sig/**/*.rbs", "bin/polyrun", "README.md", "CHANGELOG.md", "docs/SETUP_PROFILE.md", "LICENSE", "CONTRIBUTING.md", "CODE_OF_CONDUCT.md", "SECURITY.md", "polyrun.gemspec"] +
+    Dir["ext/**/extconf.rb", "ext/**/*.c", "ext/**/*.h"]
+  ).uniq.reject { |f| File.directory?(f) }
   spec.bindir = "bin"
   spec.executables = ["polyrun"]
   spec.require_paths = ["lib"]
+  spec.extensions = ["ext/polyrun_coverage_merge/extconf.rb"]
 
   spec.metadata = {
     "homepage_uri" => spec.homepage,
@@ -25,6 +29,9 @@ Gem::Specification.new do |spec|
   # Normative: zero runtime dependencies (stdlib + vendored/native code only).
   # Ruby 3.5+: `benchmark` is no longer a default gem; keep scripts and `rake bench_merge` warning-free.
   spec.add_development_dependency "benchmark", ">= 0.3"
+  spec.add_development_dependency "benchmark-ips", "~> 2"
+  spec.add_development_dependency "memory_profiler", "~> 1"
+  spec.add_development_dependency "stackprof", "~> 0.2"
   spec.add_development_dependency "rake", "~> 13.0"
   spec.add_development_dependency "rspec", "~> 3.12"
   spec.add_development_dependency "appraisal", "~> 2.5"

data/sig/polyrun/rspec.rbs

@@ -7,5 +7,15 @@ module Polyrun
     def self.install_failure_fragments!: (?only_if: untyped?) -> void
 
     def self.install_worker_ping!: () -> void
+
+    def self.install_example_debug!: (?rspec_config: untyped?) -> void
+
+    def self.install_example_timeout!: (?rspec_config: untyped?, ?seconds: Float) -> void
+
+    def self.install_example_rails_logging!: (?rspec_config: untyped?) -> void
+
+    def self.install_example_prosopite!: (?rspec_config: untyped?) -> void
+
+    def self.install_sharded_formatter_compat!: (?rspec_config: untyped?) -> void
   end
 end

data/sig/polyrun/worker_output.rbs

@@ -0,0 +1,11 @@
+module Polyrun
+  module WorkerOutput
+    def self.routing_enabled?: () -> bool
+    def self.log_path_for: (Integer | String shard) -> String
+    def self.prepare_log_dir!: () -> void
+    def self.warn_shard_log: (Integer | String shard) -> void
+    def self.spawn_worker: (Hash[Symbol, untyped] child_env, Array[String] cmd, Array[String] paths, untyped hook_cfg) -> Integer
+    def self.finish_worker: (Integer pid) -> void
+    def self.shutdown_all!: () -> void
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: polyrun
 version: !ruby/object:Gem::Version
-  version: 2.1.2
+  version: 2.2.0
 platform: ruby
 authors:
 - Andrei Makarov
@@ -23,6 +23,48 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0.3'
+- !ruby/object:Gem::Dependency
+  name: benchmark-ips
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: memory_profiler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+- !ruby/object:Gem::Dependency
+  name: stackprof
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -153,7 +195,8 @@ email:
 - contact@kiskolabs.com
 executables:
 - polyrun
-extensions: []
+extensions:
+- ext/polyrun_coverage_merge/extconf.rb
 extra_rdoc_files: []
 files:
 - CHANGELOG.md
@@ -164,8 +207,13 @@ files:
 - SECURITY.md
 - bin/polyrun
 - docs/SETUP_PROFILE.md
+- ext/polyrun_coverage_merge/coverage_merge.c
+- ext/polyrun_coverage_merge/extconf.rb
 - lib/polyrun.rb
+- lib/polyrun/benchmark/profile.rb
+- lib/polyrun/benchmark/report.rb
 - lib/polyrun/cli.rb
+- lib/polyrun/cli/benchmark_commands.rb
 - lib/polyrun/cli/ci_shard_hooks.rb
 - lib/polyrun/cli/ci_shard_run_command.rb
 - lib/polyrun/cli/ci_shard_run_parse.rb
@@ -208,6 +256,9 @@ files:
 - lib/polyrun/coverage/collector_finish.rb
 - lib/polyrun/coverage/collector_fragment_meta.rb
 - lib/polyrun/coverage/example_diff.rb
+- lib/polyrun/coverage/example_diff_snapshot.rb
+- lib/polyrun/coverage/example_diff_track_filter.rb
+- lib/polyrun/coverage/file_stats_report.rb
 - lib/polyrun/coverage/filter.rb
 - lib/polyrun/coverage/formatter.rb
 - lib/polyrun/coverage/merge.rb
@@ -222,6 +273,7 @@ files:
 - lib/polyrun/coverage/merge/html/template.html.erb
 - lib/polyrun/coverage/merge_fragment_meta.rb
 - lib/polyrun/coverage/merge_merge_two.rb
+- lib/polyrun/coverage/merge_native.rb
 - lib/polyrun/coverage/rails.rb
 - lib/polyrun/coverage/reporting.rb
 - lib/polyrun/coverage/result.rb
@@ -242,8 +294,11 @@ files:
 - lib/polyrun/database/url_builder/template_prepare.rb
 - lib/polyrun/debug.rb
 - lib/polyrun/env/ci.rb
+- lib/polyrun/export/csv.rb
+- lib/polyrun/export/markdown.rb
 - lib/polyrun/hooks.rb
 - lib/polyrun/hooks/dsl.rb
+- lib/polyrun/hooks/shell_runner.rb
 - lib/polyrun/hooks/worker_runner.rb
 - lib/polyrun/hooks/worker_shell.rb
 - lib/polyrun/log.rb
@@ -266,6 +321,7 @@ files:
 - lib/polyrun/queue/duration.rb
 - lib/polyrun/queue/file_store.rb
 - lib/polyrun/queue/file_store_pending.rb
+- lib/polyrun/queue/worker_loop.rb
 - lib/polyrun/quick.rb
 - lib/polyrun/quick/assertions.rb
 - lib/polyrun/quick/errors.rb
@@ -278,9 +334,13 @@ files:
 - lib/polyrun/reporting/failure_merge.rb
 - lib/polyrun/reporting/junit.rb
 - lib/polyrun/reporting/junit_emit.rb
+- lib/polyrun/reporting/junit_report.rb
 - lib/polyrun/reporting/rspec_failure_fragment_formatter.rb
 - lib/polyrun/reporting/rspec_junit.rb
 - lib/polyrun/rspec.rb
+- lib/polyrun/rspec/example_debug.rb
+- lib/polyrun/rspec/example_debug_instrumentation.rb
+- lib/polyrun/rspec/sharded_formatter_compat.rb
 - lib/polyrun/spec_quality.rb
 - lib/polyrun/spec_quality/config.rb
 - lib/polyrun/spec_quality/fragment.rb
@@ -303,6 +363,8 @@ files:
 - lib/polyrun/timing/summary.rb
 - lib/polyrun/timing/variance_report.rb
 - lib/polyrun/version.rb
+- lib/polyrun/worker_output.rb
+- lib/polyrun/worker_output_forwarders.rb
 - lib/polyrun/worker_ping.rb
 - polyrun.gemspec
 - sig/polyrun.rbs
@@ -313,6 +375,7 @@ files:
 - sig/polyrun/minitest.rbs
 - sig/polyrun/quick.rbs
 - sig/polyrun/rspec.rbs
+- sig/polyrun/worker_output.rbs
 - sig/polyrun/worker_ping.rbs
 homepage: https://github.com/amkisko/polyrun.rb
 licenses: