rspec_telemetry 0.3.0

data/lib/rspec_telemetry/compare_cli.rb

@@ -0,0 +1,129 @@
+# frozen_string_literal: true
+
+require "optparse"
+
+require_relative "factory_comparison"
+
+module RSpecTelemetry
+  class CompareCLI
+    def initialize(argv, out: $stdout, err: $stderr)
+      @argv = argv
+      @out = out
+      @err = err
+      @options = {all_depths: false, sort: "duration"}
+    end
+
+    def run
+      paths = parse!
+      unless paths.length == 2
+        @err.puts("Specify exactly two telemetry files: BEFORE AFTER")
+        return 1
+      end
+
+      comparison = FactoryComparison.new(paths[0], paths[1], all_depths: @options[:all_depths])
+      rows = sort_rows(comparison.rows)
+      @out.puts(render(rows, duration_label: comparison.all_depths ? "Self(ms)" : "Total(ms)"))
+      0
+    rescue Errno::ENOENT => e
+      @err.puts("File not found: #{e.message}")
+      1
+    rescue OptionParser::ParseError => e
+      @err.puts(e.message)
+      1
+    end
+
+    private
+
+    def parse!
+      parser = OptionParser.new do |options|
+        options.banner = "Usage: rspec-telemetry-compare [options] BEFORE AFTER"
+        options.on("--all-depths", "Include nested FactoryBot events") do
+          @options[:all_depths] = true
+        end
+        options.on(
+          "--sort KEY",
+          %w[duration count factory],
+          "Sort by duration, count, or factory (default: duration)"
+        ) do |value|
+          @options[:sort] = value
+        end
+        options.on("-h", "--help", "Show this help") do
+          @out.puts(options)
+          exit(0)
+        end
+      end
+
+      parser.parse(@argv)
+    end
+
+    def sort_rows(rows)
+      case @options[:sort]
+      when "count"
+        rows.sort_by { |row| [-row.count_diff.abs, row.factory] }
+      when "factory"
+        rows.sort_by(&:factory)
+      else
+        rows.sort_by { |row| [-row.duration_diff_ms.abs, row.factory] }
+      end
+    end
+
+    def render(rows, duration_label:)
+      headings = [
+        "Factory",
+        "Before",
+        "After",
+        "Diff",
+        "Change",
+        "Before #{duration_label}",
+        "After #{duration_label}",
+        "Diff(ms)",
+        "Change"
+      ]
+      body = rows.map do |row|
+        [
+          row.factory,
+          row.before_count.to_s,
+          row.after_count.to_s,
+          signed_integer(row.count_diff),
+          percent(row.count_change_percent),
+          decimal(row.before_duration_ms),
+          decimal(row.after_duration_ms),
+          signed_decimal(row.duration_diff_ms),
+          percent(row.duration_change_percent)
+        ]
+      end
+
+      widths = headings.each_index.map do |index|
+        ([headings[index]] + body.map { |columns| columns[index] }).map(&:length).max
+      end
+
+      lines = []
+      lines << format_row(headings, widths)
+      lines << widths.map { |width| "-" * width }.join("-+-")
+      body.each { |columns| lines << format_row(columns, widths) }
+      lines.join("\n")
+    end
+
+    def format_row(columns, widths)
+      columns.each_with_index.map do |value, index|
+        index.zero? ? value.ljust(widths[index]) : value.rjust(widths[index])
+      end.join(" | ")
+    end
+
+    def signed_integer(value)
+      format("%+d", value)
+    end
+
+    def decimal(value)
+      format("%.1f", value)
+    end
+
+    def signed_decimal(value)
+      format("%+.1f", value)
+    end
+
+    def percent(value)
+      value ? format("%+.1f%%", value) : "-"
+    end
+  end
+end

data/lib/rspec_telemetry/config.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module RSpecTelemetry
+  class Config
+    attr_accessor(
+      :enabled,
+      :output_path,
+      :capture_examples,
+      :capture_factory_bot,
+      :print_summary,
+      :flush_each,
+      :slow_factory_threshold_ms,
+      :slow_example_threshold_ms,
+      :summary_io,
+      :summary_limit
+    )
+
+    def initialize
+      @enabled = true
+      @output_path = self.class.default_output_path
+      @capture_examples = true
+      @capture_factory_bot = true
+      @print_summary = false
+      @flush_each = false
+      @slow_factory_threshold_ms = nil
+      @slow_example_threshold_ms = nil
+      @summary_io = $stderr
+      @summary_limit = 20
+    end
+
+    def self.default_output_path
+      suffix = ENV["TEST_ENV_NUMBER"]
+      if suffix && !suffix.empty?
+        "tmp/rspec_telemetry.#{suffix}.ndjson"
+      else
+        "tmp/rspec_telemetry.ndjson"
+      end
+    end
+  end
+end

data/lib/rspec_telemetry/console_report.rb

@@ -0,0 +1,124 @@
+# frozen_string_literal: true
+
+module RSpecTelemetry
+  class ConsoleReport
+    module Helpers
+      module_function
+
+      def fmt(ms)
+        ms = ms.to_f
+        ms >= 1000 ? format("%.2fs", ms / 1000.0) : format("%.1fms", ms)
+      end
+
+      def pct(ratio) = format("%.1f%%", ratio * 100)
+
+      def truncate(str, len) = str.length > len ? "#{str[0, len - 1]}…" : str
+
+      def section(title) = ["", title, "-" * title.length]
+    end
+
+    include Helpers
+
+    def initialize(analyzer, files_count:, top: 15)
+      @analyzer = analyzer
+      @files_count = files_count
+      @top = top
+    end
+
+    def render
+      lines = overview + slow_files + slow_examples + top_factories
+      lines << ""
+      lines << "Tip: drill into one example with `rspec-telemetry --example \"<example_id>\"`"
+      lines.join("\n")
+    end
+
+    def self.drill_down(events, example_id)
+      lines = Helpers.section("Example: #{example_id}")
+      finished = events.find { |e| e["type"] == "example.finished" }
+      if finished
+        lines << "  #{finished["full_description"]}"
+        lines << "  status: #{finished["status"]}   duration: #{Helpers.fmt(finished["duration_ms"])}"
+      end
+
+      factories = events.select { |e| e["type"] == "factory_bot.run_factory" }
+      return lines.join("\n") if factories.empty?
+
+      lines << ""
+      lines << "  FactoryBot calls (indented by nesting depth):"
+      factories.each { |f| lines << factory_line(f) }
+      total = factories.sum { |f| (f["self_duration_ms"] || f["duration_ms"]).to_f }
+      lines << ""
+      lines << "  factory self total: #{Helpers.fmt(total)} across #{factories.size} calls"
+      lines.join("\n")
+    end
+
+    def self.factory_line(fields)
+      indent = "    " + ("  " * fields["depth"].to_i)
+      traits = Array(fields["traits"]).empty? ? "" : " [#{fields["traits"].join(",")}]"
+      self_ms = fields["self_duration_ms"] || fields["duration_ms"]
+      "#{indent}#{fields["factory"]}:#{fields["strategy"]}#{traits}  " \
+        "self #{Helpers.fmt(self_ms)} / total #{Helpers.fmt(fields["duration_ms"])}"
+    end
+
+    private
+
+    def overview
+      a = @analyzer
+      section("Overview") +
+        [
+          "  files analyzed:        #{@files_count}",
+          "  examples:              #{a.example_count} (#{a.failure_count} failed, #{a.pending_count} pending)",
+          "  suite wall time:       #{fmt(a.suite_duration_ms)}",
+          "  example time (sum):    #{fmt(a.total_example_ms)}",
+          "  factory self time:     #{fmt(a.total_factory_self_ms)} (#{pct(a.factory_time_ratio)} of example time)"
+        ]
+    end
+
+    def slow_files
+      rows = @analyzer.slow_files(@top)
+      return [] if rows.empty?
+
+      section("Slowest files (sum of example time)") +
+        rows.each_with_index.map do |f, i|
+          format("  %2d. %-9s %3d ex   %s", i + 1, fmt(f.duration_ms), f.example_count, f.file_path)
+        end
+    end
+
+    def slow_examples
+      rows = @analyzer.slow_examples(@top)
+      return [] if rows.empty?
+
+      lines = section("Slowest examples")
+      rows.each_with_index do |e, i|
+        fb = e.fb_count.to_i.positive? ? "  [factories: #{fmt(e.fb_self_total_ms)} / #{e.fb_count} calls]" : ""
+        lines << format("  %2d. %-9s %s", i + 1, fmt(e.duration_ms), e.example_id)
+        lines << "        #{e.full_description}#{fb}" if e.full_description || !fb.empty?
+      end
+
+      lines
+    end
+
+    def top_factories
+      rows = @analyzer.top_factories(@top)
+      return [] if rows.empty?
+
+      lines = section("Slowest factories (by self time, excludes nested children)")
+      lines << format("      %-28s %6s %10s %10s %9s %9s", "factory:strategy", "count", "self", "total", "avg", "max")
+      rows.each_with_index do |f, i|
+        lines <<
+          format(
+            "  %2d. %-28s %6d %10s %10s %9s %9s",
+            i + 1,
+            truncate(f.key, 28),
+            f.count,
+            fmt(f.self_total_ms),
+            fmt(f.total_ms),
+            fmt(f.avg_ms),
+            fmt(f.max_ms)
+          )
+      end
+
+      lines
+    end
+  end
+end

data/lib/rspec_telemetry/factory_aggregation.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module RSpecTelemetry
+  module FactoryAggregation
+    # Shared by live summaries, CLI reports, and the viewer so ranking rules stay
+    # identical. A Struct (not Data) so collection runs on Ruby 3.1, where this is
+    # on the require path via summary.rb.
+    Stat = Struct.new(:key, :factory, :strategy, :count, :total_ms, :self_total_ms, :max_ms, keyword_init: true) do
+      def avg_ms = count.zero? ? 0.0 : total_ms / count
+    end
+
+    class Accumulator
+      def initialize
+        @rows = {}
+      end
+
+      def add(factory:, strategy:, duration_ms:, self_duration_ms: nil)
+        key = "#{factory}:#{strategy}"
+        total = duration_ms.to_f
+        self_ms = (self_duration_ms || duration_ms).to_f
+        row = (@rows[key] ||= {factory: factory, strategy: strategy, count: 0, total: 0.0, self: 0.0, max: 0.0})
+        row[:count] += 1
+        row[:total] += total
+        row[:self] += self_ms
+        row[:max] = total if total > row[:max]
+        self
+      end
+
+      def stats
+        @rows.map do |key, row|
+          Stat.new(
+            key: key,
+            factory: row[:factory],
+            strategy: row[:strategy],
+            count: row[:count],
+            total_ms: row[:total],
+            self_total_ms: row[:self],
+            max_ms: row[:max]
+          )
+        end
+      end
+
+      def top(limit = nil)
+        # Rank by self time so nested factories are not double-counted.
+        ranked = stats.sort_by { |stat| -stat.self_total_ms }
+        limit ? ranked.first(limit) : ranked
+      end
+    end
+  end
+end

data/lib/rspec_telemetry/factory_comparison.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+require_relative "ndjson"
+
+module RSpecTelemetry
+  class FactoryComparison
+    FactoryStat = Struct.new(:count, :duration_ms)
+    Row = Struct.new(
+      :factory,
+      :before_count,
+      :after_count,
+      :before_duration_ms,
+      :after_duration_ms,
+      keyword_init: true
+    ) do
+      def count_diff
+        after_count - before_count
+      end
+
+      def duration_diff_ms
+        after_duration_ms - before_duration_ms
+      end
+
+      def count_change_percent
+        change_percent(before_count, after_count)
+      end
+
+      def duration_change_percent
+        change_percent(before_duration_ms, after_duration_ms)
+      end
+
+      private
+
+      def change_percent(before_value, after_value)
+        return nil if before_value.zero?
+
+        ((after_value - before_value) / before_value.to_f) * 100
+      end
+    end
+
+    attr_reader :before_path, :after_path, :all_depths
+
+    def initialize(before_path, after_path, all_depths: false)
+      @before_path = before_path
+      @after_path = after_path
+      @all_depths = all_depths
+    end
+
+    def rows
+      before = aggregate(before_path)
+      after = aggregate(after_path)
+
+      (before.keys | after.keys).sort.map do |factory|
+        before_stat = before.fetch(factory, empty_stat)
+        after_stat = after.fetch(factory, empty_stat)
+
+        Row.new(
+          factory: factory,
+          before_count: before_stat.count,
+          after_count: after_stat.count,
+          before_duration_ms: before_stat.duration_ms,
+          after_duration_ms: after_stat.duration_ms
+        )
+      end
+    end
+
+    private
+
+    def aggregate(path)
+      stats = Hash.new { |hash, factory| hash[factory] = FactoryStat.new(0, 0.0) }
+
+      File.foreach(path) do |line|
+        event = Ndjson.parse(line)
+        next unless factory_event?(event)
+
+        stat = stats[event["factory"].to_s]
+        stat.count += 1
+        stat.duration_ms += duration_ms(event)
+      end
+
+      stats
+    end
+
+    def factory_event?(event)
+      return false unless event
+      return false unless event["type"] == "factory_bot.run_factory"
+      return true if all_depths
+
+      event["depth"].to_i.zero?
+    end
+
+    def duration_ms(event)
+      key = all_depths ? "self_duration_ms" : "duration_ms"
+      event[key].to_f
+    end
+
+    def empty_stat
+      FactoryStat.new(0, 0.0)
+    end
+  end
+end

data/lib/rspec_telemetry/formatter.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+require "rspec/core"
+require "rspec/core/formatters/base_formatter"
+
+module RSpecTelemetry
+  class Formatter < RSpec::Core::Formatters::BaseFormatter
+    RSpec::Core::Formatters.register(
+      self,
+      :start,
+      :example_started,
+      :example_finished,
+      :dump_summary,
+      :close
+    )
+
+    def start(_notification)
+      RSpecTelemetry.safely("formatter#start") { RSpecTelemetry.start! }
+    end
+
+    def example_started(notification)
+      RSpecTelemetry.safely("formatter#example_started") do
+        example = notification.example
+        # Runs before before(:each), so FactoryBot calls inside hooks get this example_id.
+        recorder.set_current_example(example.id)
+        next unless config.capture_examples
+
+        recorder.record(
+          "example.started",
+          file_path: example.file_path,
+          line_number: example.metadata[:line_number],
+          full_description: example.full_description
+        )
+      end
+    end
+
+    def example_finished(notification)
+      RSpecTelemetry.safely("formatter#example_finished") do
+        example = notification.example
+        record_example_finished(example) if config.capture_examples
+        recorder.flush
+      ensure
+        # Avoid attributing after(:suite) or later work to the last example.
+        recorder.clear_current_example
+      end
+    end
+
+    def dump_summary(notification)
+      RSpecTelemetry.safely("formatter#dump_summary") do
+        recorder.record(
+          "suite.finished",
+          example_id: nil,
+          duration_ms: (notification.duration * 1000.0).round(3),
+          example_count: notification.example_count,
+          failure_count: notification.failure_count,
+          pending_count: notification.pending_count
+        )
+      end
+    end
+
+    def close(_notification)
+      RSpecTelemetry.safely("formatter#close") { RSpecTelemetry.finish! }
+    end
+
+    private
+
+    def record_example_finished(example)
+      result = example.execution_result
+      exception = result.exception
+
+      recorder.record(
+        "example.finished",
+        file_path: example.file_path,
+        line_number: example.metadata[:line_number],
+        full_description: example.full_description,
+        status: result.status.to_s,
+        duration_ms: (result.run_time.to_f * 1000.0).round(3),
+        exception_class: exception&.class&.name,
+        exception_message: exception&.message
+      )
+    end
+
+    def recorder
+      RSpecTelemetry.recorder
+    end
+
+    def config
+      RSpecTelemetry.config
+    end
+  end
+end

data/lib/rspec_telemetry/ndjson.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+require "json"
+
+module RSpecTelemetry
+  module Ndjson
+    # Trace files are untrusted and may contain a truncated final line after a crash.
+    def self.parse(line)
+      text = scrub(line).strip
+      return nil if text.empty?
+
+      value = JSON.parse(text)
+      value.is_a?(Hash) ? value : nil
+    rescue JSON::ParserError
+      nil
+    end
+
+    # Keep downstream rendering on valid UTF-8 even when the trace has bad bytes.
+    def self.scrub(line)
+      string = line.to_s
+      string.valid_encoding? ? string : string.scrub("?")
+    end
+  end
+end

data/lib/rspec_telemetry/recorder.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+require "time"
+
+module RSpecTelemetry
+  class Recorder
+    # FactoryBot notifications read this to attach themselves to the active example.
+    EXAMPLE_ID = :rspec_telemetry_example_id
+
+    attr_reader :config, :summary
+
+    def initialize(config, writer: nil, summary: nil)
+      @config = config
+      @writer = writer || Writer.new(config.output_path, flush_each: config.flush_each)
+      @summary = summary || Summary.new(config)
+      @started = false
+    end
+
+    def start
+      return if @started || !@config.enabled
+
+      @writer.open
+      @started = true
+    end
+
+    def started?
+      @started
+    end
+
+    def record(type, fields = {})
+      return unless @config.enabled && @started
+
+      event = common_fields(type).merge(fields)
+      @writer.write(event)
+      @summary.add(event)
+      event
+    end
+
+    def flush
+      @writer.flush
+    end
+
+    def finish
+      return unless @started
+
+      SummaryPrinter.print(@summary, @config) if @config.print_summary
+      @writer.close
+      @started = false
+    end
+
+    def common_fields(type)
+      {
+        type: type,
+        timestamp: Time.now.utc.strftime("%Y-%m-%dT%H:%M:%S.%6NZ"),
+        monotonic_time: Process.clock_gettime(Process::CLOCK_MONOTONIC),
+        pid: Process.pid,
+        thread_id: self.class.thread_id,
+        example_id: Thread.current[EXAMPLE_ID]
+      }
+    end
+
+    def set_current_example(id)
+      Thread.current[EXAMPLE_ID] = id
+    end
+
+    def clear_current_example
+      Thread.current[EXAMPLE_ID] = nil
+    end
+
+    def self.thread_id
+      t = Thread.current
+      t.respond_to?(:native_thread_id) && t.native_thread_id ? t.native_thread_id : t.object_id
+    end
+  end
+end