canon 0.2.11 → 0.2.12

data/lib/tasks/performance_comparator.rb

@@ -1,100 +1,167 @@
 # frozen_string_literal: true
 
-require_relative "performance_helpers"
+require "json"
+require "open3"
+require "fileutils"
+require "table_tennis"
 
+# Compares performance between the current branch and a base branch (default:
+# main) by running the same benchmark suite in two separate Ruby processes —
+# one per branch. Each process loads its own Canon implementation from disk,
+# fully isolated from the other, and emits a JSON result. The comparator then
+# diffs the two JSON payloads and reports regressions.
+#
+# Running each branch in its own process is required because Canon uses Ruby
+# autoload extensively; loading both branches' code into a single process
+# causes constant resolution and LOAD_PATH conflicts.
 class PerformanceComparator
   REPO_ROOT = File.expand_path(File.join(__dir__, "..", ".."))
-  DEFAULT_RUN_TIME = 10 # seconds
-  DEFAULT_THRESHOLD = 0.10 # 10% (more lenient for complex operations)
+  DEFAULT_RUN_TIME = Integer(ENV.fetch("CANON_PERF_RUN_TIME", "10"))
+  DEFAULT_THRESHOLD = 0.10 # 10%
   DEFAULT_BASE = "main"
   TMP_PERF_DIR = File.join(REPO_ROOT, "tmp", "performance")
-  BENCH_SCRIPT = File.join(TMP_PERF_DIR, "benchmark_runner.rb")
-
-  # Benchmark categories - run specific subsets
-  BENCHMARK_CATEGORIES = {
-    xml_parsing: %w[xml_parse_dom_simple xml_parse_sax_simple
-                    xml_parse_dom_large xml_parse_sax_large],
-    html_parsing: %w[html_parse_simple html_parse_complex],
-    xml_comparison: %w[xml_compare_identical xml_compare_similar
-                       xml_compare_different],
-    html_comparison: %w[html_compare_identical html_compare_similar
-                        html_compare_different],
-    formatting: %w[xml_c14n_format json_format yaml_format],
-  }.freeze
+  REPORT_SCRIPT = File.expand_path("performance_report.rb", __dir__)
+
+  RED    = "\e[31m"
+  GREEN  = "\e[32m"
+  YELLOW = "\e[33m"
+  CYAN   = "\e[36m"
+  BOLD   = "\e[1m"
+  DIM    = "\e[2m"
+  CLEAR  = "\e[0m"
 
   def run
-    setup_environment
-    run_benchmarks_comparison
+    clone_base
+    current = run_report(REPO_ROOT, "current")
+    base = run_report(base_clone_dir, "base (#{DEFAULT_BASE})")
+    print_report(current, base)
+    exit(1) if regressions?(current, base)
   ensure
     cleanup
   end
 
   private
 
-  def setup_environment
-    Dir.chdir(REPO_ROOT)
+  def clone_base
+    FileUtils.rm_rf(TMP_PERF_DIR)
     FileUtils.mkdir_p(TMP_PERF_DIR)
-    FileUtils.cp(File.join(REPO_ROOT, "lib", "tasks", "benchmark_runner.rb"),
-                 BENCH_SCRIPT)
 
-    PerformanceHelpers.load_into_namespace(PerformanceHelpers::Current,
-                                           BENCH_SCRIPT)
-    PerformanceHelpers.clone_base_repo(DEFAULT_BASE, TMP_PERF_DIR, BENCH_SCRIPT)
+    puts "#{DIM}Cloning base #{DEFAULT_BASE}...#{CLEAR}"
+    repo_url, = exec("git config --get remote.origin.url")
+    out, err, status = exec(
+      "git clone --branch #{DEFAULT_BASE} --single-branch #{repo_url.strip} #{base_clone_dir}",
+    )
+    return if status.success?
+
+    raise "git clone failed: #{err}\n#{out}"
   end
 
-  def run_benchmarks_comparison
-    all_current = {}
-    all_base = {}
+  def run_report(working_dir, label)
+    puts "#{DIM}Running benchmarks for #{label}...#{CLEAR}"
+    env = {
+      "CANON_PERF_RUN_TIME" => DEFAULT_RUN_TIME.to_s,
+      "BUNDLE_GEMFILE" => File.join(working_dir, "Gemfile"),
+    }
 
-    puts PerformanceHelpers::Term.header("Performance Comparison", color: PerformanceHelpers::CYAN)
-    puts
-    puts "  #{PerformanceHelpers::DIM}Comparing#{PerformanceHelpers::CLEAR}:"
-    puts "  #{PerformanceHelpers::CYAN}  Current#{PerformanceHelpers::CLEAR}: #{PerformanceHelpers.current_branch}"
-    puts "  #{PerformanceHelpers::CYAN}  Base#{PerformanceHelpers::CLEAR}: #{DEFAULT_BASE}"
-    puts "  #{PerformanceHelpers::CYAN}  Threshold#{PerformanceHelpers::CLEAR}: #{(DEFAULT_THRESHOLD * 100).round(0)}% regression allowed"
-    puts
+    script_copy = File.join(working_dir, "tmp", "performance_report.rb")
+    FileUtils.mkdir_p(File.dirname(script_copy))
+    FileUtils.cp(REPORT_SCRIPT, script_copy)
 
-    # Run all benchmarks
-    base_runner = PerformanceHelpers::Base::BenchmarkRunner.new(
-      run_time: DEFAULT_RUN_TIME,
-    )
-    current_runner = PerformanceHelpers::Current::BenchmarkRunner.new(
-      run_time: DEFAULT_RUN_TIME,
-    )
+    stdout, stderr, status = Open3.capture3(env, "bundle", "exec", "ruby",
+                                            script_copy,
+                                            chdir: working_dir)
+    unless status.success?
+      raise "Benchmark failed for #{label}: #{stderr}\n#{stdout}"
+    end
 
-    PerformanceHelpers.run_benchmarks(
-      base_runner,
-      current_runner,
-      DEFAULT_THRESHOLD,
-      all_base,
-      all_current,
-    )
+    JSON.parse(stdout)
+  rescue JSON::ParserError => e
+    raise "Invalid JSON from #{label}: #{e.message}"
+  end
 
-    summary = PerformanceHelpers.summary_report(
-      all_current,
-      all_base,
-      DEFAULT_BASE,
-      DEFAULT_RUN_TIME,
-      DEFAULT_THRESHOLD,
-    )
+  def regressions?(current, base)
+    threshold = DEFAULT_THRESHOLD
+    current.fetch("benchmarks").any? do |label, metrics|
+      base_metrics = base.fetch("benchmarks")[label]
+      next false unless base_metrics
+
+      change = change_fraction(metrics, base_metrics)
+      change && change < -threshold
+    end
+  end
+
+  def change_fraction(curr, base)
+    base_ips = base.fetch("lower").to_f
+    curr_ips = curr.fetch("upper").to_f
+    return nil if base_ips.zero?
 
-    handle_results(summary)
+    (curr_ips - base_ips) / base_ips
   end
 
-  def handle_results(summary)
+  def print_report(current, base)
+    threshold = DEFAULT_THRESHOLD
+    rows = current.fetch("benchmarks").map do |label, metrics|
+      base_metrics = base.fetch("benchmarks")[label]
+      change = change_fraction(metrics, base_metrics)
+      status = if base_metrics.nil?
+                 "NEW"
+               elsif change < -threshold
+                 "REGRESSED"
+               else
+                 "OK"
+               end
+      {
+        benchmark: label,
+        base_ips: base_metrics&.fetch("lower")&.round(1),
+        curr_ips: metrics.fetch("upper").round(1),
+        change: change ? format("%+0.1f%%", change * 100) : "N/A",
+        status: status,
+      }
+    end
+
+    table = TableTennis.new(rows,
+                            title: "Performance Comparison",
+                            theme: :dark,
+                            headers: {
+                              benchmark: "Benchmark",
+                              base_ips: "Base IPS",
+                              curr_ips: "Curr IPS",
+                              change: "Change",
+                              status: "Status",
+                            })
+    table.render
     puts
-    if summary[:regressions].any?
-      puts "  #{PerformanceHelpers::RED}#{PerformanceHelpers::BOLD}❌ PERFORMANCE REGRESSIONS DETECTED#{PerformanceHelpers::CLEAR}"
-      puts "  #{PerformanceHelpers::RED}#{summary[:regressions].length} benchmark(s) regressed beyond threshold#{PerformanceHelpers::CLEAR}"
-      puts
-      exit(1)
+
+    print_summary(rows, threshold)
+  end
+
+  def print_summary(rows, threshold)
+    regressions = rows.select { |r| r[:status] == "REGRESSED" }
+    new_benchmarks = rows.select { |r| r[:status] == "NEW" }
+
+    if regressions.empty?
+      puts "#{GREEN}#{BOLD}✅ ALL BENCHMARKS PASSED#{CLEAR}"
     else
-      puts "  #{PerformanceHelpers::GREEN}#{PerformanceHelpers::BOLD}✅ ALL BENCHMARKS PASSED#{PerformanceHelpers::CLEAR}"
-      puts
+      puts "#{RED}#{BOLD}❌ PERFORMANCE REGRESSIONS DETECTED#{CLEAR}"
+      puts "#{RED}#{regressions.length} benchmark(s) regressed " \
+           "beyond #{(threshold * 100).round(0)}% threshold#{CLEAR}"
     end
+
+    return if new_benchmarks.empty?
+
+    puts "#{YELLOW}🆕 New benchmarks (not in base):#{CLEAR}"
+    new_benchmarks.each { |r| puts "  • #{r[:benchmark]}" }
+  end
+
+  def base_clone_dir
+    @base_clone_dir ||= File.join(TMP_PERF_DIR, "base-#{DEFAULT_BASE}")
   end
 
   def cleanup
     FileUtils.rm_rf(TMP_PERF_DIR)
   end
+
+  def exec(cmd)
+    Open3.capture3(cmd)
+  end
 end

data/lib/tasks/performance_helpers.rb

@@ -1,13 +1,10 @@
 # frozen_string_literal: true
 
-require "json"
-require "open3"
-require "tmpdir"
-require "fileutils"
-require "table_tennis" unless RUBY_ENGINE == "opal"
-
+# Performance benchmark output helpers shared between
+# +benchmark_runner.rb+ (interactive) and +performance_comparator.rb+.
+# Kept as a small module so the rake tasks below can pull in the color
+# constants without pulling in the comparator (which would clone main).
 module PerformanceHelpers
-  # ANSI color codes for terminal output
   CLEAR   = "\e[0m"
   BOLD    = "\e[1m"
   DIM     = "\e[2m"
@@ -18,246 +15,4 @@ module PerformanceHelpers
   GRAY    = "\e[90m"
   WHITE   = "\e[37m"
   MAGENTA = "\e[35m"
-
-  # Terminal formatting helpers
-  module Term
-    extend self
-
-    HL = "─"
-    VL = "│"
-    TL = "┌"
-    TR = "┐"
-    BL = "└"
-    BR = "┘"
-
-    def header(title, color: PerformanceHelpers::CYAN)
-      width = 78
-      line = HL * width
-      puts
-      puts "#{color}#{TL}#{line}#{TR}#{CLEAR}"
-      puts "#{color}#{VL}#{CLEAR}  #{BOLD}#{color}#{title}#{CLEAR}#{' ' * (width - title.length - 4)}#{color}#{VL}#{CLEAR}"
-      puts "#{color}#{BL}#{line}#{BR}#{CLEAR}"
-    end
-
-    def sep(char: HL, width: 78)
-      puts "#{DIM}#{char * width}#{CLEAR}"
-    end
-  end
-
-  module Base
-  end
-
-  module Current
-  end
-
-  class << self
-    def load_into_namespace(module_obj, file_path)
-      content = File.read(file_path, encoding: "utf-8")
-      module_obj.module_eval(content, file_path)
-    end
-
-    def ruby_exec(cmd, env: {})
-      Open3.capture3(env, cmd)
-    end
-
-    def current_branch
-      stdout, = ruby_exec("git rev-parse --abbrev-ref HEAD")
-      stdout.strip
-    end
-
-    # Clone base branch into a temp dir and return its path
-    def clone_base_repo(base, performance_dir, script)
-      puts "#{DIM}Cloning base #{base}...#{CLEAR}"
-      safe_ref = base.gsub(/[^0-9A-Za-z._-]/, "-")
-      clone_dir = File.join(performance_dir, "base-#{safe_ref}")
-      FileUtils.rm_rf(clone_dir)
-
-      repo_url, = ruby_exec("git config --get remote.origin.url")
-      repo_url = repo_url.strip
-
-      stdout, stderr, status = ruby_exec("git clone --branch #{safe_ref} --single-branch #{repo_url} #{clone_dir}")
-      raise "git clone failed: #{stderr}\n#{stdout}" unless status.success?
-
-      Dir.chdir(clone_dir) do
-        stdout, stderr, status = ruby_exec("bundle install --quiet")
-        raise "bundle install failed: #{stderr}\n#{stdout}" unless status.success?
-
-        bench_copy_dir = File.join(clone_dir, "tmp", "performance")
-        FileUtils.mkdir_p(bench_copy_dir)
-        bench_copy = File.join(bench_copy_dir, "benchmark_runner.rb")
-        File.write(bench_copy, File.read(script, encoding: "utf-8"))
-        load_into_namespace(Base, bench_copy)
-      end
-    end
-
-    def run_benchmarks(base_runner, current_runner, threshold, all_base,
-                       all_current)
-      base_results = base_runner.run_benchmarks
-      curr_results = current_runner.run_benchmarks
-
-      all_base.merge!(base_results)
-      all_current.merge!(curr_results)
-
-      # Collect comparison results for TableTennis table
-      comparison_rows = []
-
-      curr_results.each do |label, result|
-        base_result = base_results[label]
-        cmp = compare_metrics(label, result, base_result, threshold)
-        comparison_rows << cmp
-      end
-
-      print_comparison_table(comparison_rows, threshold)
-    end
-
-    def print_comparison_table(comparison_rows, threshold)
-      rows = comparison_rows.map do |cmp|
-        {
-          benchmark: cmp[:label],
-          base_ips: cmp[:base_ips]&.round(1),
-          curr_ips: cmp[:curr_ips]&.round(1),
-          change: cmp[:change] ? "#{(cmp[:change] * 100).round(1)}%" : "N/A",
-          status: if cmp[:base_ips].nil?
-                    "NEW"
-                  elsif cmp[:change] < -threshold
-                    "REGRESSED"
-                  else
-                    "OK"
-                  end,
-        }
-      end
-
-      return if rows.empty?
-
-      table = TableTennis.new(rows,
-                              title: "Performance Comparison",
-                              theme: :dark,
-                              headers: {
-                                benchmark: "Benchmark",
-                                base_ips: "Base IPS",
-                                curr_ips: "Curr IPS",
-                                change: "Change",
-                                status: "Status",
-                              })
-      table.render
-      puts
-    end
-
-    def compare_metrics(label, curr, base, threshold)
-      # Skip comparison if base result is missing (e.g., new benchmark)
-      unless base
-        return { label: label, base_ips: nil, curr_ips: nil, change: nil,
-                 regressed: false }
-      end
-
-      base_ips = base.fetch(:lower)
-      curr_ips = curr.fetch(:upper)
-      change = (curr_ips - base_ips) / base_ips.to_f
-
-      {
-        label: label,
-        base_ips: base_ips,
-        curr_ips: curr_ips,
-        change: change,
-        regressed: change < -threshold,
-      }
-    end
-
-    def summary_report(current_results, base_results, base, run_time, threshold)
-      summary = {
-        run_time: run_time,
-        threshold: threshold,
-        branch: current_branch,
-        base: base,
-        regressions: [],
-        new_benchmarks: [],
-      }
-
-      current_results.each do |label, metrics|
-        base_result = base_results[label]
-        cmp = compare_metrics(label, metrics, base_result, threshold)
-
-        # Track new benchmarks that don't exist in base
-        if base_result.nil?
-          summary[:new_benchmarks] << label
-          next
-        end
-
-        next unless cmp[:regressed]
-
-        summary[:regressions] << {
-          label: label,
-          base_ips: cmp[:base_ips],
-          curr_ips: cmp[:curr_ips],
-          delta_fraction: cmp[:change],
-        }
-      end
-
-      log_regressions(summary[:regressions], threshold)
-      log_new_benchmarks(summary[:new_benchmarks])
-      summary
-    end
-
-    def log_new_benchmarks(new_benchmarks)
-      return if new_benchmarks.empty?
-
-      puts
-      puts "#{YELLOW}🆕 New benchmarks (not in base branch):#{CLEAR}"
-      new_benchmarks.each do |label|
-        puts "  • #{label}"
-      end
-    end
-
-    def log_regressions(regressions, threshold)
-      return if regressions.empty?
-
-      puts
-      puts "#{RED}⚠️  Performance Regressions Detected#{CLEAR}"
-      puts "#{RED}   (< -#{(threshold * 100).round(2)}% IPS)#{CLEAR}"
-      puts
-      regressions.each do |regression|
-        delta = regression[:delta_fraction]
-        base_ips = regression[:base_ips]
-        curr_ips = regression[:curr_ips]
-
-        delta_str = delta ? format("%+0.2f%%", delta * 100) : "N/A"
-        base_str = base_ips ? format("%.2f", base_ips) : "N/A"
-        curr_str = curr_ips ? format("%.2f", curr_ips) : "N/A"
-
-        puts "  #{BOLD}#{regression[:label]}#{CLEAR}"
-        puts "    #{GRAY}base: #{base_str} IPS#{CLEAR}"
-        puts "    #{RED}curr: #{curr_str} IPS#{CLEAR}"
-        puts "    #{RED}change: #{delta_str}#{CLEAR}"
-        puts
-      end
-    end
-
-    private
-
-    def print_realtime_comparison(label, curr_metrics, base_metrics, threshold)
-      # Handle new benchmarks that don't exist in base
-      if base_metrics.nil?
-        curr_ips = (curr_metrics[:lower] + curr_metrics[:upper]) / 2.0
-        puts "#{format('%-30s',
-                       label)}: #{GREEN}NEW#{CLEAR} (current: #{format('%.2f',
-                                                                       curr_ips)} IPS) [N/A]\n\n"
-        return
-      end
-
-      curr_ips = curr_metrics[:upper]
-      base_ips = base_metrics[:lower]
-      return unless curr_ips && base_ips
-
-      change = (curr_ips - base_ips) / base_ips.to_f
-      color = change < -threshold ? RED : GREEN
-      status = change < -threshold ? "⚠️ REGRESSED" : "✅ OK"
-      delta_str = format("%+0.2f%%", change * 100)
-      base_str = format("%.2f", base_ips)
-      curr_str = format("%.2f", curr_ips)
-
-      puts "#{format('%-30s',
-                     label)}: #{GRAY}#{base_str}#{CLEAR} → #{color}#{curr_str}#{CLEAR} IPS " \
-           "(change: #{color}#{delta_str}#{CLEAR}) [#{color}#{status}#{CLEAR}]\n\n"
-    end
-  end
 end