png_conform 0.1.1 → 0.1.2

data/benchmarks/lib/benchmark_runner.rb

@@ -0,0 +1,204 @@
+# frozen_string_literal: true
+
+require_relative "png_conform_runner"
+require_relative "pngcheck_runner"
+require_relative "metrics_collector"
+require_relative "report_generator"
+require "yaml"
+
+# Main orchestrator for benchmark execution.
+#
+# Coordinates file discovery, tool execution, metrics collection,
+# and report generation for comparing PNG validation tools.
+class BenchmarkRunner
+  attr_reader :config, :metrics_collector, :runners
+
+  def initialize(config = {})
+    @config = deep_merge(default_config, config)
+    @metrics_collector = MetricsCollector.new
+    @runners = initialize_runners
+  end
+
+  # Run the complete benchmark suite.
+  #
+  # @return [Hash] Benchmark results
+  def run
+    puts "Initializing benchmark..."
+    validate_tools
+
+    files = discover_files
+    puts "Found #{files.size} PNG files to test"
+
+    return { error: "No files found matching pattern" } if files.empty?
+
+    puts "\nRunning benchmarks..."
+    puts "  Iterations: #{config[:iterations]}"
+    puts "  Warmup runs: #{config[:warmup_runs]}"
+    puts ""
+
+    run_benchmarks(files)
+
+    puts "\nGenerating report..."
+    generate_report
+  end
+
+  private
+
+  def default_config
+    {
+      test_files: {
+        pattern: "spec/fixtures/pngsuite/**/*.png",
+        exclude: [],
+        limit: nil,
+      },
+      iterations: 3,
+      warmup_runs: 1,
+      timeout: 30,
+      tools: {
+        png_conform: { enabled: true, options: {} },
+        pngcheck: { enabled: true, options: {} },
+      },
+      output: {
+        format: "text",
+        file: nil,
+        verbose: true,
+      },
+    }
+  end
+
+  def initialize_runners
+    runners = {}
+
+    if config[:tools][:png_conform][:enabled]
+      runners[:png_conform] = PngConformRunner.new(
+        config[:tools][:png_conform][:options],
+      )
+    end
+
+    if config[:tools][:pngcheck][:enabled]
+      runners[:pngcheck] = PngcheckRunner.new(
+        config[:tools][:pngcheck][:options],
+      )
+    end
+
+    runners
+  end
+
+  def validate_tools
+    puts "Checking tool availability..."
+
+    runners.each do |name, runner|
+      if runner.available?
+        puts "  ✓ #{name} is available"
+      else
+        puts "  ✗ #{name} is NOT available"
+        if name == :pngcheck
+          puts "    Install with: brew install pngcheck (macOS) or apt-get install pngcheck (Linux)"
+        end
+      end
+    end
+
+    puts ""
+  end
+
+  def discover_files
+    pattern = config.dig(:test_files, :pattern)
+    exclude = config.dig(:test_files, :exclude) || []
+    limit = config.dig(:test_files, :limit)
+
+    files = Dir.glob(pattern).select { |f| File.file?(f) }
+
+    # Apply exclusions
+    exclude.each do |pattern|
+      files.reject! { |f| File.fnmatch?(pattern, f) }
+    end
+
+    # Apply limit if specified
+    files = files.first(limit) if limit
+
+    files.sort
+  end
+
+  def run_benchmarks(files)
+    total_iterations = config[:iterations] + config[:warmup_runs]
+    total_runs = files.size * runners.size * total_iterations
+    current_run = 0
+
+    runners.each do |tool_name, runner|
+      next unless runner.available?
+
+      puts "Benchmarking #{tool_name}..."
+
+      files.each do |file|
+        # Warmup runs (not recorded)
+        config[:warmup_runs].times do
+          current_run += 1
+          if config[:output][:verbose]
+            print_progress("Warmup", current_run,
+                           total_runs)
+          end
+          runner.run(file)
+        end
+
+        # Actual benchmark runs (recorded)
+        config[:iterations].times do |_iteration|
+          current_run += 1
+          if config[:output][:verbose]
+            print_progress(tool_name, current_run,
+                           total_runs)
+          end
+
+          metrics = runner.run(file)
+          metrics_collector.record_run(
+            tool_name.to_s,
+            file,
+            metrics,
+          )
+        end
+      end
+
+      puts "" if config[:output][:verbose]
+    end
+  end
+
+  def print_progress(label, current, total)
+    pct = (current.to_f / total * 100).round(1)
+    bar_width = 40
+    filled = (bar_width * current / total).to_i
+    bar = "=" * filled + " " * (bar_width - filled)
+
+    print "\r  [#{bar}] #{pct}% (#{current}/#{total}) #{label}     "
+    $stdout.flush
+  end
+
+  def generate_report
+    report_gen = ReportGenerator.new(metrics_collector, config[:output])
+    report = report_gen.generate(config[:output][:format])
+
+    if config[:output][:file]
+      File.write(config[:output][:file], report)
+      puts "Report saved to: #{config[:output][:file]}"
+    else
+      puts "\n"
+      puts report
+    end
+
+    {
+      summary: metrics_collector.summary,
+      report: report,
+    }
+  end
+
+  # Deep merge two hashes
+  def deep_merge(hash1, hash2)
+    result = hash1.dup
+    hash2.each do |key, value|
+      result[key] = if result[key].is_a?(Hash) && value.is_a?(Hash)
+                      deep_merge(result[key], value)
+                    else
+                      value
+                    end
+    end
+    result
+  end
+end

data/benchmarks/lib/metrics_collector.rb

@@ -0,0 +1,193 @@
+# frozen_string_literal: true
+
+# Collects and analyzes performance metrics from benchmark runs.
+#
+# Tracks execution time, memory usage, throughput, and calculates
+# statistical measures for comparison between tools.
+class MetricsCollector
+  attr_reader :runs
+
+  def initialize
+    @runs = []
+  end
+
+  # Record a single benchmark run.
+  #
+  # @param tool [String] Name of the tool
+  # @param file [String] File path
+  # @param metrics [Hash] Performance metrics
+  def record_run(tool, file, metrics)
+    @runs << {
+      tool: tool,
+      file: file,
+      execution_time: metrics[:execution_time],
+      memory_used: metrics[:memory_used],
+      peak_memory: metrics[:peak_memory],
+      success: metrics[:result][:success],
+      exit_code: metrics[:result][:exit_code],
+      timed_out: metrics[:result][:timed_out],
+      timestamp: Time.now,
+    }
+  end
+
+  # Get all runs for a specific tool.
+  #
+  # @param tool [String] Tool name
+  # @return [Array<Hash>] Runs for the tool
+  def runs_for_tool(tool)
+    @runs.select { |run| run[:tool] == tool }
+  end
+
+  # Calculate statistics for a tool.
+  #
+  # @param tool [String] Tool name
+  # @return [Hash] Statistical measures
+  def calculate_statistics(tool)
+    tool_runs = runs_for_tool(tool)
+    return nil if tool_runs.empty?
+
+    execution_times = tool_runs.map { |r| r[:execution_time] }.compact
+    memory_values = tool_runs.map { |r| r[:peak_memory] }.compact
+
+    {
+      tool: tool,
+      total_runs: tool_runs.size,
+      successful_runs: tool_runs.count { |r| r[:success] },
+      failed_runs: tool_runs.count { |r| !r[:success] },
+      timeouts: tool_runs.count { |r| r[:timed_out] },
+      execution_time: calculate_stats(execution_times),
+      memory: calculate_stats(memory_values),
+      throughput: calculate_throughput(tool_runs),
+    }
+  end
+
+  # Compare two tools.
+  #
+  # @param tool1 [String] First tool name
+  # @param tool2 [String] Second tool name
+  # @return [Hash] Comparison results
+  def compare_tools(tool1, tool2)
+    stats1 = calculate_statistics(tool1)
+    stats2 = calculate_statistics(tool2)
+
+    return nil if stats1.nil? || stats2.nil?
+
+    time1 = stats1[:execution_time][:mean]
+    time2 = stats2[:execution_time][:mean]
+    mem1 = stats1[:memory][:mean]
+    mem2 = stats2[:memory][:mean]
+
+    faster_tool = time1 < time2 ? tool1 : tool2
+    time_diff_pct = ((time1 - time2).abs / [time1, time2].min * 100).round(1)
+    time_multiplier = ([time1, time2].max / [time1, time2].min).round(2)
+
+    memory_efficient = mem1 < mem2 ? tool1 : tool2
+    mem_diff_pct = ((mem1 - mem2).abs / [mem1, mem2].min * 100).round(1)
+    mem_multiplier = ([mem1, mem2].max / [mem1, mem2].min).round(2)
+
+    {
+      tool1: tool1,
+      tool2: tool2,
+      faster_tool: faster_tool,
+      time_difference_percent: time_diff_pct,
+      time_multiplier: time_multiplier,
+      memory_efficient_tool: memory_efficient,
+      memory_difference_percent: mem_diff_pct,
+      memory_multiplier: mem_multiplier,
+      stats: {
+        tool1 => stats1,
+        tool2 => stats2,
+      },
+    }
+  end
+
+  # Export raw data for external analysis.
+  #
+  # @return [Array<Hash>] All run data
+  def export_raw_data
+    @runs
+  end
+
+  # Get summary statistics across all tools.
+  #
+  # @return [Hash] Summary data
+  def summary
+    tools = @runs.map { |r| r[:tool] }.uniq
+
+    {
+      total_runs: @runs.size,
+      tools: tools,
+      files_tested: @runs.map { |r| r[:file] }.uniq.size,
+      tool_statistics: tools.map { |tool| calculate_statistics(tool) }.compact,
+    }
+  end
+
+  private
+
+  # Calculate statistical measures for a dataset.
+  #
+  # @param values [Array<Numeric>] Data values
+  # @return [Hash] Statistical measures
+  def calculate_stats(values)
+    return nil if values.empty?
+
+    sorted = values.sort
+    size = values.size
+
+    {
+      mean: (values.sum / size.to_f).round(3),
+      median: calculate_median(sorted),
+      std_dev: calculate_std_dev(values).round(3),
+      min: sorted.first.round(3),
+      max: sorted.last.round(3),
+      count: size,
+    }
+  end
+
+  # Calculate median value.
+  #
+  # @param sorted_values [Array<Numeric>] Sorted array
+  # @return [Float] Median value
+  def calculate_median(sorted_values)
+    size = sorted_values.size
+    mid = size / 2
+
+    if size.even?
+      ((sorted_values[mid - 1] + sorted_values[mid]) / 2.0).round(3)
+    else
+      sorted_values[mid].round(3)
+    end
+  end
+
+  # Calculate standard deviation.
+  #
+  # @param values [Array<Numeric>] Data values
+  # @return [Float] Standard deviation
+  def calculate_std_dev(values)
+    return 0.0 if values.size <= 1
+
+    mean = values.sum / values.size.to_f
+    variance = values.sum { |v| (v - mean)**2 } / values.size.to_f
+    Math.sqrt(variance)
+  end
+
+  # Calculate throughput metrics.
+  #
+  # @param runs [Array<Hash>] Run data
+  # @return [Hash] Throughput metrics
+  def calculate_throughput(runs)
+    successful = runs.select { |r| r[:success] }
+    return nil if successful.empty?
+
+    total_time = # Convert to seconds
+      successful.sum do |r|
+        r[:execution_time]
+      end / 1000.0
+    files_count = successful.size
+
+    {
+      files_per_second: (files_count / total_time).round(2),
+      avg_time_per_file: (total_time / files_count * 1000).round(3), # milliseconds
+    }
+  end
+end

data/benchmarks/lib/png_conform_runner.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+require_relative "tool_runner"
+
+# Runner for the png_conform validation tool.
+#
+# Executes png_conform CLI and collects performance metrics.
+class PngConformRunner < ToolRunner
+  def initialize(options = {})
+    # Use the exe/png_conform in the project root
+    command = File.expand_path("../../exe/png_conform", __dir__)
+    super("png_conform", command, options)
+  end
+
+  # Run png_conform on a single file.
+  #
+  # @param file_path [String] Path to the PNG file
+  # @return [Hash] Performance metrics and validation results
+  def run(file_path)
+    return error_result("File not found: #{file_path}") unless File.exist?(file_path)
+
+    measure_performance do
+      cli_options = build_cli_options
+      cmd = "#{command} check #{cli_options} #{file_path}"
+      result = execute_command(cmd)
+
+      {
+        file: file_path,
+        tool: name,
+        success: result[:exit_code].zero?,
+        exit_code: result[:exit_code],
+        stdout: result[:stdout],
+        stderr: result[:stderr],
+        timed_out: result[:timed_out],
+      }
+    end
+  end
+
+  private
+
+  def build_cli_options
+    cli_opts = options[:cli_options] || []
+
+    # Add --quiet by default to reduce output parsing overhead
+    cli_opts << "--quiet" unless cli_opts.include?("--quiet") ||
+      cli_opts.include?("-q") ||
+      cli_opts.include?("--verbose")
+
+    cli_opts.join(" ")
+  end
+
+  def error_result(message)
+    {
+      execution_time: 0,
+      memory_used: 0,
+      peak_memory: 0,
+      result: {
+        file: nil,
+        tool: name,
+        success: false,
+        exit_code: -1,
+        stdout: "",
+        stderr: message,
+        timed_out: false,
+      },
+    }
+  end
+end

data/benchmarks/lib/pngcheck_runner.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+require_relative "tool_runner"
+
+# Runner for the pngcheck validation tool.
+#
+# Executes the system pngcheck binary and collects performance metrics.
+class PngcheckRunner < ToolRunner
+  def initialize(options = {})
+    super("pngcheck", "pngcheck", options)
+  end
+
+  # Run pngcheck on a single file.
+  #
+  # @param file_path [String] Path to the PNG file
+  # @return [Hash] Performance metrics and validation results
+  def run(file_path)
+    return error_result("File not found: #{file_path}") unless File.exist?(file_path)
+    return error_result("pngcheck not found in PATH") unless available?
+
+    measure_performance do
+      cli_options = build_cli_options
+      cmd = "pngcheck #{cli_options} #{file_path}"
+      result = execute_command(cmd)
+
+      {
+        file: file_path,
+        tool: name,
+        success: result[:exit_code].zero?,
+        exit_code: result[:exit_code],
+        stdout: result[:stdout],
+        stderr: result[:stderr],
+        timed_out: result[:timed_out],
+      }
+    end
+  end
+
+  private
+
+  def build_cli_options
+    cli_opts = options[:cli_options] || []
+
+    # Add -q (quiet) by default for consistent comparison
+    cli_opts << "-q" unless cli_opts.include?("-q") ||
+      cli_opts.include?("-v") ||
+      cli_opts.include?("-vv")
+
+    cli_opts.join(" ")
+  end
+
+  def error_result(message)
+    {
+      execution_time: 0,
+      memory_used: 0,
+      peak_memory: 0,
+      result: {
+        file: nil,
+        tool: name,
+        success: false,
+        exit_code: -1,
+        stdout: "",
+        stderr: message,
+        timed_out: false,
+      },
+    }
+  end
+end