perfgate 0.1.0

data/exe/perfgate

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "baseline"
+require "baseline/cli"
+
+Baseline::CLI.start(ARGV)

data/lib/perfgate/cli/compare_command.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+require "optparse"
+require_relative "../config"
+require_relative "../comparison/engine"
+require_relative "../policy/engine"
+require_relative "../storage/filesystem"
+require_relative "../report/console"
+require_relative "../report/markdown"
+
+module Perfgate
+  class CLI
+    # Implements `baseline compare` (spec section 10.3): loads two
+    # already-produced result bundles, runs them through the comparison
+    # engine and policy engine, saves the comparison document, prints a
+    # console (or Markdown, with --format markdown) report, and exits
+    # with the CI exit code from spec section 17.
+    class CompareCommand
+      def initialize(argv)
+        @argv = argv.dup
+        @options = { config: "perfgate.yml" }
+      end
+
+      def call
+        parse_options!
+        require_bundle_options!
+        config = load_configuration
+
+        comparison_result = compare(config)
+        policy_result = Policy::Engine.evaluate(comparison_result: comparison_result, config: config)
+
+        comparison_path = save(config, comparison_result)
+        report(comparison_result, policy_result, comparison_path)
+        policy_result.fetch("exit_code")
+      end
+
+      private
+
+      def compare(config)
+        baseline_run = load_bundle(@options[:baseline])
+        candidate_run = load_bundle(@options[:candidate])
+
+        Comparison::Engine.compare(baseline_run: baseline_run, candidate_run: candidate_run, config: config)
+      end
+
+      def require_bundle_options!
+        return if @options[:baseline] && @options[:candidate]
+
+        raise Perfgate::ConfigurationError, "baseline compare requires both --baseline and --candidate"
+      end
+
+      def load_configuration
+        config = Perfgate::Config.load(@options[:config])
+        Perfgate.configuration = config
+        config
+      end
+
+      def load_bundle(reference)
+        Storage::Filesystem.new(root: File.dirname(reference)).load_run(reference)
+      end
+
+      def save(config, comparison_result)
+        Storage::Filesystem.new(root: @options[:output] || config.storage_path).save_comparison(comparison_result)
+      end
+
+      def parse_options!
+        option_parser.parse!(@argv)
+      end
+
+      def option_parser
+        OptionParser.new do |opts|
+          opts.on("--baseline PATH") { |v| @options[:baseline] = v }
+          opts.on("--candidate PATH") { |v| @options[:candidate] = v }
+          opts.on("--config PATH") { |v| @options[:config] = v }
+          opts.on("--output PATH") { |v| @options[:output] = v }
+          opts.on("--format FORMAT") { |v| @options[:format] = v }
+        end
+      end
+
+      def report(comparison_result, policy_result, comparison_path)
+        if @options[:format] == "markdown"
+          puts Report::Markdown.render(comparison_result: comparison_result, policy_result: policy_result,
+                                       comparison_path: comparison_path)
+        else
+          puts Report::Console.render(comparison_result: comparison_result, policy_result: policy_result)
+          puts "Machine-readable result: #{comparison_path}"
+        end
+      end
+    end
+  end
+end

data/lib/perfgate/cli/run_command.rb

@@ -0,0 +1,123 @@
+# frozen_string_literal: true
+
+require "optparse"
+require_relative "../config"
+require_relative "../execution/process_runner"
+require_relative "../serialization/run_result"
+require_relative "../storage/filesystem"
+require_relative "run_comparison_reporter"
+
+module Perfgate
+  class CLI
+    # Implements `baseline run` (spec section 10.2): discovers workloads
+    # via the RSpec integration, executes each workload's warmup+samples
+    # in an isolated child process, and writes a filesystem result
+    # bundle. With --compare PATH, immediately compares the fresh run
+    # against a reference bundle and reports/exits like `baseline
+    # compare` would -- this is the single combined step the spec's
+    # GitHub Actions example (section 19.1) invokes. --format markdown
+    # additionally writes a summary.md into the output directory for a
+    # GitHub job summary.
+    class RunCommand
+      def initialize(argv)
+        @argv = argv.dup
+        @options = { config: "perfgate.yml" }
+      end
+
+      def call
+        parse_options!
+        config = load_configuration
+        run_result = execute(config)
+        run_dir = save(config, run_result)
+
+        @options[:compare] ? compare_and_report(config, run_result, run_dir) : run_only_report(run_result, run_dir)
+      end
+
+      private
+
+      def load_configuration
+        config = Perfgate::Config.load(@options[:config])
+        Perfgate.configuration = config
+        config
+      end
+
+      def execute(config)
+        require "perfgate/rspec"
+        load_spec_files
+        Perfgate::RSpec::Discovery.call
+
+        results = Perfgate.registry.map { |workload| Execution::ProcessRunner.new(workload).call }
+        Serialization::RunResult.build(results, config: config)
+      end
+
+      def save(config, run_result)
+        Storage::Filesystem.new(root: @options[:output] || config.storage_path).save_run(run_result)
+      end
+
+      def parse_options!
+        option_parser.parse!(@argv)
+      end
+
+      def option_parser # rubocop:disable Metrics/AbcSize
+        OptionParser.new do |opts|
+          opts.on("--config PATH") { |v| @options[:config] = v }
+          opts.on("--output PATH") { |v| @options[:output] = v }
+          opts.on("--only PATTERN") { |v| @options[:only] = v }
+          opts.on("--format FORMAT") { |v| @options[:format] = v }
+          opts.on("--fail-on MODE") { |v| @options[:fail_on] = v }
+          opts.on("--seed N", Integer) { |v| @options[:seed] = v }
+          opts.on("--profile NAME") { |v| @options[:profile] = v }
+          opts.on("--compare PATH") { |v| @options[:compare] = v }
+        end
+      end
+
+      def load_spec_files
+        options = ::RSpec::Core::ConfigurationOptions.new(spec_paths)
+        options.configure(::RSpec.configuration)
+        ::RSpec.configuration.load_spec_files
+      end
+
+      def spec_paths
+        @argv.empty? ? ["spec"] : @argv
+      end
+
+      def run_only_report(run_result, run_dir)
+        puts "baseline run: #{run_result["workloads"].size} workload(s) -> #{run_dir}"
+        run_result["workloads"].each { |workload| report_workload(workload) }
+        write_run_only_summary(run_result, run_dir)
+        exit_code(run_result)
+      end
+
+      def report_workload(workload)
+        summary = workload.dig("summary", "duration_ns")
+        if workload["status"] == "completed" && summary
+          median_ms = summary["median"] / 1_000_000.0
+          puts format("  %<id>-40s median=%<median>.2fms (n=%<count>d)",
+                      id: workload["id"], median: median_ms, count: workload["samples"].size)
+        else
+          puts "  #{workload["id"]}: #{workload["status"]} (#{workload["error"]})"
+        end
+      end
+
+      def exit_code(run_result)
+        run_result["workloads"].any? { |w| w["status"] == "error" } ? 1 : 0
+      end
+
+      def write_run_only_summary(run_result, run_dir)
+        return unless @options[:format] == "markdown"
+
+        rows = run_result["workloads"].map { |w| "| #{w["id"]} | #{w["status"]} |" }
+        content = (["## Baseline Run", "", "No reference bundle was compared against.", "",
+                    "| Workload | Status |", "|---|---|"] + rows).join("\n")
+        File.write(File.join(run_dir, "summary.md"), content)
+      end
+
+      def compare_and_report(config, run_result, run_dir)
+        reporter = RunComparisonReporter.new(reference_path: @options[:compare],
+                                             output_root: @options[:output] || config.storage_path,
+                                             format: @options[:format])
+        reporter.call(config: config, run_result: run_result, run_dir: run_dir)
+      end
+    end
+  end
+end

data/lib/perfgate/cli/run_comparison_reporter.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+require_relative "../comparison/engine"
+require_relative "../policy/engine"
+require_relative "../report/console"
+require_relative "../report/markdown"
+require_relative "../storage/filesystem"
+
+module Perfgate
+  class CLI
+    # Handles `baseline run`'s optional --compare PATH step: loads the
+    # reference bundle (tolerating a missing one, since the spec's
+    # GitHub Actions example downloads it with continue-on-error),
+    # compares it against the freshly-run result, saves and reports the
+    # comparison, and returns the CI exit code. Split out of RunCommand
+    # to keep both classes under RuboCop's length limits.
+    class RunComparisonReporter
+      def initialize(reference_path:, output_root:, format:)
+        @reference_path = reference_path
+        @output_root = output_root
+        @format = format
+      end
+
+      def call(config:, run_result:, run_dir:)
+        reference_run = load_reference
+        return missing_baseline_report(config, run_dir) unless reference_run
+
+        comparison_result = Comparison::Engine.compare(baseline_run: reference_run, candidate_run: run_result,
+                                                       config: config)
+        report_comparison(config, comparison_result, run_dir)
+      end
+
+      private
+
+      def load_reference
+        Storage::Filesystem.new(root: File.dirname(@reference_path)).load_run(@reference_path)
+      rescue Perfgate::ResultBundleError
+        nil
+      end
+
+      def missing_baseline_report(config, run_dir)
+        policy_result = Policy::Engine.evaluate_missing_baseline(config: config)
+        puts "baseline run: no baseline found at #{@reference_path} -> #{policy_result["status"]}"
+        write_summary(run_dir, "## Baseline Run\n\nNo baseline was found at `#{@reference_path}`.") if markdown?
+        policy_result.fetch("exit_code")
+      end
+
+      def report_comparison(config, comparison_result, run_dir)
+        policy_result = Policy::Engine.evaluate(comparison_result: comparison_result, config: config)
+        comparison_path = Storage::Filesystem.new(root: @output_root).save_comparison(comparison_result)
+
+        render(comparison_result, policy_result, comparison_path, run_dir)
+        policy_result.fetch("exit_code")
+      end
+
+      def render(comparison_result, policy_result, comparison_path, run_dir)
+        markdown = Report::Markdown.render(comparison_result: comparison_result, policy_result: policy_result,
+                                           comparison_path: comparison_path)
+        if markdown?
+          puts markdown
+          write_summary(run_dir, markdown)
+        else
+          puts Report::Console.render(comparison_result: comparison_result, policy_result: policy_result)
+          puts "Machine-readable result: #{comparison_path}"
+        end
+      end
+
+      def markdown?
+        @format == "markdown"
+      end
+
+      def write_summary(run_dir, content)
+        File.write(File.join(run_dir, "summary.md"), content)
+      end
+    end
+  end
+end

data/lib/perfgate/cli.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module Perfgate
+  # Entry point for the `baseline` executable. `run` and `compare` are
+  # implemented (Milestones 1-4); `init`, `report`, `doctor`, and
+  # `schema` are planned for later milestones (spec section 10).
+  class CLI
+    def self.start(argv)
+      new(argv).run
+    end
+
+    def initialize(argv)
+      @argv = argv
+    end
+
+    def run
+      dispatch(*@argv)
+    rescue Perfgate::ConfigurationError => e
+      warn "baseline: #{e.message}"
+      2
+    rescue Perfgate::ResultBundleError, Perfgate::WorkloadError => e
+      warn "baseline: #{e.message}"
+      3
+    rescue Perfgate::Error => e
+      warn "baseline: #{e.message}"
+      1
+    end
+
+    private
+
+    def dispatch(command = nil, *rest)
+      case command
+      when "run" then run_run_command(rest)
+      when "compare" then run_compare_command(rest)
+      when nil then usage
+      else unknown_command(command)
+      end
+    end
+
+    def run_run_command(rest)
+      require_relative "cli/run_command"
+      RunCommand.new(rest).call
+    end
+
+    def run_compare_command(rest)
+      require_relative "cli/compare_command"
+      CompareCommand.new(rest).call
+    end
+
+    def usage
+      warn "usage: baseline <command> [options]"
+      1
+    end
+
+    def unknown_command(command)
+      warn "baseline: unknown or not-yet-implemented command #{command.inspect}"
+      1
+    end
+  end
+end

data/lib/perfgate/comparison/deterministic_metric_decision.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+require_relative "metric_change"
+
+module Perfgate
+  module Comparison
+    # Deterministic decision path used for count-like metrics (currently
+    # only sql_count): no statistical test is applied, since query
+    # counts don't carry the same run-to-run noise that timing and
+    # allocation metrics do (spec section 16.3).
+    module DeterministicMetricDecision
+      module_function
+
+      def call(metric, baseline_samples, candidate_samples, config)
+        change = MetricChange.summarize(metric, baseline_samples, candidate_samples)
+        thresholds = config.dig(:comparison, :practical_thresholds, metric.to_sym) || {}
+        decision = verdict(change, thresholds)
+
+        MetricChange.result(change, confidence: nil, practically_significant: decision != "pass", noisy: false,
+                                    decision: decision)
+      end
+
+      def verdict(change, thresholds)
+        warning_absolute = thresholds[:warning_absolute] || 0
+        failure_percent = thresholds[:failure_percent] || Float::INFINITY
+
+        return "pass" if change[:absolute_change] <= 0
+        return "pass" if change[:absolute_change] < warning_absolute
+
+        change[:change_percent] >= failure_percent ? "fail" : "warn"
+      end
+    end
+  end
+end

data/lib/perfgate/comparison/diagnostics.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+module Perfgate
+  module Comparison
+    # Simple, deterministic diagnostic rules (spec 20.3). These exist to
+    # point a reader at a likely explanation for a regression, never to
+    # claim a source-code root cause: "SQL query count increased by 5"
+    # is a fact about the sample data, not a guess about which line of
+    # code changed.
+    module Diagnostics
+      module_function
+
+      def for_workload(metrics)
+        [
+          sql_count_rule(metrics),
+          sql_duration_without_count_rule(metrics),
+          allocations_rule(metrics),
+          duration_without_resource_change_rule(metrics),
+          *noise_rules(metrics)
+        ].compact
+      end
+
+      def environment_changed_rules(compatibility)
+        return [] if compatibility["status"] == "compatible"
+
+        compatibility.fetch("differences", []).map do |difference|
+          "Environment changed: #{difference["field"]} (#{difference["severity"]})."
+        end
+      end
+
+      def regressed?(metric_result)
+        metric_result && metric_result["decision"] != "pass" && (metric_result["absolute_change"] || 0).positive?
+      end
+
+      def sql_count_rule(metrics)
+        metric = metrics["sql_count"]
+        return nil unless regressed?(metric)
+
+        "SQL query count increased by #{metric["absolute_change"].to_i}."
+      end
+
+      def sql_duration_without_count_rule(metrics)
+        return nil unless regressed?(metrics["sql_duration"]) && !regressed?(metrics["sql_count"])
+
+        "SQL duration increased without a corresponding increase in query count."
+      end
+
+      def allocations_rule(metrics)
+        metric = metrics["allocations"]
+        return nil unless regressed?(metric)
+
+        "Allocations increased by #{metric["change_percent"]}%."
+      end
+
+      def duration_without_resource_change_rule(metrics)
+        return nil unless regressed?(metrics["duration"])
+
+        resource_metrics = metrics.values_at("sql_count", "sql_duration", "allocations").compact
+        return nil if resource_metrics.empty? || resource_metrics.any? { |metric| metric["decision"] != "pass" }
+
+        "Duration increased while SQL and allocation metrics remained stable."
+      end
+
+      def noise_rules(metrics)
+        metrics.select { |_name, metric| metric["noisy"] }
+               .map { |name, _metric| "Sample variability is high for #{name}; treat this decision with caution." }
+      end
+    end
+  end
+end

data/lib/perfgate/comparison/engine.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+require_relative "workload_comparison"
+require_relative "diagnostics"
+require_relative "../fingerprints/compatibility"
+
+module Perfgate
+  module Comparison
+    # Builds the schema_version 1 comparison-result document (spec
+    # section 14.2) from a baseline run-result and a candidate
+    # run-result. This is the seam between the fingerprinting/statistics
+    # primitives and the CLI: it never touches storage or exit codes
+    # (that's Policy::Engine's job), only produces the structured
+    # decision document.
+    #
+    # Compatibility is checked first (spec section 15): when the runs are
+    # incompatible, no per-workload metric decisions are computed at all
+    # -- Baseline never silently compares runs it can't vouch for.
+    module Engine
+      SCHEMA_VERSION = 1
+
+      module_function
+
+      def compare(baseline_run:, candidate_run:, config: Perfgate.configuration)
+        compatibility = evaluate_compatibility(baseline_run, candidate_run, config)
+        workloads = workloads_for(compatibility, baseline_run, candidate_run, config)
+
+        build_document(baseline_run, candidate_run, compatibility, workloads)
+      end
+
+      def build_document(baseline_run, candidate_run, compatibility, workloads)
+        {
+          "schema_version" => SCHEMA_VERSION,
+          "baseline_run_id" => baseline_run["run_id"],
+          "candidate_run_id" => candidate_run["run_id"],
+          "compatibility" => compatibility,
+          "decision" => overall_decision(compatibility, workloads),
+          "workloads" => workloads,
+          "diagnostics" => diagnostics(baseline_run, candidate_run, compatibility)
+        }
+      end
+
+      def workloads_for(compatibility, baseline_run, candidate_run, config)
+        return [] if compatibility["status"] == "incompatible"
+
+        WorkloadComparison.compare_all(baseline_run, candidate_run, config)
+      end
+
+      def evaluate_compatibility(baseline_run, candidate_run, config)
+        Fingerprints::Compatibility.evaluate(
+          baseline_components: baseline_run.fetch("fingerprint", {}),
+          candidate_components: candidate_run.fetch("fingerprint", {}), config: config
+        )
+      end
+
+      def overall_decision(compatibility, workloads)
+        return "incompatible" if compatibility["status"] == "incompatible"
+
+        decisions = workloads.map { |w| w["decision"] }
+        return "fail" if decisions.include?("fail")
+        return "warn" if decisions.intersect?(%w[warn incomparable removed_workload new_workload])
+
+        "pass"
+      end
+
+      def diagnostics(baseline_run, candidate_run, compatibility)
+        diagnostics = []
+        diagnostics << { "code" => "empty_baseline", "message" => "baseline run has no workloads" } if
+          baseline_run.fetch("workloads", []).empty?
+        diagnostics << { "code" => "empty_candidate", "message" => "candidate run has no workloads" } if
+          candidate_run.fetch("workloads", []).empty?
+        Diagnostics.environment_changed_rules(compatibility).each do |message|
+          diagnostics << { "code" => "environment_changed", "message" => message }
+        end
+        diagnostics
+      end
+    end
+  end
+end

data/lib/perfgate/comparison/metric_change.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+require_relative "../statistics/summary"
+
+module Perfgate
+  module Comparison
+    # Shared helpers for building a metric comparison result and for
+    # computing the baseline->candidate change that every decision path
+    # (statistical or deterministic) needs (spec section 14.2's
+    # `workloads[].metrics` entries).
+    module MetricChange
+      module_function
+
+      def inconclusive(metric)
+        base_fields(metric).merge("decision" => "inconclusive")
+      end
+
+      def base_fields(metric)
+        {
+          "metric" => metric.to_s,
+          "baseline_median" => nil,
+          "candidate_median" => nil,
+          "change_percent" => nil,
+          "absolute_change" => nil,
+          "confidence" => nil,
+          "practically_significant" => false,
+          "noisy" => false
+        }
+      end
+
+      def summarize(metric, baseline_samples, candidate_samples)
+        baseline_summary = Statistics::Summary.call(baseline_samples)
+        candidate_summary = Statistics::Summary.call(candidate_samples)
+        absolute_change = candidate_summary["median"] - baseline_summary["median"]
+        change_percent = percent_change(baseline_summary["median"], absolute_change)
+
+        build_change(metric, baseline_summary, candidate_summary, absolute_change, change_percent)
+      end
+
+      def build_change(metric, baseline_summary, candidate_summary, absolute_change, change_percent)
+        {
+          metric: metric.to_s,
+          baseline_summary: baseline_summary,
+          candidate_summary: candidate_summary,
+          absolute_change: absolute_change,
+          change_percent: change_percent
+        }
+      end
+
+      def percent_change(baseline_median, absolute_change)
+        return 0.0 if baseline_median.zero?
+
+        ((absolute_change / baseline_median.to_f) * 100).round(2)
+      end
+
+      def result(change, confidence:, practically_significant:, noisy:, decision:)
+        change_fields(change).merge(
+          "confidence" => confidence, "practically_significant" => practically_significant,
+          "noisy" => noisy, "decision" => decision
+        )
+      end
+
+      def change_fields(change)
+        {
+          "metric" => change[:metric],
+          "baseline_median" => change[:baseline_summary]["median"],
+          "candidate_median" => change[:candidate_summary]["median"],
+          "change_percent" => change[:change_percent],
+          "absolute_change" => change[:absolute_change]
+        }
+      end
+
+      # A metric is "noisy" when its own baseline spread (MAD relative to
+      # its median) is large enough that a modest shift in medians could
+      # plausibly be explained by run-to-run variance alone (spec 16.5).
+      def noisy?(baseline_summary, config)
+        median = baseline_summary["median"]
+        return false if median.zero?
+
+        (baseline_summary["mad"] / median.to_f) > config.comparison_noise_ratio_threshold
+      end
+    end
+  end
+end

data/lib/perfgate/comparison/metric_decision.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require_relative "metric_change"
+require_relative "statistical_metric_decision"
+require_relative "deterministic_metric_decision"
+
+module Perfgate
+  module Comparison
+    # Decides pass/warn/fail/inconclusive for a single metric on a single
+    # workload (spec section 16). A regression is only ever reported as
+    # "fail" when it clears both bars: a practically-significant change
+    # (spec 16.4's thresholds) and statistical support that it isn't just
+    # sampling noise (spec 16.3's Mann-Whitney test) -- see
+    # spikes/FINDINGS.md for why Milestone 0 settled on requiring both.
+    # `sql_count` is the one exception: it's compared deterministically
+    # (spec 16.3), since query counts are rarely continuous/noisy the way
+    # timing and allocation metrics are.
+    module MetricDecision
+      DETERMINISTIC_METRICS = %w[sql_count].freeze
+
+      module_function
+
+      def call(metric:, baseline_samples:, candidate_samples:, config: Perfgate.configuration)
+        return MetricChange.inconclusive(metric) unless enough_samples?(baseline_samples, candidate_samples, config)
+
+        if DETERMINISTIC_METRICS.include?(metric.to_s)
+          DeterministicMetricDecision.call(metric, baseline_samples, candidate_samples, config)
+        else
+          StatisticalMetricDecision.call(metric, baseline_samples, candidate_samples, config)
+        end
+      end
+
+      def enough_samples?(baseline_samples, candidate_samples, config)
+        minimum = config.comparison_minimum_samples
+        baseline_samples.size >= minimum && candidate_samples.size >= minimum
+      end
+    end
+  end
+end