perfgate 0.1.0

data/lib/perfgate/report/markdown.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+module Perfgate
+  module Report
+    # Renders the schema_version 1 comparison-result document (plus its
+    # Policy::Engine verdict) as a Markdown report, per spec 20.2: the
+    # overall decision, run identities, compatibility status, a
+    # per-workload/per-metric table, noise warnings, diagnostics, a
+    # pointer to the machine-readable JSON, and an explanation of the
+    # exit code. This is what `baseline run --format markdown` writes
+    # and what the GitHub Actions example appends to the job summary.
+    module Markdown
+      DURATION_METRICS = %w[duration sql_duration].freeze
+
+      module_function
+
+      def render(comparison_result:, policy_result:, comparison_path: nil)
+        [
+          header(policy_result),
+          identities(comparison_result),
+          "**Compatibility:** #{comparison_result.dig("compatibility", "status")}",
+          workloads_table(comparison_result),
+          diagnostics_section(comparison_result),
+          machine_readable_section(comparison_path),
+          exit_code_section(policy_result)
+        ].compact.join("\n\n")
+      end
+
+      def header(policy_result)
+        "## Baseline Performance Assurance\n\n**Overall:** #{policy_result["status"].upcase}"
+      end
+
+      def identities(comparison_result)
+        "- Baseline run: `#{comparison_result["baseline_run_id"]}`\n" \
+          "- Candidate run: `#{comparison_result["candidate_run_id"]}`"
+      end
+
+      def workloads_table(comparison_result)
+        workloads = comparison_result.fetch("workloads", [])
+        return "_No workloads were compared._" if workloads.empty?
+
+        ([table_header] + workloads.flat_map { |workload| workload_rows(workload) }).join("\n")
+      end
+
+      def table_header
+        "| Workload | Metric | Baseline | Candidate | Change | Decision |\n|---|---|---|---|---|---|"
+      end
+
+      def workload_rows(workload)
+        return [summary_row(workload)] if workload["metrics"].empty?
+
+        workload["metrics"].map { |name, metric| metric_row(workload["id"], name, metric) }
+      end
+
+      def summary_row(workload)
+        "| #{workload["id"]} | - | - | - | - | #{workload["decision"].upcase} |"
+      end
+
+      def metric_row(workload_id, name, metric)
+        noise = metric["noisy"] ? " ⚠️ noisy" : ""
+        "| #{workload_id} | #{name} | #{format_value(name, metric["baseline_median"])} | " \
+          "#{format_value(name, metric["candidate_median"])} | #{format_change(metric["change_percent"])} | " \
+          "#{metric["decision"].upcase}#{noise} |"
+      end
+
+      def format_change(percent)
+        percent ? format("%+.1f%%", percent) : "n/a"
+      end
+
+      def format_value(name, value)
+        return "n/a" if value.nil?
+
+        DURATION_METRICS.include?(name) ? format("%.2fms", value / 1_000_000.0) : value.to_s
+      end
+
+      def diagnostics_section(comparison_result)
+        messages = diagnostic_messages(comparison_result)
+        return nil if messages.empty?
+
+        "**Diagnostics:**\n#{messages.map { |message| "- #{message}" }.join("\n")}"
+      end
+
+      def diagnostic_messages(comparison_result)
+        workload_messages = comparison_result.fetch("workloads", []).flat_map { |w| w["diagnostics"] || [] }
+        run_messages = comparison_result.fetch("diagnostics", []).map { |d| d["message"] }
+        workload_messages + run_messages
+      end
+
+      def machine_readable_section(comparison_path)
+        return nil unless comparison_path
+
+        "Machine-readable result: `#{comparison_path}`"
+      end
+
+      def exit_code_section(policy_result)
+        "Exit code `#{policy_result["exit_code"]}` (#{policy_result["status"]}). " \
+          "See the exit-code table in the docs for what each status means for CI."
+      end
+    end
+  end
+end

data/lib/perfgate/rspec/discovery.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require_relative "workload_builder"
+
+module Perfgate
+  module RSpec
+    # Discovers RSpec examples tagged for Baseline (`:baseline` metadata,
+    # spec section 9.1) after spec files have been loaded, and registers
+    # a Workload for each one.
+    #
+    # `::RSpec.world.all_examples` is a private RSpec::Core API, but it is
+    # widely relied on by tooling that inspects a loaded suite and has
+    # been stable across RSpec 3.x releases.
+    module Discovery
+      module_function
+
+      def call(registry: Perfgate.registry, builder: default_builder)
+        ::RSpec.world.all_examples.each do |example|
+          next unless example.metadata[:perfgate]
+
+          registry.register(builder.build(example))
+        end
+        registry
+      end
+
+      def default_builder
+        WorkloadBuilder.new(defaults: Perfgate.configuration.execution_defaults)
+      end
+    end
+  end
+end

data/lib/perfgate/rspec/id_resolver.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require "pathname"
+
+module Perfgate
+  module RSpec
+    # Computes the stable workload identifier for an RSpec example, per
+    # spec section 9.2: "<spec file relative path>:<RSpec example full
+    # description>", unless overridden via `perfgate: { id: "..." }`.
+    module IdResolver
+      module_function
+
+      def resolve(example)
+        explicit_id(example) || "#{relative_spec_path(example)}:#{example.full_description}"
+      end
+
+      def explicit_id(example)
+        metadata = example.metadata[:perfgate]
+        return nil unless metadata.is_a?(Hash)
+
+        metadata[:id]
+      end
+
+      def relative_spec_path(example)
+        absolute = File.expand_path(example.metadata[:file_path])
+        Pathname.new(absolute).relative_path_from(Pathname.pwd).to_s
+      end
+    end
+  end
+end

data/lib/perfgate/rspec/workload_builder.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+require_relative "../workloads/workload"
+require_relative "id_resolver"
+
+module Perfgate
+  module RSpec
+    # Builds a Workloads::Workload that wraps a single RSpec example.
+    #
+    # Each call re-instantiates the example group and re-runs the example
+    # via RSpec's own `Example#run(instance, reporter)`, which is safe to
+    # invoke repeatedly: it resets the example's execution result and
+    # exercises the full before/around/after hook chain each time (spec
+    # section 12.2, "reset workload state" + "execute measured samples").
+    # A ::RSpec::Core::NullReporter discards RSpec's own reporting, since
+    # Baseline does its own result collection.
+    class WorkloadBuilder
+      def initialize(defaults:)
+        @defaults = defaults
+      end
+
+      def build(example)
+        options = example.metadata[:perfgate]
+        options = {} unless options.is_a?(Hash)
+
+        Workloads::Workload.new(
+          id: IdResolver.resolve(example),
+          samples: options.fetch(:samples, @defaults.fetch(:samples)),
+          warmup: options.fetch(:warmup, @defaults.fetch(:warmup)),
+          metrics: options.fetch(:metrics, @defaults.fetch(:metrics))
+        ) { run_example(example) }
+      end
+
+      private
+
+      def run_example(example)
+        instance = example.example_group.new
+        example.run(instance, ::RSpec::Core::NullReporter)
+
+        exception = example.execution_result.exception
+        return unless exception
+
+        raise Perfgate::WorkloadError,
+              "workload #{example.full_description.inspect} failed: #{exception.message}"
+      end
+    end
+  end
+end

data/lib/perfgate/rspec.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+require "rspec/core"
+require_relative "rspec/id_resolver"
+require_relative "rspec/workload_builder"
+require_relative "rspec/discovery"
+
+module Perfgate
+  # RSpec integration entry point. Require "baselined/rspec" (typically
+  # from spec_helper.rb) to enable `:baseline`-tagged examples and
+  # `Perfgate.measure`. See spec section 9 for the public API contract.
+  module RSpec
+  end
+end

data/lib/perfgate/serialization/run_result.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+require "securerandom"
+require "time"
+require_relative "../statistics/summary"
+require_relative "../fingerprints/components"
+
+module Perfgate
+  module Serialization
+    # Builds the schema_version 1 run-result document described in spec
+    # section 14.1. Milestone 5 will add source metadata; fingerprinting
+    # (spec section 15) is populated here so the comparison engine can
+    # decide compatibility without re-deriving it from scratch.
+    module RunResult
+      SCHEMA_VERSION = 1
+
+      module_function
+
+      def build(workload_results, config: Perfgate.configuration)
+        {
+          "schema_version" => SCHEMA_VERSION,
+          "run_id" => SecureRandom.uuid,
+          "created_at" => Time.now.utc.iso8601,
+          "fingerprint" => Fingerprints::Components.collect(config: config),
+          "workloads" => workload_results.map { |result| build_workload(result) }
+        }
+      end
+
+      def build_workload(result)
+        samples = result.fetch("samples")
+
+        {
+          "id" => result.fetch("id"),
+          "status" => result.fetch("status"),
+          "error" => result["error"],
+          "definition_hash" => result["definition_hash"],
+          "samples" => samples,
+          "summary" => summarize(samples)
+        }
+      end
+
+      # Every metric key present in at least one sample gets its own
+      # summary block (spec section 14.1 shows this for duration_ns, but
+      # the same shape applies to sql_count, sql_duration_ns, etc. once
+      # those metrics are enabled).
+      def summarize(samples)
+        metric_keys(samples).each_with_object({}) do |key, summary|
+          values = samples.map { |sample| sample[key] }.compact
+          summary[key] = Statistics::Summary.call(values)
+        end
+      end
+
+      def metric_keys(samples)
+        samples.each_with_object([]) { |sample, keys| keys.concat(sample.keys) }.uniq
+      end
+    end
+  end
+end

data/lib/perfgate/statistics/mann_whitney_u.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+module Perfgate
+  module Statistics
+    # One-sided Mann-Whitney U test: estimates how likely it is that the
+    # candidate distribution is drawn from a "worse" (larger-valued)
+    # population than the baseline distribution, without assuming
+    # normality (spec section 16.3). Uses a normal approximation with a
+    # tie correction, adapted from the Milestone 0 spike
+    # (spikes/regression_injection.rb) that validated this approach
+    # against a seeded +20% duration regression.
+    #
+    # Baseline never surfaces this p-value directly to users (spec
+    # 16.3: "do not expose p-values alone as user-facing proof") -- it is
+    # combined with a practical-significance threshold by
+    # Comparison::MetricDecision.
+    module MannWhitneyU
+      module_function
+
+      # Returns p, the probability of observing rank sums this extreme
+      # (or more) under the null hypothesis of no difference, tested
+      # against the one-sided alternative that `candidate` tends to be
+      # larger than `baseline`. Smaller p is stronger evidence the
+      # candidate is worse.
+      def one_sided_p(baseline, candidate)
+        return 1.0 if baseline.empty? || candidate.empty?
+
+        z = z_score(baseline, candidate)
+        return 0.5 if z.nil?
+
+        0.5 * Math.erfc(z / Math.sqrt(2))
+      end
+
+      # Standardized U statistic for the candidate sample, or nil when the
+      # null distribution has zero variance (e.g. every sample tied).
+      def z_score(baseline, candidate)
+        n1 = baseline.size
+        n2 = candidate.size
+        u_candidate = candidate_rank_sum(baseline, candidate) - (n2 * (n2 + 1) / 2.0)
+        std_u = standard_deviation_u(n1, n2)
+        return nil if std_u.zero?
+
+        (u_candidate - (n1 * n2 / 2.0)) / std_u
+      end
+
+      def standard_deviation_u(baseline_size, candidate_size)
+        Math.sqrt(baseline_size * candidate_size * (baseline_size + candidate_size + 1) / 12.0)
+      end
+
+      def candidate_rank_sum(baseline, candidate)
+        ranks = rank(baseline.map { |v| [v, :baseline] } + candidate.map { |v| [v, :candidate] })
+        ranks.each_index.sum { |i| ranks[i][1] == :candidate ? ranks[i][2] : 0 }
+      end
+
+      # Assigns tied (averaged) ranks to a list of [value, label] pairs,
+      # returning [value, label, rank] triples sorted by value.
+      def rank(pairs)
+        sorted = pairs.sort_by { |value, _label| value }
+        ranked = Array.new(sorted.size)
+
+        index = 0
+        index = assign_tie_group(sorted, ranked, index) while index < sorted.size
+
+        ranked
+      end
+
+      # Finds the run of tied values starting at start_index, assigns
+      # them all the same averaged rank, and returns the index just past
+      # the run.
+      def assign_tie_group(sorted, ranked, start_index)
+        end_index = tie_group_end(sorted, start_index)
+        average_rank = ((start_index + 1) + (end_index + 1)) / 2.0
+        (start_index..end_index).each { |k| ranked[k] = sorted[k] + [average_rank] }
+        end_index + 1
+      end
+
+      def tie_group_end(sorted, start_index)
+        end_index = start_index
+        end_index += 1 while end_index + 1 < sorted.size && sorted[end_index + 1][0] == sorted[start_index][0]
+        end_index
+      end
+    end
+  end
+end

data/lib/perfgate/statistics/summary.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module Perfgate
+  module Statistics
+    # Basic summary statistics for a set of raw sample values, used to
+    # populate the `summary` block of a run result (spec sections 14.1
+    # and 16.2).
+    module Summary
+      module_function
+
+      def call(values)
+        return empty if values.empty?
+
+        sorted = values.sort
+        med = median(sorted)
+
+        central_tendency(values, sorted, med).merge("count" => values.size)
+      end
+
+      def central_tendency(values, sorted, med)
+        {
+          "mean" => mean(values).round,
+          "median" => med.round,
+          "min" => sorted.first.round,
+          "max" => sorted.last.round,
+          "mad" => mad(sorted, med).round,
+          "p90" => percentile(sorted, 90).round
+        }
+      end
+
+      def mean(values)
+        values.sum / values.size.to_f
+      end
+
+      def median(sorted)
+        percentile(sorted, 50)
+      end
+
+      # Median absolute deviation: a robust spread measure, less sensitive
+      # to outliers than standard deviation.
+      def mad(sorted, med)
+        deviations = sorted.map { |v| (v - med).abs }.sort
+        percentile(deviations, 50)
+      end
+
+      def percentile(sorted, pct)
+        return sorted.first.to_f if sorted.size == 1
+
+        rank = (pct / 100.0) * (sorted.size - 1)
+        lower = sorted[rank.floor]
+        upper = sorted[rank.ceil]
+        lower + ((upper - lower) * (rank - rank.floor))
+      end
+
+      def empty
+        { "mean" => 0, "median" => 0, "min" => 0, "max" => 0, "mad" => 0, "p90" => 0, "count" => 0 }
+      end
+    end
+  end
+end

data/lib/perfgate/storage/adapter.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Perfgate
+  module Storage
+    # Interface every storage backend implements (spec section 18.1).
+    class Adapter
+      def save_run(run_result)
+        raise NotImplementedError, "#{self.class} must implement #save_run"
+      end
+
+      def load_run(reference)
+        raise NotImplementedError, "#{self.class} must implement #load_run"
+      end
+
+      def list_runs(filters = {})
+        raise NotImplementedError, "#{self.class} must implement #list_runs"
+      end
+
+      def save_comparison(comparison_result)
+        raise NotImplementedError, "#{self.class} must implement #save_comparison"
+      end
+    end
+  end
+end

data/lib/perfgate/storage/archive.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "rubygems/package"
+require "stringio"
+require "zlib"
+
+module Perfgate
+  module Storage
+    # Packs/unpacks the portable baseline-run-<run-id>.tar.gz archive
+    # format (spec 18.2), used to move a run bundle outside of a
+    # Filesystem adapter's own root -- e.g. as a manually-downloaded CI
+    # artifact. Extraction rejects any entry that would escape the
+    # destination directory (spec 22: reject path traversal).
+    module Archive
+      module_function
+
+      def write(archive_path, dir, run_id)
+        tar_io = StringIO.new
+        Gem::Package::TarWriter.new(tar_io) do |tar|
+          Dir.glob("**/*", File::FNM_DOTMATCH, base: dir).each { |entry| add_entry(tar, dir, run_id, entry) }
+        end
+
+        Zlib::GzipWriter.open(archive_path) { |gz| gz.write(tar_io.string) }
+      end
+
+      def extract(archive_path, into)
+        Zlib::GzipReader.open(archive_path) do |gz|
+          Gem::Package::TarReader.new(gz) { |tar| extract_entries(tar, into) }
+        end
+      end
+
+      def add_entry(tar, dir, run_id, entry)
+        return if [".", ".."].include?(File.basename(entry))
+
+        source = File.join(dir, entry)
+        name = File.join(run_id, entry)
+        if File.directory?(source)
+          tar.mkdir(name, 0o755)
+        else
+          tar.add_file(name, 0o644) { |io| io.write(File.read(source)) }
+        end
+      end
+
+      def extract_entries(tar, into)
+        tar.each do |entry|
+          destination = safe_destination(into, entry.full_name)
+          entry.directory? ? FileUtils.mkdir_p(destination) : extract_file(entry, destination)
+        end
+      end
+
+      def extract_file(entry, destination)
+        FileUtils.mkdir_p(File.dirname(destination))
+        File.write(destination, entry.read)
+      end
+
+      def safe_destination(into, entry_name)
+        base = File.join(into, "runs")
+        destination = File.expand_path(File.join(base, entry_name))
+
+        unless destination.start_with?("#{File.expand_path(base)}/")
+          raise Perfgate::ResultBundleError, "archive entry #{entry_name.inspect} escapes the destination directory"
+        end
+
+        destination
+      end
+    end
+  end
+end

data/lib/perfgate/storage/filesystem.rb

@@ -0,0 +1,121 @@
+# frozen_string_literal: true
+
+require "digest"
+require "fileutils"
+require "json"
+require "securerandom"
+require_relative "adapter"
+require_relative "archive"
+
+module Perfgate
+  module Storage
+    # Default MVP storage adapter (spec section 18.2). Layout:
+    #
+    #   <root>/runs/<run-id>/manifest.json
+    #   <root>/runs/<run-id>/run.json
+    #   <root>/runs/<run-id>/checksums.json
+    #   <root>/comparisons/<comparison-id>.json
+    class Filesystem < Adapter
+      def initialize(root:)
+        super()
+        @root = root
+      end
+
+      def save_run(run_result)
+        run_id = run_result.fetch("run_id")
+        dir = run_directory(run_id)
+        FileUtils.mkdir_p(dir)
+
+        run_json = JSON.pretty_generate(run_result)
+        File.write(File.join(dir, "run.json"), run_json)
+        File.write(File.join(dir, "manifest.json"), JSON.pretty_generate(manifest_for(run_result)))
+        File.write(File.join(dir, "checksums.json"), JSON.pretty_generate(checksums_for(run_json)))
+
+        dir
+      end
+
+      def load_run(reference)
+        dir = File.directory?(reference) ? reference : run_directory(reference)
+        run_json_path = File.join(dir, "run.json")
+
+        raise Perfgate::ResultBundleError, "no run result found at #{run_json_path}" unless File.exist?(run_json_path)
+
+        verify_checksum!(dir, run_json_path)
+        JSON.parse(File.read(run_json_path))
+      end
+
+      def list_runs(filters = {}) # rubocop:disable Lint/UnusedMethodArgument
+        base = File.join(@root, "runs")
+        return [] unless File.directory?(base)
+
+        Dir.children(base).sort
+      end
+
+      def save_comparison(comparison_result)
+        dir = File.join(@root, "comparisons")
+        FileUtils.mkdir_p(dir)
+
+        comparison_id = SecureRandom.uuid
+        path = File.join(dir, "#{comparison_id}.json")
+        File.write(path, JSON.pretty_generate(comparison_result))
+
+        path
+      end
+
+      # Packs a run bundle's directory into the portable
+      # baseline-run-<run-id>.tar.gz format (spec 18.2), for manual
+      # transfer or storage outside the filesystem adapter's own root
+      # (e.g. as a CI artifact). Returns the archive's path.
+      def export_archive(run_id, into: @root)
+        dir = run_directory(run_id)
+        raise Perfgate::ResultBundleError, "no run bundle found at #{dir}" unless File.directory?(dir)
+
+        FileUtils.mkdir_p(into)
+        archive_path = File.join(into, "baseline-run-#{run_id}.tar.gz")
+        Archive.write(archive_path, dir, run_id)
+        archive_path
+      end
+
+      # Unpacks a baseline-run-<run-id>.tar.gz archive into `into`,
+      # rejecting any entry that would escape the destination directory
+      # (spec 22: reject path traversal in archives). Returns the
+      # extracted run directory path.
+      def import_archive(archive_path, into: @root)
+        Archive.extract(archive_path, into)
+        File.join(into, "runs", File.basename(archive_path, ".tar.gz").sub(/^baseline-run-/, ""))
+      end
+
+      private
+
+      def run_directory(run_id)
+        File.join(@root, "runs", run_id)
+      end
+
+      def manifest_for(run_result)
+        {
+          "schema_version" => run_result["schema_version"],
+          "run_id" => run_result["run_id"],
+          "created_at" => run_result["created_at"],
+          "workload_ids" => run_result.fetch("workloads").map { |w| w["id"] }
+        }
+      end
+
+      def checksums_for(run_json)
+        { "run.json" => "sha256:#{Digest::SHA256.hexdigest(run_json)}" }
+      end
+
+      def verify_checksum!(dir, run_json_path)
+        checksums_path = File.join(dir, "checksums.json")
+        return unless File.exist?(checksums_path)
+
+        expected = JSON.parse(File.read(checksums_path))["run.json"]
+        return unless expected
+
+        actual = "sha256:#{Digest::SHA256.hexdigest(File.read(run_json_path))}"
+        return if actual == expected
+
+        raise Perfgate::ResultBundleError, "checksum mismatch for #{run_json_path}"
+      end
+    end
+  end
+end

data/lib/perfgate/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Perfgate
+  VERSION = "0.1.0"
+end