perfgate 0.1.0

data/lib/perfgate/comparison/statistical_metric_decision.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+require_relative "metric_change"
+require_relative "../statistics/mann_whitney_u"
+
+module Perfgate
+  module Comparison
+    # Statistical decision path used for continuous, noisy metrics
+    # (duration, sql_duration, allocations): combines a practical
+    # threshold check with a one-sided Mann-Whitney U test, and
+    # downgrades a would-be fail to a warn when the metric is noisy
+    # (spec sections 16.3-16.5).
+    module StatisticalMetricDecision
+      module_function
+
+      def call(metric, baseline_samples, candidate_samples, config)
+        change = MetricChange.summarize(metric, baseline_samples, candidate_samples)
+        thresholds = threshold_for(metric, config)
+        p_value = Statistics::MannWhitneyU.one_sided_p(baseline_samples, candidate_samples)
+
+        significance = significance_flags(change, thresholds, p_value, config)
+        noisy = MetricChange.noisy?(change[:baseline_summary], config)
+        decision = decide(significance[:exceeds_failure], significance[:statistically_significant],
+                          significance[:practically_significant], noisy)
+
+        MetricChange.result(change, confidence: (1 - p_value).round(4),
+                                    practically_significant: significance[:practically_significant], noisy: noisy,
+                                    decision: decision)
+      end
+
+      def significance_flags(change, thresholds, p_value, config)
+        alpha = 1 - config.comparison_confidence_level
+        clears_floor = change[:absolute_change].abs >= thresholds[:minimum_absolute]
+        {
+          practically_significant: breaches?(change, thresholds[:warning_percent], thresholds[:minimum_absolute]),
+          exceeds_failure: breaches?(change, thresholds[:failure_percent], thresholds[:minimum_absolute]),
+          statistically_significant: clears_floor && change[:absolute_change].positive? && p_value < alpha
+        }
+      end
+
+      def decide(exceeds_failure, statistically_significant, practically_significant, noisy)
+        if exceeds_failure && statistically_significant
+          noisy ? "warn" : "fail"
+        elsif practically_significant || statistically_significant
+          "warn"
+        else
+          "pass"
+        end
+      end
+
+      def threshold_for(metric, config)
+        raw = config.dig(:comparison, :practical_thresholds, metric.to_sym) || {}
+        {
+          warning_percent: raw[:warning_percent] || Float::INFINITY,
+          failure_percent: raw[:failure_percent] || Float::INFINITY,
+          # minimum_absolute_ms is configured in milliseconds, but samples
+          # (and therefore change[:absolute_change]) are always in the
+          # metric's raw unit, nanoseconds for duration/sql_duration - so
+          # it must be converted before comparison.
+          minimum_absolute: minimum_absolute_ns(raw)
+        }
+      end
+
+      def minimum_absolute_ns(raw)
+        return raw[:minimum_absolute_ms] * 1_000_000 if raw[:minimum_absolute_ms]
+
+        raw[:minimum_absolute] || 0
+      end
+
+      def breaches?(change, percent_threshold, minimum_absolute)
+        change[:change_percent].abs >= percent_threshold && change[:absolute_change].abs >= minimum_absolute
+      end
+    end
+  end
+end

data/lib/perfgate/comparison/workload_comparison.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+require_relative "metric_decision"
+require_relative "diagnostics"
+
+module Perfgate
+  module Comparison
+    # Matches baseline/candidate workloads by id and produces each
+    # workload's decision entry in the comparison-result document (spec
+    # 14.2), including the deterministic per-workload diagnostics (spec
+    # 20.3). Split out of Engine to keep both modules under RuboCop's
+    # module-length limit.
+    module WorkloadComparison
+      # Maps the raw sample keys instrumentation writes (spec section 13)
+      # to the metric names practical_thresholds/fingerprint config uses
+      # (spec section 16.4). gc_* keys are deliberately left unmapped:
+      # GC activity is diagnostic-only and never drives a decision.
+      SAMPLE_KEY_TO_METRIC = {
+        "duration_ns" => "duration",
+        "sql_count" => "sql_count",
+        "sql_duration_ns" => "sql_duration",
+        "allocations" => "allocations"
+      }.freeze
+
+      module_function
+
+      def compare_all(baseline_run, candidate_run, config)
+        baseline_by_id = index_by_id(baseline_run)
+        candidate_by_id = index_by_id(candidate_run)
+
+        (baseline_by_id.keys | candidate_by_id.keys).map do |id|
+          compare_one(id, baseline_by_id[id], candidate_by_id[id], config)
+        end
+      end
+
+      def index_by_id(run)
+        run.fetch("workloads", []).to_h { |workload| [workload["id"], workload] }
+      end
+
+      def compare_one(id, baseline_workload, candidate_workload, config)
+        return missing_workload_result(id, "new_workload") unless baseline_workload
+        return missing_workload_result(id, "removed_workload") unless candidate_workload
+
+        definition_changed = baseline_workload["definition_hash"] != candidate_workload["definition_hash"]
+        return incomparable_workload_result(id) if definition_changed
+
+        metrics = metric_decisions(baseline_workload, candidate_workload, config)
+        { "id" => id, "decision" => workload_decision(metrics), "metrics" => metrics,
+          "diagnostics" => Diagnostics.for_workload(metrics) }
+      end
+
+      def missing_workload_result(id, reason)
+        message = if reason == "new_workload"
+                    "new workload with no prior baseline run"
+                  else
+                    "workload removed since the baseline run"
+                  end
+        { "id" => id, "decision" => reason, "metrics" => {}, "diagnostics" => [message] }
+      end
+
+      def incomparable_workload_result(id)
+        { "id" => id, "decision" => "incomparable", "metrics" => {},
+          "diagnostics" => ["Workload definition changed since the baseline run; comparison skipped."] }
+      end
+
+      def metric_decisions(baseline_workload, candidate_workload, config)
+        sample_keys(baseline_workload, candidate_workload).each_with_object({}) do |sample_key, acc|
+          metric = SAMPLE_KEY_TO_METRIC[sample_key]
+          next unless metric
+
+          baseline_samples = samples_for(baseline_workload, sample_key)
+          candidate_samples = samples_for(candidate_workload, sample_key)
+          next if baseline_samples.empty? || candidate_samples.empty?
+
+          acc[metric] = MetricDecision.call(metric: metric, baseline_samples: baseline_samples,
+                                            candidate_samples: candidate_samples, config: config)
+        end
+      end
+
+      def sample_keys(baseline_workload, candidate_workload)
+        (baseline_workload.fetch("summary", {}).keys + candidate_workload.fetch("summary", {}).keys).uniq
+      end
+
+      def samples_for(workload, sample_key)
+        workload.fetch("samples", []).filter_map { |sample| sample[sample_key] }
+      end
+
+      def workload_decision(metrics)
+        decisions = metrics.values.map { |m| m["decision"] }
+        return "fail" if decisions.include?("fail")
+        return "warn" if decisions.include?("warn")
+        return "inconclusive" if decisions.include?("inconclusive") && decisions.all? { |d| d != "pass" }
+
+        "pass"
+      end
+    end
+  end
+end

data/lib/perfgate/config/defaults.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+module Perfgate
+  class Config
+    # Default configuration values, matching the illustrative baseline.yml
+    # in spec section 11. `storage.path` intentionally uses the base
+    # directory from section 18.2's layout (`.baseline`) rather than the
+    # `.baseline/results` shown in section 11's example, since the two
+    # sections of the spec disagree and 18.2 is the more specific
+    # authority on directory layout.
+    module Defaults
+      HASH = {
+        version: 1,
+        profile: "default",
+        execution: {
+          samples: 8,
+          warmup: 2,
+          seed: 12_345,
+          order: "defined",
+          fail_fast: false,
+          isolation: "process_per_workload"
+        },
+        metrics: {
+          duration: { enabled: true },
+          sql_count: { enabled: true },
+          sql_duration: { enabled: true },
+          allocations: { enabled: true },
+          gc: { enabled: true },
+          memory: { enabled: false }
+        },
+        comparison: {
+          minimum_samples: 5,
+          confidence_level: 0.95,
+          # How much of a metric's own noise (MAD relative to its median)
+          # we tolerate before treating a would-be "fail" as unreliable
+          # and downgrading it to "warn" instead (spec 16.5). The spec
+          # calls for noise-aware downgrading but doesn't name a default
+          # ratio, so 0.5 (MAD up to half the median) was chosen as a
+          # conservative starting point pending real-world tuning.
+          noise_ratio_threshold: 0.5,
+          practical_thresholds: {
+            duration: { warning_percent: 10, failure_percent: 20, minimum_absolute_ms: 10 },
+            sql_count: { warning_absolute: 1, failure_percent: 20 },
+            sql_duration: { warning_percent: 15, failure_percent: 30, minimum_absolute_ms: 5 },
+            allocations: { warning_percent: 15, failure_percent: 30 }
+          }
+        },
+        policy: {
+          fail_on: "fail",
+          incompatible: "warn",
+          missing_baseline: "warn",
+          new_workload: "warn",
+          removed_workload: "warn"
+        },
+        fingerprint: {
+          strict: %w[
+            ruby_engine ruby_version rails_version baseline_version_major
+            database_adapter database_version_major workload_definition_hash dataset_hash
+          ],
+          informational: %w[
+            operating_system cpu_model cpu_count memory_bytes
+            ci_provider runner_image dependency_lock_hash
+          ]
+        },
+        storage: { adapter: "filesystem", path: ".perfgate" },
+        telemetry: { enabled: false }
+      }.freeze
+    end
+  end
+end

data/lib/perfgate/config/env_overrides.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module Perfgate
+  class Config
+    # Applies PERFGATE_-prefixed environment variable overrides on top of
+    # a merged configuration hash, per spec section 11 ("environment-
+    # variable overrides use a documented PERFGATE_ prefix"). Any leaf
+    # path can be overridden, e.g. PERFGATE_EXECUTION_SAMPLES=12 overrides
+    # execution.samples. The override is type-coerced based on the
+    # existing default value at that path.
+    module EnvOverrides
+      PREFIX = "PERFGATE_"
+
+      module_function
+
+      def apply(hash, env: ENV)
+        each_leaf_path(hash) do |path, current_value|
+          env_key = PREFIX + path.join("_").upcase
+          next unless env.key?(env_key)
+
+          set_path(hash, path, coerce(env[env_key], current_value))
+        end
+        hash
+      end
+
+      def each_leaf_path(hash, prefix = [], &block)
+        hash.each do |key, value|
+          path = prefix + [key]
+          if value.is_a?(Hash)
+            each_leaf_path(value, path, &block)
+          else
+            block.call(path, value)
+          end
+        end
+      end
+
+      def set_path(hash, path, value)
+        *init, last = path
+        target = init.reduce(hash) { |h, k| h[k] }
+        target[last] = value
+      end
+
+      def coerce(raw, current_value)
+        case current_value
+        when Integer then Integer(raw)
+        when Float then Float(raw)
+        when true, false then %w[1 true yes on].include?(raw.downcase)
+        when Array then raw.split(",").map(&:strip)
+        else raw
+        end
+      end
+    end
+  end
+end

data/lib/perfgate/config/schema.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Perfgate
+  class Config
+    # Declares which keys baseline.yml recognizes at each nesting level.
+    # A Hash value means "this key has its own nested keys"; `true` marks
+    # a leaf value. Used to enforce "unknown keys fail validation" from
+    # spec section 11.
+    module Schema
+      TREE = {
+        version: true,
+        profile: true,
+        execution: {
+          samples: true,
+          warmup: true,
+          seed: true,
+          order: true,
+          fail_fast: true,
+          isolation: true
+        },
+        metrics: {
+          duration: { enabled: true },
+          sql_count: { enabled: true },
+          sql_duration: { enabled: true },
+          allocations: { enabled: true },
+          gc: { enabled: true },
+          memory: { enabled: true }
+        },
+        comparison: {
+          minimum_samples: true,
+          confidence_level: true,
+          noise_ratio_threshold: true,
+          practical_thresholds: {
+            duration: { warning_percent: true, failure_percent: true, minimum_absolute_ms: true },
+            sql_count: { warning_absolute: true, failure_percent: true },
+            sql_duration: { warning_percent: true, failure_percent: true, minimum_absolute_ms: true },
+            allocations: { warning_percent: true, failure_percent: true }
+          }
+        },
+        policy: {
+          fail_on: true,
+          incompatible: true,
+          missing_baseline: true,
+          new_workload: true,
+          removed_workload: true
+        },
+        fingerprint: { strict: true, informational: true },
+        storage: { adapter: true, path: true },
+        telemetry: { enabled: true }
+      }.freeze
+    end
+  end
+end

data/lib/perfgate/config/validator.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+module Perfgate
+  class Config
+    # Validates a merged configuration hash against Schema::TREE and a
+    # handful of range/type rules that matter for Milestone 1 (sample and
+    # warmup counts). Raises Perfgate::ConfigurationError on any problem,
+    # per spec section 11 ("unknown keys fail validation").
+    module Validator
+      MINIMUM_SAMPLES = 3
+
+      module_function
+
+      def call(hash)
+        check_unknown_keys(hash, Schema::TREE, [])
+        check_version(hash)
+        check_execution(hash)
+        hash
+      end
+
+      def check_unknown_keys(hash, schema, path)
+        hash.each_key do |key|
+          check_known_key(key, schema, path)
+
+          nested_schema = schema[key]
+          next unless nested_schema.is_a?(Hash)
+
+          value = hash[key]
+          unless value.is_a?(Hash)
+            raise Perfgate::ConfigurationError, "expected #{(path + [key]).join(".")} to be a mapping"
+          end
+
+          check_unknown_keys(value, nested_schema, path + [key])
+        end
+      end
+
+      def check_known_key(key, schema, path)
+        return if schema.key?(key)
+
+        raise Perfgate::ConfigurationError, "unknown configuration key: #{(path + [key]).join(".")}"
+      end
+
+      def check_version(hash)
+        return if hash[:version] == 1
+
+        raise Perfgate::ConfigurationError, "unsupported configuration version: #{hash[:version].inspect} (expected 1)"
+      end
+
+      def check_execution(hash)
+        samples = hash.dig(:execution, :samples)
+        warmup = hash.dig(:execution, :warmup)
+
+        if samples && (!samples.is_a?(Integer) || samples < MINIMUM_SAMPLES)
+          raise Perfgate::ConfigurationError,
+                "execution.samples must be an integer >= #{MINIMUM_SAMPLES}, got #{samples.inspect}"
+        end
+
+        return unless warmup && (!warmup.is_a?(Integer) || warmup.negative?)
+
+        raise Perfgate::ConfigurationError, "execution.warmup must be a non-negative integer, got #{warmup.inspect}"
+      end
+    end
+  end
+end

data/lib/perfgate/config.rb

@@ -0,0 +1,136 @@
+# frozen_string_literal: true
+
+require "yaml"
+require_relative "config/defaults"
+require_relative "config/schema"
+require_relative "config/validator"
+require_relative "config/env_overrides"
+
+module Perfgate
+  # Loads, validates, and provides typed access to baseline.yml (spec
+  # section 11). Configuration is a plain merged Hash under the hood;
+  # this class only adds convenience readers for the values Milestone 1
+  # actually acts on (execution sample/warmup counts, enabled metrics,
+  # storage path). Later milestones will add readers for comparison,
+  # policy, and fingerprint sections as those are implemented.
+  class Config
+    class << self
+      # A Config built entirely from defaults, with no file on disk.
+      def default
+        new(deep_dup(Defaults::HASH))
+      end
+
+      # Loads baseline.yml (or the given path). A missing file is treated
+      # as an empty configuration, i.e. pure defaults.
+      def load(path = "perfgate.yml")
+        raw = read_yaml(path)
+        merged = deep_merge(deep_dup(Defaults::HASH), raw)
+        EnvOverrides.apply(merged)
+        Validator.call(merged)
+        new(merged)
+      end
+
+      private
+
+      def read_yaml(path)
+        return {} unless File.exist?(path)
+
+        content = YAML.safe_load_file(path, permitted_classes: [Symbol], symbolize_names: true)
+        content || {}
+      rescue Psych::SyntaxError => e
+        raise Perfgate::ConfigurationError, "invalid YAML in #{path}: #{e.message}"
+      end
+
+      # A small recursive dup that avoids Marshal (forbidden project-wide
+      # for serialization) while still deep-copying nested hashes/arrays
+      # so mutating a loaded Config never mutates the frozen defaults.
+      def deep_dup(value)
+        case value
+        when Hash
+          value.each_with_object({}) { |(k, v), acc| acc[k] = deep_dup(v) }
+        when Array
+          value.map { |v| deep_dup(v) }
+        else
+          value
+        end
+      end
+
+      def deep_merge(base, override)
+        base.merge(override) do |_key, base_val, override_val|
+          if base_val.is_a?(Hash) && override_val.is_a?(Hash)
+            deep_merge(base_val, override_val)
+          else
+            override_val
+          end
+        end
+      end
+    end
+
+    # The default dataset fingerprint hook (spec section 12.3): apps that
+    # care about dataset drift affecting comparability can override this
+    # with a callable of their own via `Perfgate.configure`.
+    DEFAULT_DATASET_FINGERPRINT = -> { ENV.fetch("PERFGATE_DATASET_VERSION", "unspecified") }
+
+    attr_reader :to_h
+    attr_accessor :dataset_fingerprint
+
+    def initialize(hash)
+      @to_h = hash
+      @dataset_fingerprint = DEFAULT_DATASET_FINGERPRINT
+    end
+
+    def execution_samples
+      to_h.dig(:execution, :samples)
+    end
+
+    def execution_warmup
+      to_h.dig(:execution, :warmup)
+    end
+
+    def enabled_metrics
+      to_h.fetch(:metrics).select { |_name, opts| opts[:enabled] }.keys
+    end
+
+    # Defaults handed to newly-discovered workloads that don't override
+    # samples/warmup/metrics themselves (spec section 9.3).
+    def execution_defaults
+      { samples: execution_samples, warmup: execution_warmup, metrics: enabled_metrics }
+    end
+
+    def storage_path
+      to_h.dig(:storage, :path)
+    end
+
+    def comparison_minimum_samples
+      to_h.dig(:comparison, :minimum_samples)
+    end
+
+    def comparison_confidence_level
+      to_h.dig(:comparison, :confidence_level)
+    end
+
+    def comparison_noise_ratio_threshold
+      to_h.dig(:comparison, :noise_ratio_threshold)
+    end
+
+    def practical_thresholds
+      to_h.dig(:comparison, :practical_thresholds)
+    end
+
+    def fingerprint_strict_fields
+      to_h.dig(:fingerprint, :strict)
+    end
+
+    def fingerprint_informational_fields
+      to_h.dig(:fingerprint, :informational)
+    end
+
+    def policy
+      to_h.fetch(:policy)
+    end
+
+    def dig(*keys)
+      to_h.dig(*keys)
+    end
+  end
+end

data/lib/perfgate/errors.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Perfgate
+  # Base class for all Baseline-raised errors.
+  class Error < StandardError; end
+
+  # Raised when baseline.yml is missing, malformed, or fails schema validation.
+  class ConfigurationError < Error; end
+
+  # Raised when a workload cannot be executed as configured (e.g. missing
+  # RSpec example, invalid metadata, duplicate workload id).
+  class WorkloadError < Error; end
+
+  # Raised when a candidate and baseline run cannot be safely compared.
+  class IncompatibleRunError < Error; end
+
+  # Raised when a result bundle is malformed, tampered with, or fails
+  # checksum verification.
+  class ResultBundleError < Error; end
+end

data/lib/perfgate/execution/process_runner.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+require "json"
+require_relative "runner"
+
+module Perfgate
+  module Execution
+    # Runs a workload's warmup + samples inside a fresh child process
+    # (spec section 12.1: default "process_per_workload" isolation). The
+    # child relays its result to the parent as JSON over a pipe -- Marshal
+    # is deliberately avoided per the project's cross-process
+    # serialization policy (data crossing a process boundary must not be
+    # able to instantiate arbitrary Ruby objects).
+    #
+    # Note for database-backed workloads: an in-memory SQLite database
+    # does not survive fork (each child effectively starts with an empty
+    # database), so process isolation requires a file-based or
+    # server-based test database. This mirrors an existing constraint on
+    # Rails' own parallel test runners and isn't specific to Baseline.
+    class ProcessRunner
+      def initialize(workload, runner_class: Runner)
+        @workload = workload
+        @runner_class = runner_class
+      end
+
+      def call
+        ensure_fork_supported!
+
+        reader, writer = IO.pipe
+        pid = fork_child(reader, writer)
+        writer.close
+        payload = reader.read
+        reader.close
+        _pid, status = Process.waitpid2(pid)
+
+        parse_result(payload, status)
+      end
+
+      private
+
+      def ensure_fork_supported!
+        return if Process.respond_to?(:fork)
+
+        raise Perfgate::Error, "process isolation requires Process.fork, which this Ruby platform does not support"
+      end
+
+      def fork_child(reader, writer)
+        Process.fork do
+          reader.close
+          result = @runner_class.new(@workload).call
+          writer.write(JSON.generate(result))
+          writer.close
+          exit!(0)
+        end
+      end
+
+      def parse_result(payload, status)
+        if payload.nil? || payload.empty?
+          {
+            "id" => @workload.id,
+            "status" => "error",
+            "samples" => [],
+            "error" => "workload process exited without a result (exit status #{status.exitstatus})"
+          }
+        else
+          JSON.parse(payload)
+        end
+      end
+    end
+  end
+end