rspec-mergify 0.0.0.dev

data/README.md

@@ -0,0 +1,66 @@
+# rspec-mergify
+
+RSpec plugin for [Mergify CI Insights](https://docs.mergify.com/ci-insights/).
+
+## Features
+
+- **Test tracing** — Sends OpenTelemetry traces for every test to Mergify's API
+- **Flaky test detection** — Intelligently reruns tests to detect flakiness with budget constraints
+- **Test quarantine** — Quarantines failing tests so they don't block CI
+
+## Installation
+
+Add to your Gemfile:
+
+```ruby
+gem 'rspec-mergify'
+```
+
+Then run `bundle install`.
+
+## Configuration
+
+Set the `MERGIFY_TOKEN` environment variable with your Mergify API token.
+
+The plugin activates automatically when running in CI (detected via the `CI` environment variable). To enable outside CI, set `RSPEC_MERGIFY_ENABLE=true`.
+
+### Environment Variables
+
+| Variable | Description | Default |
+|---|---|---|
+| `MERGIFY_TOKEN` | Mergify API authentication token | (required) |
+| `MERGIFY_API_URL` | Mergify API endpoint | `https://api.mergify.com` |
+| `RSPEC_MERGIFY_ENABLE` | Force-enable outside CI | `false` |
+| `RSPEC_MERGIFY_DEBUG` | Print spans to console | `false` |
+| `MERGIFY_TRACEPARENT` | W3C distributed trace context | — |
+| `MERGIFY_TEST_JOB_NAME` | Mergify test job name | — |
+
+## Development
+
+### Prerequisites
+
+- Ruby >= 3.1 (`.ruby-version` pins to 3.4.4 — use [rbenv](https://github.com/rbenv/rbenv) or [mise](https://mise.jdx.dev/) to install it)
+- Bundler
+
+### Setup
+
+```bash
+rbenv install          # install the Ruby version from .ruby-version (if needed)
+bundle install
+```
+
+### Running Tests
+
+```bash
+bundle exec rspec
+```
+
+### Linting
+
+```bash
+bundle exec rubocop
+```
+
+## License
+
+GPL-3.0-only

data/lib/mergify/rspec/ci_insights.rb

@@ -0,0 +1,174 @@
+# frozen_string_literal: true
+
+require 'securerandom'
+require 'opentelemetry-sdk'
+require_relative 'utils'
+require_relative 'synchronous_batch_span_processor'
+require_relative 'resources/ci'
+require_relative 'resources/git'
+require_relative 'resources/github_actions'
+require_relative 'resources/jenkins'
+require_relative 'resources/mergify'
+require_relative 'resources/rspec'
+
+module Mergify
+  module RSpec
+    # Central orchestrator for CI Insights: sets up OpenTelemetry tracing,
+    # manages the tracer provider, and coordinates flaky detection and quarantine.
+    # rubocop:disable Metrics/ClassLength
+    class CIInsights
+      attr_reader :token, :repo_name, :api_url, :test_run_id,
+                  :tracer_provider, :tracer, :exporter,
+                  :branch_name,
+                  :flaky_detector, :flaky_detector_error_message, :quarantined_tests
+
+      # rubocop:disable Metrics/MethodLength
+      def initialize
+        @token = ENV.fetch('MERGIFY_TOKEN', nil)
+        @repo_name = Utils.repository_name
+        @api_url = ENV.fetch('MERGIFY_API_URL', 'https://api.mergify.com')
+        @test_run_id = SecureRandom.hex(8)
+        @tracer_provider = nil
+        @tracer = nil
+        @exporter = nil
+        @branch_name = nil
+        @flaky_detector = nil
+        @flaky_detector_error_message = nil
+        @quarantined_tests = nil
+
+        setup_tracing if Utils.in_ci?
+      end
+      # rubocop:enable Metrics/MethodLength
+
+      def mark_test_as_quarantined_if_needed(example_id) # rubocop:disable Naming/PredicateMethod
+        return false unless @quarantined_tests&.include?(example_id)
+
+        @quarantined_tests.mark_as_used(example_id)
+        true
+      end
+
+      private
+
+      def setup_tracing
+        processor, exp = build_processor
+        return unless processor
+
+        @exporter = exp
+        resource = build_resource
+        @tracer_provider = OpenTelemetry::SDK::Trace::TracerProvider.new(resource: resource)
+        @tracer_provider.add_span_processor(processor)
+        @tracer = @tracer_provider.tracer('rspec-mergify', Mergify::RSpec::VERSION)
+        @branch_name = extract_branch_name(resource)
+        load_flaky_detector
+        load_quarantine
+      end
+
+      def build_processor
+        if debug_mode? || test_mode?
+          build_in_memory_processor
+        elsif @token && @repo_name
+          build_otlp_processor
+        else
+          [nil, nil]
+        end
+      end
+
+      def debug_mode?
+        ENV.key?('RSPEC_MERGIFY_DEBUG')
+      end
+
+      def test_mode?
+        ENV['_RSPEC_MERGIFY_TEST'] == 'true'
+      end
+
+      def build_in_memory_processor
+        exp = OpenTelemetry::SDK::Trace::Export::InMemorySpanExporter.new
+        processor = OpenTelemetry::SDK::Trace::Export::SimpleSpanProcessor.new(exp)
+        [processor, exp]
+      end
+
+      def build_otlp_processor
+        owner, repo = Utils.split_full_repo_name(@repo_name)
+        endpoint = "#{@api_url}/v1/ci/#{owner}/repositories/#{repo}/traces"
+        exp = create_otlp_exporter(endpoint)
+        processor = SynchronousBatchSpanProcessor.new(exp)
+        [processor, exp]
+      end
+
+      # rubocop:disable Metrics/MethodLength
+      def build_resource
+        resources = [
+          Resources::CI.detect,
+          Resources::Git.detect,
+          Resources::GitHubActions.detect,
+          Resources::Jenkins.detect,
+          Resources::Mergify.detect,
+          Resources::RSpec.detect
+        ]
+        base = resources.reduce(OpenTelemetry::SDK::Resources::Resource.create({})) do |merged, r|
+          merged.merge(r)
+        end
+        run_id_resource = OpenTelemetry::SDK::Resources::Resource.create('test.run.id' => @test_run_id)
+        base.merge(run_id_resource)
+      end
+      # rubocop:enable Metrics/MethodLength
+
+      def extract_branch_name(resource)
+        attrs = resource.attribute_enumerator.to_h
+        attrs['vcs.ref.base.name'] || attrs['vcs.ref.head.name']
+      end
+
+      # rubocop:disable Metrics/MethodLength
+      def create_otlp_exporter(endpoint)
+        require 'opentelemetry-exporter-otlp'
+        original_env = ENV.fetch('OTEL_EXPORTER_OTLP_TRACES_ENDPOINT', nil)
+        ENV['OTEL_EXPORTER_OTLP_TRACES_ENDPOINT'] = endpoint
+        begin
+          OpenTelemetry::Exporter::OTLP::Exporter.new(
+            endpoint: endpoint,
+            headers: { 'Authorization' => "Bearer #{@token}" },
+            compression: 'gzip'
+          )
+        ensure
+          if original_env
+            ENV['OTEL_EXPORTER_OTLP_TRACES_ENDPOINT'] = original_env
+          else
+            ENV.delete('OTEL_EXPORTER_OTLP_TRACES_ENDPOINT')
+          end
+        end
+      end
+      # rubocop:enable Metrics/MethodLength
+
+      # rubocop:disable Metrics/MethodLength
+      def load_flaky_detector
+        return unless @token && @repo_name
+        return unless Utils.env_truthy?('_MERGIFY_TEST_NEW_FLAKY_DETECTION')
+
+        require_relative 'flaky_detection'
+        mode = @branch_name ? 'new' : 'unhealthy'
+        @flaky_detector = FlakyDetector.new(
+          token: @token,
+          url: @api_url,
+          full_repository_name: @repo_name,
+          mode: mode
+        )
+      rescue StandardError => e
+        @flaky_detector_error_message = "Could not load flaky detector: #{e.message}"
+      end
+      # rubocop:enable Metrics/MethodLength
+
+      def load_quarantine
+        return unless @token && @repo_name && @branch_name
+
+        require_relative 'quarantine'
+        @quarantined_tests = Quarantine.new(
+          api_url: @api_url,
+          token: @token,
+          repo_name: @repo_name,
+          branch_name: @branch_name
+        )
+      end
+    end
+    # rubocop:enable Metrics/ClassLength
+  end
+end

data/lib/mergify/rspec/configuration.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+require 'set'
+
+module Mergify
+  module RSpec
+    # Registers RSpec hooks for quarantine and flaky detection, and adds the
+    # CI Insights formatter when running inside CI.
+    module Configuration
+      module_function
+
+      # rubocop:disable Metrics/MethodLength,Metrics/BlockLength,Metrics/AbcSize
+      # rubocop:disable Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity
+      def setup!
+        ::RSpec.configure do |config|
+          # Add formatter when in CI
+          config.add_formatter(Mergify::RSpec::Formatter) if Utils.in_ci?
+
+          # Flaky detection: prepare session with all example IDs
+          config.before(:suite) do
+            ci = Mergify::RSpec.ci_insights
+            fd = ci&.flaky_detector
+            if fd
+              all_ids = ::RSpec.world.example_groups.flat_map(&:descendants).flat_map(&:examples).map(&:id)
+              fd.prepare_for_session(all_ids)
+            end
+          end
+
+          # Quarantine: mark tests before execution
+          config.before(:each) do |example|
+            ci = Mergify::RSpec.ci_insights
+            next unless ci&.quarantined_tests&.include?(example.id)
+
+            ci.quarantined_tests.mark_as_used(example.id)
+            example.metadata[:mergify_quarantined] = true
+          end
+
+          # Flaky detection: rerun tests within budget
+          config.around(:each) do |example|
+            ci = Mergify::RSpec.ci_insights
+            fd = ci&.flaky_detector
+
+            example.run
+
+            # Feed metrics from the initial run so the detector can evaluate
+            if fd
+              run_time = example.execution_result.run_time || 0.0
+              status = example.execution_result.status
+              fd.fill_metrics_from_report(example.id, 'setup', 0.0, status)
+              fd.fill_metrics_from_report(example.id, 'call', run_time, status)
+              fd.fill_metrics_from_report(example.id, 'teardown', 0.0, status)
+            end
+
+            next unless fd&.rerunning_test?(example.id)
+
+            fd.set_test_deadline(example.id)
+            next if fd.test_too_slow?(example.id)
+
+            example.metadata[:mergify_flaky_detection] = true
+            example.metadata[:mergify_new_test] = true if fd.mode == 'new'
+
+            distinct_outcomes = Set.new
+            distinct_outcomes.add(example.execution_result.status) if example.execution_result.status
+
+            rerun_count = 0
+            until example.metadata[:is_last_rerun]
+              example.metadata[:is_last_rerun] = fd.last_rerun_for_test?(example.id)
+
+              # Reset example state for rerun
+              example.instance_variable_set(:@exception, nil)
+              if example.example_group_instance
+                memoized = example.example_group_instance.instance_variable_get(:@__memoized)
+                memoized&.clear
+              end
+
+              example.run
+              distinct_outcomes.add(example.execution_result.status)
+              rerun_count += 1
+            end
+
+            is_flaky = distinct_outcomes.include?(:passed) &&
+                       distinct_outcomes.include?(:failed)
+            example.metadata[:mergify_flaky] = true if is_flaky
+            example.metadata[:mergify_rerun_count] = rerun_count
+          end
+
+          # Quarantine: override failed quarantined test results
+          config.after(:each) do |example|
+            next unless example.metadata[:mergify_quarantined] && example.exception
+
+            example.instance_variable_set(:@exception, nil)
+            example.execution_result.status = :pending
+            example.execution_result.pending_message = 'Test is quarantined from Mergify CI Insights'
+          end
+        end
+      end
+      # rubocop:enable Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity
+      # rubocop:enable Metrics/MethodLength,Metrics/BlockLength,Metrics/AbcSize
+    end
+  end
+end

data/lib/mergify/rspec/flaky_detection.rb

@@ -0,0 +1,252 @@
+# frozen_string_literal: true
+
+require 'net/http'
+require 'json'
+require 'uri'
+require 'set'
+require_relative 'utils'
+
+module Mergify
+  module RSpec
+    # Manages intelligent test rerunning with budget constraints for flaky detection.
+    # rubocop:disable Metrics/ClassLength
+    class FlakyDetector
+      # Per-test tracking metrics.
+      class TestMetrics
+        attr_accessor :initial_setup_duration, :initial_call_duration, :initial_teardown_duration,
+                      :rerun_count, :deadline, :prevented_timeout, :total_duration
+
+        def initialize
+          @initial_setup_duration = 0.0
+          @initial_call_duration = 0.0
+          @initial_teardown_duration = 0.0
+          @rerun_count = 0
+          @deadline = nil
+          @prevented_timeout = false
+          @total_duration = 0.0
+        end
+
+        def initial_duration
+          @initial_setup_duration + @initial_call_duration + @initial_teardown_duration
+        end
+
+        def remaining_time
+          return 0.0 if @deadline.nil?
+
+          [(@deadline - Time.now.to_f), 0.0].max
+        end
+
+        def will_exceed_deadline?
+          return false if @deadline.nil?
+
+          (Time.now.to_f + initial_duration) >= @deadline
+        end
+
+        def fill_from_report(phase, duration, _status)
+          case phase
+          when 'setup'
+            @initial_setup_duration = duration if @initial_setup_duration.zero?
+          when 'call'
+            @initial_call_duration = duration if @initial_call_duration.zero?
+            @rerun_count += 1
+          when 'teardown'
+            @initial_teardown_duration = duration if @initial_teardown_duration.zero?
+          end
+          @total_duration += duration
+        end
+      end
+
+      attr_reader :tests_to_process, :budget, :mode
+
+      def initialize(token:, url:, full_repository_name:, mode:)
+        @token = token
+        @url = url
+        @full_repository_name = full_repository_name
+        @mode = mode
+        @metrics = {}
+        @over_length_tests = Set.new
+        @tests_to_process = []
+        @budget = 0.0
+
+        fetch_context
+        validate!
+      end
+
+      # rubocop:disable Metrics/MethodLength,Metrics/AbcSize
+      def prepare_for_session(test_ids)
+        existing = Set.new(@context[:existing_test_names])
+        unhealthy = Set.new(@context[:unhealthy_test_names])
+
+        @tests_to_process =
+          if @mode == 'new'
+            test_ids.reject { |id| existing.include?(id) }
+          else
+            test_ids.select { |id| unhealthy.include?(id) }
+          end
+
+        budget_ratio = if @mode == 'new'
+                         @context[:budget_ratio_for_new_tests]
+                       else
+                         @context[:budget_ratio_for_unhealthy_tests]
+                       end
+
+        mean_duration_s = @context[:existing_tests_mean_duration_ms] / 1000.0
+        existing_count = @context[:existing_test_names].size
+        min_budget_s = @context[:min_budget_duration_ms] / 1000.0
+
+        ratio_budget = budget_ratio * mean_duration_s * existing_count
+        @budget = [ratio_budget, min_budget_s].max
+      end
+      # rubocop:enable Metrics/MethodLength,Metrics/AbcSize
+
+      # rubocop:disable Metrics/MethodLength
+      def fill_metrics_from_report(test_id, phase, duration, status)
+        if status == :skipped
+          @metrics.delete(test_id)
+          return
+        end
+
+        return unless @tests_to_process.include?(test_id)
+
+        if test_id.length > @context[:max_test_name_length]
+          @over_length_tests.add(test_id)
+          return
+        end
+
+        # Only initialize metrics when the first phase is "setup"
+        return if !@metrics.key?(test_id) && phase != 'setup'
+
+        @metrics[test_id] ||= TestMetrics.new
+        @metrics[test_id].fill_from_report(phase, duration, status)
+      end
+      # rubocop:enable Metrics/MethodLength
+
+      def rerunning_test?(test_id)
+        @metrics.key?(test_id) && @metrics[test_id].rerun_count >= 1
+      end
+
+      def test_rerun?(test_id)
+        @metrics.key?(test_id) && @metrics[test_id].rerun_count > 1
+      end
+
+      def set_test_deadline(test_id, timeout: nil)
+        return unless @metrics.key?(test_id)
+
+        remaining_tests = [remaining_tests_count, 1].max
+        per_test_budget = remaining_budget / remaining_tests
+
+        allocated =
+          if timeout
+            [per_test_budget, timeout * 0.9].min
+          else
+            per_test_budget
+          end
+
+        @metrics[test_id].deadline = Time.now.to_f + allocated
+      end
+
+      def test_too_slow?(test_id)
+        return false unless @metrics.key?(test_id)
+
+        metrics = @metrics[test_id]
+        min_exec = @context[:min_test_execution_count]
+        (metrics.initial_duration * min_exec) > metrics.remaining_time
+      end
+
+      def last_rerun_for_test?(test_id)
+        return false unless @metrics.key?(test_id)
+
+        metrics = @metrics[test_id]
+        metrics.will_exceed_deadline? || metrics.rerun_count >= @context[:max_test_execution_count]
+      end
+
+      def test_metrics(test_id)
+        @metrics[test_id]
+      end
+
+      # rubocop:disable Metrics/MethodLength,Metrics/AbcSize
+      def make_report
+        lines = []
+        lines << 'Mergify Flaky Detection Report'
+        lines << "  Mode        : #{@mode}"
+        lines << "  Budget      : #{format('%.2f', @budget)}s"
+        lines << "  Budget used : #{format('%.2f', budget_used)}s"
+        lines << "  Tests tracked: #{@metrics.size}"
+        lines << ''
+
+        @metrics.each do |test_id, m|
+          lines << "  #{test_id}"
+          lines << "    Reruns       : #{m.rerun_count}"
+          lines << "    Initial dur  : #{format('%.3f', m.initial_duration)}s"
+          lines << "    Total dur    : #{format('%.3f', m.total_duration)}s"
+          lines << "    Timeout warn : #{m.prevented_timeout}" if m.prevented_timeout
+        end
+
+        lines << '' unless @over_length_tests.empty?
+        @over_length_tests.each do |id|
+          lines << "  WARNING: test name too long (skipped): #{id[0, 80]}..."
+        end
+
+        lines.join("\n")
+      end
+      # rubocop:enable Metrics/MethodLength,Metrics/AbcSize
+
+      private
+
+      # rubocop:disable Metrics/AbcSize
+      def fetch_context
+        owner, repo = Utils.split_full_repo_name(@full_repository_name)
+        uri = URI("#{@url}/v1/ci/#{owner}/repositories/#{repo}/flaky-detection-context")
+
+        http = Net::HTTP.new(uri.host, uri.port)
+        http.use_ssl = uri.scheme == 'https'
+        http.open_timeout = 10
+        http.read_timeout = 10
+
+        request = Net::HTTP::Get.new(uri)
+        request['Authorization'] = "Bearer #{@token}"
+
+        response = http.request(request)
+        parse_context(response.body)
+      end
+      # rubocop:enable Metrics/AbcSize
+
+      # rubocop:disable Metrics/MethodLength,Metrics/AbcSize
+      def parse_context(body)
+        data = JSON.parse(body, symbolize_names: true)
+        @context = {
+          budget_ratio_for_new_tests: data[:budget_ratio_for_new_tests].to_f,
+          budget_ratio_for_unhealthy_tests: data[:budget_ratio_for_unhealthy_tests].to_f,
+          existing_test_names: Array(data[:existing_test_names]),
+          existing_tests_mean_duration_ms: data[:existing_tests_mean_duration_ms].to_f,
+          unhealthy_test_names: Array(data[:unhealthy_test_names]),
+          max_test_execution_count: data[:max_test_execution_count].to_i,
+          max_test_name_length: data[:max_test_name_length].to_i,
+          min_budget_duration_ms: data[:min_budget_duration_ms].to_f,
+          min_test_execution_count: data[:min_test_execution_count].to_i
+        }
+      end
+      # rubocop:enable Metrics/MethodLength,Metrics/AbcSize
+
+      def validate!
+        return unless @mode == 'new' && @context[:existing_test_names].empty?
+
+        raise 'Cannot use "new" mode without existing test names in the context'
+      end
+
+      def remaining_budget
+        used = budget_used
+        [@budget - used, 0.0].max
+      end
+
+      def budget_used
+        @metrics.sum { |_, m| m.total_duration }
+      end
+
+      def remaining_tests_count
+        @tests_to_process.count { |id| !@metrics.key?(id) || @metrics[id].deadline.nil? }
+      end
+    end
+    # rubocop:enable Metrics/ClassLength
+  end
+end