rspec-mergify 0.2.0-aarch64-linux

data/lib/mergify/rspec/flaky_detection.rb

@@ -0,0 +1,215 @@
+# frozen_string_literal: true
+
+require 'set'
+require_relative 'utils'
+require_relative 'native'
+require_relative 'version'
+
+module Mergify
+  module RSpec
+    # Signals that flaky detection must not run for this session, and that this
+    # is expected rather than a failure: the repository has not opted in (the
+    # server responds with 404) or there is no baseline of recorded tests yet.
+    # Callers skip silently instead of surfacing an error banner.
+    class FlakyDetectionDisabledError < StandardError; end
+
+    # Manages intelligent test rerunning with budget constraints for flaky detection.
+    # rubocop:disable-next Metrics/ClassLength
+    class FlakyDetector
+      # Per-test tracking metrics.
+      class TestMetrics
+        attr_accessor :initial_setup_duration, :initial_call_duration, :initial_teardown_duration,
+                      :rerun_count, :deadline, :prevented_timeout, :total_duration
+
+        def initialize
+          @initial_setup_duration = 0.0
+          @initial_call_duration = 0.0
+          @initial_teardown_duration = 0.0
+          @rerun_count = 0
+          @deadline = nil
+          @prevented_timeout = false
+          @total_duration = 0.0
+        end
+
+        def initial_duration
+          @initial_setup_duration + @initial_call_duration + @initial_teardown_duration
+        end
+
+        def remaining_time
+          return 0.0 if @deadline.nil?
+
+          [(@deadline - Time.now.to_f), 0.0].max
+        end
+
+        def will_exceed_deadline?
+          return false if @deadline.nil?
+
+          (Time.now.to_f + initial_duration) >= @deadline
+        end
+
+        def fill_from_report(phase, duration, _status)
+          case phase
+          when 'setup'
+            @initial_setup_duration = duration if @initial_setup_duration.zero?
+          when 'call'
+            @initial_call_duration = duration if @initial_call_duration.zero?
+            @rerun_count += 1
+          when 'teardown'
+            @initial_teardown_duration = duration if @initial_teardown_duration.zero?
+          end
+          @total_duration += duration
+        end
+      end
+
+      attr_reader :tests_to_process, :budget, :mode
+
+      def initialize(token:, url:, full_repository_name:, mode:)
+        @token = token
+        @url = url
+        @full_repository_name = full_repository_name
+        @mode = mode
+        @metrics = {}
+        @over_length_tests = Set.new
+        @tests_to_process = []
+        @budget = 0.0
+
+        @context = fetch_context
+        raise FlakyDetectionDisabledError unless Native::Budget.should_run(@context, @mode)
+      end
+
+      # Which tests this session reruns, and how long it may spend doing it,
+      # both come from the shared budget engine -- so a Ruby suite and a Python
+      # one facing the same context spend the same time.
+      #
+      # The engine sizes the budget from the existing tests *in this session*,
+      # where this class counted every existing test the context knew about. A
+      # session running part of a suite was handed the whole suite's budget; it
+      # now gets its own.
+      def prepare_for_session(test_ids)
+        plan = Native::Budget.compute(@context, @mode, test_ids, [])
+
+        @tests_to_process = plan['tests_to_process']
+        # The engine works in milliseconds; everything downstream compares
+        # against RSpec's durations, which are seconds.
+        @budget = plan['available_budget_ms'] / 1000.0
+      end
+
+      # rubocop:disable-next Metrics/MethodLength
+      def fill_metrics_from_report(test_id, phase, duration, status)
+        if status == :skipped
+          @metrics.delete(test_id)
+          return
+        end
+
+        return unless @tests_to_process.include?(test_id)
+
+        if test_id.length > @context['max_test_name_length']
+          @over_length_tests.add(test_id)
+          return
+        end
+
+        # Only initialize metrics when the first phase is "setup"
+        return if !@metrics.key?(test_id) && phase != 'setup'
+
+        @metrics[test_id] ||= TestMetrics.new
+        @metrics[test_id].fill_from_report(phase, duration, status)
+      end
+
+      def rerunning_test?(test_id)
+        @metrics.key?(test_id) && @metrics[test_id].rerun_count >= 1
+      end
+
+      def test_rerun?(test_id)
+        @metrics.key?(test_id) && @metrics[test_id].rerun_count > 1
+      end
+
+      def set_test_deadline(test_id, timeout: nil)
+        return unless @metrics.key?(test_id)
+
+        remaining_tests = [remaining_tests_count, 1].max
+        per_test_budget = remaining_budget / remaining_tests
+
+        allocated =
+          if timeout
+            [per_test_budget, timeout * 0.9].min
+          else
+            per_test_budget
+          end
+
+        @metrics[test_id].deadline = Time.now.to_f + allocated
+      end
+
+      def test_too_slow?(test_id)
+        return false unless @metrics.key?(test_id)
+
+        metrics = @metrics[test_id]
+        min_exec = @context['min_test_execution_count']
+        (metrics.initial_duration * min_exec) > metrics.remaining_time
+      end
+
+      def last_rerun_for_test?(test_id)
+        return false unless @metrics.key?(test_id)
+
+        metrics = @metrics[test_id]
+        metrics.will_exceed_deadline? || metrics.rerun_count >= @context['max_test_execution_count']
+      end
+
+      def test_metrics(test_id)
+        @metrics[test_id]
+      end
+
+      # rubocop:disable-next Metrics/MethodLength,Metrics/AbcSize
+      def make_report
+        lines = []
+        lines << 'Mergify Flaky Detection Report'
+        lines << "  Mode        : #{@mode}"
+        lines << "  Budget      : #{format('%.2f', @budget)}s"
+        lines << "  Budget used : #{format('%.2f', budget_used)}s"
+        lines << "  Tests tracked: #{@metrics.size}"
+        lines << ''
+
+        @metrics.each do |test_id, m|
+          lines << "  #{test_id}"
+          lines << "    Reruns       : #{m.rerun_count}"
+          lines << "    Initial dur  : #{format('%.3f', m.initial_duration)}s"
+          lines << "    Total dur    : #{format('%.3f', m.total_duration)}s"
+          lines << "    Timeout warn : #{m.prevented_timeout}" if m.prevented_timeout
+        end
+
+        lines << '' unless @over_length_tests.empty?
+        @over_length_tests.each do |id|
+          lines << "  WARNING: test name too long (skipped): #{id[0, 80]}..."
+        end
+
+        lines.join("\n")
+      end
+
+      private
+
+      # A nil context means the repository has not opted into flaky detection,
+      # which is the expected default rather than a failure.
+      def fetch_context
+        raise FlakyDetectionDisabledError unless Native.available?
+
+        owner, repo = Utils.split_full_repo_name(@full_repository_name)
+        context = Native::Client.new(@url, @token, owner, repo, VERSION).fetch_flaky_context
+        raise FlakyDetectionDisabledError if context.nil?
+
+        context
+      end
+
+      def remaining_budget
+        used = budget_used
+        [@budget - used, 0.0].max
+      end
+
+      def budget_used
+        @metrics.sum { |_, m| m.total_duration }
+      end
+
+      def remaining_tests_count
+        @tests_to_process.count { |id| !@metrics.key?(id) || @metrics[id].deadline.nil? }
+      end
+    end
+  end
+end

data/lib/mergify/rspec/formatter.rb

@@ -0,0 +1,218 @@
+# frozen_string_literal: true
+
+require 'rspec/core/formatters/base_formatter'
+require 'opentelemetry-sdk'
+
+module Mergify
+  module RSpec
+    # RSpec formatter that creates OpenTelemetry spans for Mergify Test Insights and
+    # prints a terminal report. It is purely observational and does not modify
+    # test execution.
+    # rubocop:disable-next Metrics/ClassLength
+    class Formatter < ::RSpec::Core::Formatters::BaseFormatter
+      ::RSpec::Core::Formatters.register self,
+                                         :start,
+                                         :example_started,
+                                         :example_finished,
+                                         :example_pending,
+                                         :stop
+
+      # rubocop:disable-next Metrics/MethodLength
+      def start(notification)
+        super
+
+        @ci_insights = Mergify::RSpec.ci_insights
+        return unless @ci_insights&.tracer
+
+        extract_distributed_trace_context
+
+        @session_span = @ci_insights.tracer.start_span(
+          'rspec session start',
+          with_parent: @parent_context,
+          attributes: { 'test.scope' => 'session' }
+        )
+        @has_error = false
+        @example_spans = {}
+      end
+
+      def example_started(notification)
+        return unless @ci_insights&.tracer && @session_span
+
+        example = notification.example
+        parent_context = OpenTelemetry::Trace.context_with_span(@session_span)
+        quarantined = @ci_insights.mark_test_as_quarantined_if_needed(example.id)
+
+        span = @ci_insights.tracer.start_span(
+          example.id,
+          with_parent: parent_context,
+          attributes: build_example_attributes(example, quarantined)
+        )
+        @example_spans[example.id] = span
+      end
+
+      # rubocop:disable-next Metrics/MethodLength
+      def example_finished(notification)
+        return unless @example_spans
+
+        example = notification.example
+        span = @example_spans.delete(example.id)
+        return unless span
+
+        result = example.execution_result
+        status = result.status.to_s
+        span.set_attribute('test.case.result.status', status)
+        set_flaky_attributes(span, example)
+
+        if result.status == :failed
+          set_error_attributes(span, result.exception)
+          @has_error = true
+        else
+          span.status = OpenTelemetry::Trace::Status.ok
+        end
+
+        span.finish
+      end
+
+      def example_pending(notification)
+        return unless @example_spans
+
+        example = notification.example
+        span = @example_spans.delete(example.id)
+        return unless span
+
+        span.set_attribute('test.case.result.status', 'skipped')
+        span.finish
+      end
+
+      def stop(_notification)
+        finish_session_span
+        print_report
+        flush_and_shutdown
+      end
+
+      private
+
+      def extract_distributed_trace_context
+        traceparent = ENV.fetch('MERGIFY_TRACEPARENT', nil)
+        @parent_context = if traceparent
+                            propagator = OpenTelemetry::Trace::Propagation::TraceContext::TextMapPropagator.new
+                            propagator.extract({ 'traceparent' => traceparent })
+                          end
+      end
+
+      def build_example_attributes(example, quarantined)
+        {
+          'test.scope' => 'case',
+          'code.filepath' => example.metadata[:file_path].delete_prefix('./'),
+          'code.function' => example.description,
+          'code.lineno' => example.metadata[:line_number] || 0,
+          'code.namespace' => example.example_group.description,
+          'code.file.path' => File.expand_path(example.metadata[:file_path]),
+          'code.line.number' => example.metadata[:line_number] || 0,
+          'cicd.test.quarantined' => quarantined
+        }
+      end
+
+      def set_flaky_attributes(span, example)
+        meta = example.metadata
+
+        rerun_count = meta[:mergify_rerun_count]
+        span.set_attribute('cicd.test.rerun_count', rerun_count) unless rerun_count.nil?
+
+        flaky = meta[:mergify_flaky]
+        span.set_attribute('cicd.test.flaky', flaky) unless flaky.nil?
+
+        flaky_detection = meta[:mergify_flaky_detection]
+        span.set_attribute('cicd.test.flaky_detection', flaky_detection) unless flaky_detection.nil?
+
+        new_test = meta[:mergify_new_test]
+        span.set_attribute('cicd.test.new', new_test) unless new_test.nil?
+      end
+
+      def set_error_attributes(span, exception)
+        span.set_attribute('exception.type', exception.class.to_s)
+        span.set_attribute('exception.message', exception.message)
+        span.set_attribute('exception.stacktrace', exception.backtrace&.join("\n") || '')
+        span.status = OpenTelemetry::Trace::Status.error(exception.message)
+      end
+
+      def finish_session_span
+        return unless @session_span
+
+        @session_span.status = if @has_error
+                                 OpenTelemetry::Trace::Status.error('One or more tests failed')
+                               else
+                                 OpenTelemetry::Trace::Status.ok
+                               end
+        @session_span.finish
+      end
+
+      # rubocop:disable-next Metrics/MethodLength
+      def print_report
+        output.puts ''
+        output.puts '--- Mergify CI ---'
+
+        unless @ci_insights
+          output.puts 'Mergify Test Insights is not configured.'
+          return
+        end
+
+        print_configuration_warnings
+        print_flaky_report
+        print_quarantine_report
+        output.puts "MERGIFY_TEST_RUN_ID=#{@ci_insights.test_run_id}"
+        output.puts '------------------'
+      end
+
+      def print_configuration_warnings
+        output.puts 'WARNING: MERGIFY_TOKEN is not set. Traces will not be sent to Mergify.' unless @ci_insights.token
+
+        return if @ci_insights.repo_name
+
+        output.puts 'WARNING: Could not detect repository name. ' \
+                    'Please set GITHUB_REPOSITORY or configure a git remote.'
+      end
+
+      def print_flaky_report
+        return unless @ci_insights.flaky_detector.respond_to?(:make_report)
+
+        report = @ci_insights.flaky_detector.make_report
+        output.puts report if report
+      end
+
+      def print_quarantine_report
+        return unless @ci_insights.quarantined_tests.respond_to?(:report)
+
+        report = @ci_insights.quarantined_tests.report
+        output.puts report if report
+      end
+
+      def flush_and_shutdown # rubocop:disable Metrics/MethodLength
+        return unless @ci_insights&.tracer_provider
+
+        begin
+          @ci_insights.tracer_provider.force_flush
+        rescue StandardError => e
+          print_export_error(e)
+        end
+
+        begin
+          @ci_insights.tracer_provider.shutdown
+        rescue StandardError => e
+          output.puts "Error while shutting down the tracer: #{e.message}"
+        end
+      end
+
+      def print_export_error(error)
+        output.puts "Error while exporting traces: #{error.message}"
+        output.puts ''
+        output.puts 'Common issues:'
+        output.puts '  - Your MERGIFY_TOKEN might not be set or could be invalid'
+        output.puts '  - Mergify Test Insights might not be enabled for this repository'
+        output.puts '  - There might be a network connectivity issue with the Mergify API'
+        output.puts ''
+        output.puts 'Documentation: https://docs.mergify.com/ci-insights/test-frameworks/'
+      end
+    end
+  end
+end

data/lib/mergify/rspec/native.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+module Mergify
+  module RSpec
+    # The Rust extension over mergify-ci-core: CI detection shared with the
+    # pytest and TypeScript clients.
+    #
+    # Loading is best-effort by design. A precompiled gem carries one extension
+    # per Ruby under `<ruby>/`, the source gem compiles one alongside this file,
+    # and a platform we publish neither for gets neither. Detection is one input
+    # to telemetry, not the point of the gem, so an absent extension degrades
+    # what we can report rather than breaking the suite under test -- the same
+    # fail-open posture the napi binding takes when a platform has no prebuilt
+    # binary. `load_error` keeps the reason, for callers that want to say so.
+    module Native
+      # Raised when a Mergify API call fails outright.
+      #
+      # Distinct from StandardError on purpose: callers degrade on an API
+      # failure, and a bare rescue there would swallow genuine bugs in the
+      # binding as though the backend were down.
+      class ApiError < StandardError; end
+
+      class << self
+        # The LoadError that prevented the extension loading, or nil.
+        attr_accessor :load_error
+
+        def available?
+          load_error.nil?
+        end
+      end
+    end
+  end
+end
+
+begin
+  # Precompiled gems ship lib/mergify/rspec/<ruby>/mergify_ci.<dlext>.
+  require_relative "#{RUBY_VERSION.to_f}/mergify_ci"
+rescue LoadError
+  begin
+    # Source gem, and any local `rake compile`.
+    require_relative 'mergify_ci'
+  rescue LoadError => e
+    Mergify::RSpec::Native.load_error = e
+  end
+end
+
+unless Mergify::RSpec::Native.available?
+  # Stand in for the extension so callers can just ask, and get the same answer
+  # they would get from a machine that is not in CI. Branching on availability
+  # at every call site would only spread the same nil back through the caller.
+  module Mergify
+    module RSpec
+      module Native
+        class << self
+          def detect_provider
+            nil
+          end
+
+          def detect_repository_name
+            nil
+          end
+
+          def detect_attributes
+            {}
+          end
+        end
+      end
+    end
+  end
+end

data/lib/mergify/rspec/quarantine.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+require 'set'
+require_relative 'utils'
+require_relative 'native'
+require_relative 'version'
+
+module Mergify
+  module RSpec
+    # Fetches quarantined test names from the Mergify API and tracks which are used.
+    #
+    # The fetch itself -- pagination, the RFC 8288 `next` links, the status
+    # codes that mean "not subscribed" rather than "broken" -- belongs to the
+    # shared Rust client now, so every Mergify test client reads a quarantine
+    # list the same way. What stays here is what RSpec cares about: which of
+    # those tests this session actually ran, and the report at the end.
+    class Quarantine
+      attr_reader :quarantined_tests, :init_error_msg
+
+      def initialize(api_url:, token:, repo_name:, branch_name:)
+        @repo_name = repo_name
+        @branch_name = branch_name
+        @quarantined_tests = []
+        @used_tests = Set.new
+        @init_error_msg = nil
+
+        fetch(api_url, token, branch_name)
+      end
+
+      def include?(example_id)
+        @quarantined_tests.include?(example_id)
+      end
+
+      def mark_as_used(example_id)
+        @used_tests.add(example_id)
+      end
+
+      # rubocop:disable-next Metrics/MethodLength,Metrics/AbcSize
+      def report
+        used, unused = @quarantined_tests.partition { |t| @used_tests.include?(t) }
+
+        lines = []
+        lines << 'Mergify Quarantine Report'
+        lines << "  Repository : #{@repo_name}"
+        lines << "  Branch     : #{@branch_name}"
+        lines << "  Quarantined tests from API: #{@quarantined_tests.size}"
+        lines << ''
+        lines << "  Quarantined tests run (#{used.size}):"
+        used.each { |t| lines << "    - #{t}" }
+        lines << ''
+        lines << "  Unused quarantined tests (#{unused.size}):"
+        unused.each { |t| lines << "    - #{t}" }
+        lines.join("\n")
+      end
+
+      private
+
+      # A nil list means the repository has no quarantine subscription, which is
+      # not an error: the session simply quarantines nothing. Anything that went
+      # genuinely wrong is recorded and the suite carries on -- this plugin has
+      # never let the backend fail a test run.
+      def fetch(api_url, token, branch_name)
+        unless Native.available?
+          @init_error_msg = "Mergify native extension unavailable: #{Native.load_error}"
+          return
+        end
+
+        owner, repo = Utils.split_full_repo_name(@repo_name)
+        client = Native::Client.new(api_url, token, owner, repo, VERSION)
+        @quarantined_tests = client.fetch_quarantine(branch_name) || []
+      rescue Utils::InvalidRepositoryFullNameError, Native::ApiError => e
+        @init_error_msg = e.message
+      end
+    end
+  end
+end

data/lib/mergify/rspec/resources/rspec.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require 'opentelemetry-sdk'
+require 'rspec/core/version'
+
+module Mergify
+  module RSpec
+    module Resources
+      # Detects OpenTelemetry Resource attributes for RSpec.
+      module RSpec
+        module_function
+
+        def detect
+          OpenTelemetry::SDK::Resources::Resource.create(
+            'test.framework' => 'rspec',
+            'test.framework.version' => ::RSpec::Core::Version::STRING
+          )
+        end
+      end
+    end
+  end
+end

data/lib/mergify/rspec/synchronous_batch_span_processor.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+require 'opentelemetry-sdk'
+
+module Mergify
+  module RSpec
+    class ExportError < StandardError; end
+
+    # A span processor that queues spans in memory and exports them all in one
+    # batch when force_flush is called. This avoids HTTP requests during test
+    # execution.
+    class SynchronousBatchSpanProcessor < OpenTelemetry::SDK::Trace::Export::SimpleSpanProcessor
+      def initialize(exporter)
+        super
+        @queue = []
+      end
+
+      def on_finish(span)
+        return unless span.context.trace_flags.sampled?
+
+        @queue << span.to_span_data
+      end
+
+      def force_flush(timeout: nil) # rubocop:disable Lint/UnusedMethodArgument
+        spans = @queue.dup
+        @queue.clear
+        result = @span_exporter.export(spans)
+        raise ExportError, 'Failed to export traces' unless result == OpenTelemetry::SDK::Trace::Export::SUCCESS
+
+        OpenTelemetry::SDK::Trace::Export::SUCCESS
+      end
+    end
+  end
+end

data/lib/mergify/rspec/utils.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module Mergify
+  module RSpec
+    # Utility methods shared across the rspec-mergify gem.
+    #
+    # CI detection used to live here -- the provider table, the git shelling
+    # out, the per-provider environment mappings. It is the Rust core's now, via
+    # the extension in Mergify::RSpec::Native, so that every Mergify test client
+    # detects identically. What is left is what is genuinely Ruby's: parsing a
+    # repository name the API needs split, and deciding whether the plugin
+    # should switch itself on at all.
+    module Utils
+      module_function
+
+      # Raised when a repository full name (owner/repo) is malformed.
+      class InvalidRepositoryFullNameError < StandardError; end
+
+      TRUTHY_STRINGS = %w[y yes t true on 1].freeze
+      FALSY_STRINGS  = %w[n no f false off 0].freeze
+
+      # Convert a string to a boolean.
+      # Truthy: y yes t true on 1
+      # Falsy:  n no f false off 0
+      # Raises ArgumentError for anything else.
+      def strtobool(string)
+        return true  if TRUTHY_STRINGS.include?(string.downcase)
+        return false if FALSY_STRINGS.include?(string.downcase)
+
+        raise ArgumentError, "Could not convert '#{string}' to boolean"
+      end
+
+      # Returns true when the named environment variable holds a truthy value.
+      def env_truthy?(key)
+        TRUTHY_STRINGS.include?(ENV.fetch(key, '').downcase)
+      end
+
+      # Returns true when the suite is running inside CI or when
+      # RSPEC_MERGIFY_ENABLE is set to a truthy value.
+      #
+      # Deliberately not the core's provider detection: this asks whether the
+      # plugin should run, which an unrecognised CI or a developer setting
+      # RSPEC_MERGIFY_ENABLE both answer yes to.
+      def in_ci?
+        env_truthy?('CI') || env_truthy?('RSPEC_MERGIFY_ENABLE')
+      end
+
+      # Split "owner/repo" into [owner, repo].
+      # Raises InvalidRepositoryFullNameError when the format is wrong.
+      def split_full_repo_name(full_repo_name)
+        parts = full_repo_name.split('/')
+        return parts if parts.size == 2
+
+        raise InvalidRepositoryFullNameError, "Invalid repository name: #{full_repo_name}"
+      end
+    end
+  end
+end