datadog-ci 1.4.1 → 1.5.0

data/lib/datadog/ci/test_retries/component.rb

@@ -1,7 +1,15 @@
 # frozen_string_literal: true
 
+require_relative "driver/no_retry"
+require_relative "driver/retry_failed"
+require_relative "driver/retry_new"
+
 require_relative "strategy/no_retry"
 require_relative "strategy/retry_failed"
+require_relative "strategy/retry_new"
+
+require_relative "../ext/telemetry"
+require_relative "../utils/telemetry"
 
 module Datadog
   module CI
@@ -10,73 +18,94 @@ module Datadog
       # - retrying failed tests - improve success rate of CI pipelines
       # - retrying new tests - detect flaky tests as early as possible to prevent them from being merged
       class Component
-        attr_reader :retry_failed_tests_enabled, :retry_failed_tests_max_attempts,
-          :retry_failed_tests_total_limit, :retry_failed_tests_count
+        FIBER_LOCAL_CURRENT_RETRY_DRIVER_KEY = :__dd_current_retry_driver
 
         def initialize(
           retry_failed_tests_enabled:,
           retry_failed_tests_max_attempts:,
-          retry_failed_tests_total_limit:
+          retry_failed_tests_total_limit:,
+          retry_new_tests_enabled:,
+          unique_tests_client:
         )
-          @retry_failed_tests_enabled = retry_failed_tests_enabled
-          @retry_failed_tests_max_attempts = retry_failed_tests_max_attempts
-          @retry_failed_tests_total_limit = retry_failed_tests_total_limit
-          # counter that store the current number of failed tests retried
-          @retry_failed_tests_count = 0
+          no_retries_strategy = Strategy::NoRetry.new
+
+          retry_failed_strategy = Strategy::RetryFailed.new(
+            enabled: retry_failed_tests_enabled,
+            max_attempts: retry_failed_tests_max_attempts,
+            total_limit: retry_failed_tests_total_limit
+          )
 
+          retry_new_strategy = Strategy::RetryNew.new(
+            enabled: retry_new_tests_enabled,
+            unique_tests_client: unique_tests_client
+          )
+
+          # order is important, we should try to retry new tests first
+          @retry_strategies = [retry_new_strategy, retry_failed_strategy, no_retries_strategy]
           @mutex = Mutex.new
         end
 
-        def configure(library_settings)
-          @retry_failed_tests_enabled &&= library_settings.flaky_test_retries_enabled?
+        def configure(library_settings, test_session)
+          # let all strategies configure themselves
+          @retry_strategies.each do |strategy|
+            strategy.configure(library_settings, test_session)
+          end
         end
 
         def with_retries(&block)
-          # @type var retry_strategy: Strategy::Base
-          retry_strategy = nil
-
-          test_finished_callback = lambda do |test_span|
-            if retry_strategy.nil?
-              # we always run test at least once and after first pass create a correct retry strategy
-              retry_strategy = build_strategy(test_span)
-            else
-              # after each retry we record the result, strategy will decide if we should retry again
-              retry_strategy&.record_retry(test_span)
-            end
-          end
-
-          test_visibility_component.set_test_finished_callback(test_finished_callback)
+          reset_retries!
 
           loop do
             yield
 
-            break unless retry_strategy&.should_retry?
+            break unless should_retry?
           end
         ensure
-          test_visibility_component.remove_test_finished_callback
+          reset_retries!
         end
 
-        def build_strategy(test_span)
+        def build_driver(test_span)
           @mutex.synchronize do
-            if should_retry_failed_test?(test_span)
-              Datadog.logger.debug("Failed test retry starts")
-              @retry_failed_tests_count += 1
-
-              Strategy::RetryFailed.new(max_attempts: @retry_failed_tests_max_attempts)
-            else
-              Strategy::NoRetry.new
-            end
+            # find the first strategy that covers the test span and let it build the driver
+            strategy = @retry_strategies.find { |strategy| strategy.covers?(test_span) }
+
+            raise "No retry strategy found for test span: #{test_span.name}" if strategy.nil?
+
+            strategy.build_driver(test_span)
           end
         end
 
+        def record_test_finished(test_span)
+          if current_retry_driver.nil?
+            # we always run test at least once and after the first pass create a correct retry driver
+            self.current_retry_driver = build_driver(test_span)
+          else
+            # after each retry we record the result, the driver will decide if we should retry again
+            current_retry_driver&.record_retry(test_span)
+          end
+        end
+
+        def record_test_span_duration(tracer_span)
+          current_retry_driver&.record_duration(tracer_span.duration)
+        end
+
+        # this API is targeted on Cucumber instrumentation or any other that cannot leverage #with_retries method
+        def reset_retries!
+          self.current_retry_driver = nil
+        end
+
+        def should_retry?
+          !!current_retry_driver&.should_retry?
+        end
+
         private
 
-        def should_retry_failed_test?(test_span)
-          @retry_failed_tests_enabled && !!test_span&.failed? && @retry_failed_tests_count < @retry_failed_tests_total_limit
+        def current_retry_driver
+          Thread.current[FIBER_LOCAL_CURRENT_RETRY_DRIVER_KEY]
         end
 
-        def test_visibility_component
-          Datadog.send(:components).test_visibility
+        def current_retry_driver=(driver)
+          Thread.current[FIBER_LOCAL_CURRENT_RETRY_DRIVER_KEY] = driver
         end
       end
     end

data/lib/datadog/ci/test_retries/driver/base.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Datadog
+  module CI
+    module TestRetries
+      module Driver
+        # Driver is the class responsible for the current test retry mechanism.
+        # It receives signals about each retry execution and steers the current retry strategy.
+        class Base
+          def should_retry?
+            false
+          end
+
+          def record_retry(test_span)
+            test_span&.set_tag(Ext::Test::TAG_IS_RETRY, "true")
+          end
+
+          # duration in float seconds
+          def record_duration(duration)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/test_retries/driver/no_retry.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+require_relative "base"
+
+module Datadog
+  module CI
+    module TestRetries
+      module Driver
+        class NoRetry < Base
+          def record_retry(test_span)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/test_retries/driver/retry_failed.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require_relative "base"
+
+require_relative "../../ext/test"
+
+module Datadog
+  module CI
+    module TestRetries
+      module Driver
+        class RetryFailed < Base
+          attr_reader :max_attempts
+
+          def initialize(max_attempts:)
+            @max_attempts = max_attempts
+
+            @attempts = 0
+            @passed_once = false
+          end
+
+          def should_retry?
+            @attempts < @max_attempts && !@passed_once
+          end
+
+          def record_retry(test_span)
+            super
+
+            @attempts += 1
+            @passed_once = true if test_span&.passed?
+
+            Datadog.logger.debug { "Retry Attempts [#{@attempts} / #{@max_attempts}], Passed: [#{@passed_once}]" }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/test_retries/driver/retry_new.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+require_relative "base"
+
+require_relative "../../ext/test"
+
+module Datadog
+  module CI
+    module TestRetries
+      module Driver
+        # retry every new test up to 10 times (early flake detection)
+        class RetryNew < Base
+          def initialize(test_span, max_attempts_thresholds:)
+            @max_attempts_thresholds = max_attempts_thresholds
+            @attempts = 0
+            # will be changed based on test span duration
+            @max_attempts = 10
+
+            mark_new_test(test_span)
+          end
+
+          def should_retry?
+            @attempts < @max_attempts
+          end
+
+          def record_retry(test_span)
+            super
+
+            @attempts += 1
+            mark_new_test(test_span)
+
+            Datadog.logger.debug { "Retry Attempts [#{@attempts} / #{@max_attempts}]" }
+          end
+
+          def record_duration(duration)
+            @max_attempts = @max_attempts_thresholds.max_attempts_for_duration(duration)
+
+            Datadog.logger.debug { "Recorded test duration of [#{duration}], new Max Attempts value is [#{@max_attempts}]" }
+          end
+
+          private
+
+          def mark_new_test(test_span)
+            test_span.set_tag(Ext::Test::TAG_IS_NEW, "true")
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/test_retries/null_component.rb

@@ -6,13 +6,7 @@ module Datadog
   module CI
     module TestRetries
       class NullComponent < Component
-        attr_reader :retry_failed_tests_enabled, :retry_failed_tests_max_attempts, :retry_failed_tests_total_limit
-
         def initialize
-          # enabled only by remote settings
-          @retry_failed_tests_enabled = false
-          @retry_failed_tests_max_attempts = 0
-          @retry_failed_tests_total_limit = 0
         end
 
         def configure(library_settings)
@@ -22,6 +16,13 @@ module Datadog
           no_action = proc {}
           yield no_action
         end
+
+        def reset_retries!
+        end
+
+        def should_retry?
+          false
+        end
       end
     end
   end

data/lib/datadog/ci/test_retries/strategy/base.rb

@@ -1,16 +1,23 @@
 # frozen_string_literal: true
 
+require_relative "../driver/no_retry"
+
 module Datadog
   module CI
     module TestRetries
       module Strategy
+        # Strategies are subcomponents of the retry mechanism. They are responsible for
+        # determining which tests should be retried and how.
         class Base
-          def should_retry?
-            false
+          def covers?(test_span)
+            true
+          end
+
+          def configure(_library_settings, _test_session)
           end
 
-          def record_retry(test_span)
-            test_span&.set_tag(Ext::Test::TAG_IS_RETRY, "true")
+          def build_driver(_test_span)
+            Driver::NoRetry.new
           end
         end
       end

data/lib/datadog/ci/test_retries/strategy/no_retry.rb

@@ -7,8 +7,6 @@ module Datadog
     module TestRetries
       module Strategy
         class NoRetry < Base
-          def record_retry(test_span)
-          end
         end
       end
     end

data/lib/datadog/ci/test_retries/strategy/retry_failed.rb

@@ -2,33 +2,50 @@
 
 require_relative "base"
 
-require_relative "../../ext/test"
+require_relative "../driver/retry_failed"
 
 module Datadog
   module CI
     module TestRetries
       module Strategy
         class RetryFailed < Base
-          attr_reader :max_attempts
-
-          def initialize(max_attempts:)
+          attr_reader :enabled, :max_attempts,
+            :total_limit, :retried_count
+
+          def initialize(
+            enabled:,
+            max_attempts:,
+            total_limit:
+          )
+            @enabled = enabled
             @max_attempts = max_attempts
+            @total_limit = total_limit
+            @retried_count = 0
+          end
+
+          def covers?(test_span)
+            return false unless @enabled
+
+            if @retried_count >= @total_limit
+              Datadog.logger.debug do
+                "Retry failed tests limit reached: [#{@retried_count}] out of [#{@total_limit}]"
+              end
+              @enabled = false
+            end
 
-            @attempts = 0
-            @passed_once = false
+            @enabled && !!test_span&.failed?
           end
 
-          def should_retry?
-            @attempts < @max_attempts && !@passed_once
+          def configure(library_settings, test_session)
+            @enabled &&= library_settings.flaky_test_retries_enabled?
           end
 
-          def record_retry(test_span)
-            super
+          def build_driver(test_span)
+            Datadog.logger.debug { "#{test_span.name} failed, will be retried" }
 
-            @attempts += 1
-            @passed_once = true if test_span&.passed?
+            @retried_count += 1
 
-            Datadog.logger.debug { "Retry Attempts [#{@attempts} / #{@max_attempts}], Passed: [#{@passed_once}]" }
+            Driver::RetryFailed.new(max_attempts: max_attempts)
           end
         end
       end

data/lib/datadog/ci/test_retries/strategy/retry_new.rb

@@ -0,0 +1,132 @@
+# frozen_string_literal: true
+
+require_relative "base"
+
+require_relative "../driver/retry_new"
+
+module Datadog
+  module CI
+    module TestRetries
+      module Strategy
+        class RetryNew < Base
+          DEFAULT_TOTAL_TESTS_COUNT = 100
+
+          attr_reader :enabled, :max_attempts_thresholds, :unique_tests_set, :total_limit, :retried_count
+
+          def initialize(
+            enabled:,
+            unique_tests_client:
+          )
+            @enabled = enabled
+            @unique_tests_set = Set.new
+            # total maximum number of new tests to retry (will be set based on the total number of tests in the session)
+            @total_limit = 0
+            @retried_count = 0
+
+            @unique_tests_client = unique_tests_client
+          end
+
+          def covers?(test_span)
+            return false unless @enabled
+
+            if @retried_count >= @total_limit
+              Datadog.logger.debug do
+                "Retry new tests limit reached: [#{@retried_count}] out of [#{@total_limit}]"
+              end
+              @enabled = false
+              mark_test_session_faulty(Datadog::CI.active_test_session)
+            end
+
+            @enabled && !test_span.skipped? && is_new_test?(test_span)
+          end
+
+          def configure(library_settings, test_session)
+            @enabled &&= library_settings.early_flake_detection_enabled?
+
+            return unless @enabled
+
+            # mark early flake detection enabled for test session
+            test_session.set_tag(Ext::Test::TAG_EARLY_FLAKE_ENABLED, "true")
+
+            set_max_attempts_thresholds(library_settings)
+            calculate_total_retries_limit(library_settings, test_session)
+            fetch_known_unique_tests(test_session)
+          end
+
+          def build_driver(test_span)
+            Datadog.logger.debug do
+              "#{test_span.name} is new, will be retried"
+            end
+            @retried_count += 1
+
+            Driver::RetryNew.new(test_span, max_attempts_thresholds: @max_attempts_thresholds)
+          end
+
+          private
+
+          def mark_test_session_faulty(test_session)
+            test_session&.set_tag(Ext::Test::TAG_EARLY_FLAKE_ABORT_REASON, Ext::Test::EARLY_FLAKE_FAULTY)
+          end
+
+          def is_new_test?(test_span)
+            test_id = Utils::TestRun.datadog_test_id(test_span.name, test_span.test_suite_name)
+
+            result = !@unique_tests_set.include?(test_id)
+
+            if result
+              Datadog.logger.debug do
+                "#{test_id} is not found in the unique tests set, it is a new test"
+              end
+            end
+
+            result
+          end
+
+          def set_max_attempts_thresholds(library_settings)
+            @max_attempts_thresholds = library_settings.slow_test_retries
+            Datadog.logger.debug do
+              "Slow test retries thresholds: #{@max_attempts_thresholds.entries}"
+            end
+          end
+
+          def calculate_total_retries_limit(library_settings, test_session)
+            percentage_limit = library_settings.faulty_session_threshold
+            tests_count = test_session.total_tests_count.to_i
+            if tests_count.zero?
+              Datadog.logger.debug do
+                "Total tests count is zero, using default value for the total number of tests: [#{DEFAULT_TOTAL_TESTS_COUNT}]"
+              end
+
+              tests_count = DEFAULT_TOTAL_TESTS_COUNT
+            end
+            @total_limit = (tests_count * percentage_limit / 100.0).ceil
+            Datadog.logger.debug do
+              "Retry new tests total limit is [#{@total_limit}] (#{percentage_limit}%) of #{tests_count}"
+            end
+          end
+
+          def fetch_known_unique_tests(test_session)
+            @unique_tests_set = @unique_tests_client.fetch_unique_tests(test_session)
+            if @unique_tests_set.empty?
+              @enabled = false
+              mark_test_session_faulty(test_session)
+
+              Datadog.logger.warn(
+                "Disabling early flake detection because there are no known tests (possible reason: no test runs in default branch)"
+              )
+            end
+
+            # report how many unique tests were found
+            Datadog.logger.debug do
+              "Found [#{@unique_tests_set.size}] known unique tests"
+            end
+            Utils::Telemetry.distribution(
+              Ext::Telemetry::METRIC_EFD_UNIQUE_TESTS_RESPONSE_TESTS,
+              @unique_tests_set.size.to_f
+            )
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/test_retries/unique_tests_client.rb

@@ -0,0 +1,132 @@
+# frozen_string_literal: true
+
+require "json"
+
+require_relative "../ext/telemetry"
+require_relative "../ext/transport"
+require_relative "../transport/telemetry"
+require_relative "../utils/telemetry"
+require_relative "../utils/test_run"
+
+module Datadog
+  module CI
+    module TestRetries
+      # fetch a list of unique known tests from the backend
+      class UniqueTestsClient
+        class Response
+          def initialize(http_response)
+            @http_response = http_response
+            @json = nil
+          end
+
+          def ok?
+            resp = @http_response
+            !resp.nil? && resp.ok?
+          end
+
+          def tests
+            res = Set.new
+
+            payload
+              .fetch("data", {})
+              .fetch("attributes", {})
+              .fetch("tests", {})
+              .each do |_test_module, suites_hash|
+                suites_hash.each do |test_suite, tests|
+                  tests.each do |test_name|
+                    res << Utils::TestRun.datadog_test_id(test_name, test_suite)
+                  end
+                end
+              end
+
+            res
+          end
+
+          private
+
+          def payload
+            cached = @json
+            return cached unless cached.nil?
+
+            resp = @http_response
+            return @json = {} if resp.nil? || !ok?
+
+            begin
+              @json = JSON.parse(resp.payload)
+            rescue JSON::ParserError => e
+              Datadog.logger.error("Failed to parse unique known tests response payload: #{e}. Payload was: #{resp.payload}")
+              @json = {}
+            end
+          end
+        end
+
+        def initialize(dd_env:, api: nil, config_tags: {})
+          @api = api
+          @dd_env = dd_env
+          @config_tags = config_tags
+        end
+
+        def fetch_unique_tests(test_session)
+          api = @api
+          return Set.new unless api
+
+          request_payload = payload(test_session)
+          Datadog.logger.debug("Fetching unique known tests with request: #{request_payload}")
+
+          http_response = api.api_request(
+            path: Ext::Transport::DD_API_UNIQUE_TESTS_PATH,
+            payload: request_payload
+          )
+
+          Transport::Telemetry.api_requests(
+            Ext::Telemetry::METRIC_EFD_UNIQUE_TESTS_REQUEST,
+            1,
+            compressed: http_response.request_compressed
+          )
+          Utils::Telemetry.distribution(Ext::Telemetry::METRIC_EFD_UNIQUE_TESTS_REQUEST_MS, http_response.duration_ms)
+          Utils::Telemetry.distribution(
+            Ext::Telemetry::METRIC_EFD_UNIQUE_TESTS_RESPONSE_BYTES,
+            http_response.response_size.to_f,
+            {Ext::Telemetry::TAG_RESPONSE_COMPRESSED => http_response.gzipped_content?.to_s}
+          )
+
+          unless http_response.ok?
+            Transport::Telemetry.api_requests_errors(
+              Ext::Telemetry::METRIC_EFD_UNIQUE_TESTS_REQUEST_ERRORS,
+              1,
+              error_type: http_response.telemetry_error_type,
+              status_code: http_response.code
+            )
+          end
+
+          Response.new(http_response).tests
+        end
+
+        private
+
+        def payload(test_session)
+          {
+            "data" => {
+              "id" => Datadog::Core::Environment::Identity.id,
+              "type" => Ext::Transport::DD_API_UNIQUE_TESTS_TYPE,
+              "attributes" => {
+                "repository_url" => test_session.git_repository_url,
+                "service" => test_session.service,
+                "env" => @dd_env,
+                "sha" => test_session.git_commit_sha,
+                "configurations" => {
+                  Ext::Test::TAG_OS_PLATFORM => test_session.os_platform,
+                  Ext::Test::TAG_OS_ARCHITECTURE => test_session.os_architecture,
+                  Ext::Test::TAG_OS_VERSION => test_session.os_version,
+                  Ext::Test::TAG_RUNTIME_NAME => test_session.runtime_name,
+                  Ext::Test::TAG_RUNTIME_VERSION => test_session.runtime_version,
+                  "custom" => @config_tags
+                }
+              }
+            }
+          }.to_json
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/test_session.rb

@@ -12,6 +12,8 @@ module Datadog
     #
     # @public_api
     class TestSession < ConcurrentSpan
+      attr_accessor :total_tests_count
+
       # Finishes the current test session.
       # @return [void]
       def finish