datadog-ci 1.4.0 → 1.5.0

data/lib/datadog/ci/test_optimisation/component.rb

@@ -99,6 +99,7 @@ module Datadog
 
         def start_coverage(test)
           return if !enabled? || !code_coverage?
+
           Telemetry.code_coverage_started(test)
           coverage_collector&.start
         end
@@ -109,13 +110,15 @@ module Datadog
           Telemetry.code_coverage_finished(test)
 
           coverage = coverage_collector&.stop
+
+          # if test was skipped, we discard coverage data
+          return if test.skipped?
+
           if coverage.nil? || coverage.empty?
             Telemetry.code_coverage_is_empty
             return
           end
 
-          return if test.skipped?
-
           test_source_file = test.source_file
 
           # cucumber's gherkin files are not covered by the code coverage collector
@@ -140,8 +143,8 @@ module Datadog
         def mark_if_skippable(test)
           return if !enabled? || !skipping_tests?
 
-          skippable_test_id = Utils::TestRun.skippable_test_id(test.name, test.test_suite_name, test.parameters)
-          if @skippable_tests.include?(skippable_test_id)
+          datadog_test_id = Utils::TestRun.datadog_test_id(test.name, test.test_suite_name, test.parameters)
+          if @skippable_tests.include?(datadog_test_id)
             if forked?
               Datadog.logger.warn { "Intelligent test runner is not supported for forking test runners yet" }
               return
@@ -149,9 +152,9 @@ module Datadog
 
             test.set_tag(Ext::Test::TAG_ITR_SKIPPED_BY_ITR, "true")
 
-            Datadog.logger.debug { "Marked test as skippable: #{skippable_test_id}" }
+            Datadog.logger.debug { "Marked test as skippable: #{datadog_test_id}" }
           else
-            Datadog.logger.debug { "Test is not skippable: #{skippable_test_id}" }
+            Datadog.logger.debug { "Test is not skippable: #{datadog_test_id}" }
           end
         end
 

data/lib/datadog/ci/test_optimisation/skippable.rb

@@ -36,7 +36,7 @@ module Datadog
                 next unless test_data["type"] == Ext::Test::ITR_TEST_SKIPPING_MODE
 
                 attrs = test_data["attributes"] || {}
-                res << Utils::TestRun.skippable_test_id(attrs["name"], attrs["suite"], attrs["parameters"])
+                res << Utils::TestRun.datadog_test_id(attrs["name"], attrs["suite"], attrs["parameters"])
               end
 
             res

data/lib/datadog/ci/test_retries/component.rb

@@ -1,7 +1,15 @@
 # frozen_string_literal: true
 
+require_relative "driver/no_retry"
+require_relative "driver/retry_failed"
+require_relative "driver/retry_new"
+
 require_relative "strategy/no_retry"
 require_relative "strategy/retry_failed"
+require_relative "strategy/retry_new"
+
+require_relative "../ext/telemetry"
+require_relative "../utils/telemetry"
 
 module Datadog
   module CI
@@ -10,73 +18,94 @@ module Datadog
       # - retrying failed tests - improve success rate of CI pipelines
       # - retrying new tests - detect flaky tests as early as possible to prevent them from being merged
       class Component
-        attr_reader :retry_failed_tests_enabled, :retry_failed_tests_max_attempts,
-          :retry_failed_tests_total_limit, :retry_failed_tests_count
+        FIBER_LOCAL_CURRENT_RETRY_DRIVER_KEY = :__dd_current_retry_driver
 
         def initialize(
           retry_failed_tests_enabled:,
           retry_failed_tests_max_attempts:,
-          retry_failed_tests_total_limit:
+          retry_failed_tests_total_limit:,
+          retry_new_tests_enabled:,
+          unique_tests_client:
         )
-          @retry_failed_tests_enabled = retry_failed_tests_enabled
-          @retry_failed_tests_max_attempts = retry_failed_tests_max_attempts
-          @retry_failed_tests_total_limit = retry_failed_tests_total_limit
-          # counter that store the current number of failed tests retried
-          @retry_failed_tests_count = 0
+          no_retries_strategy = Strategy::NoRetry.new
+
+          retry_failed_strategy = Strategy::RetryFailed.new(
+            enabled: retry_failed_tests_enabled,
+            max_attempts: retry_failed_tests_max_attempts,
+            total_limit: retry_failed_tests_total_limit
+          )
 
+          retry_new_strategy = Strategy::RetryNew.new(
+            enabled: retry_new_tests_enabled,
+            unique_tests_client: unique_tests_client
+          )
+
+          # order is important, we should try to retry new tests first
+          @retry_strategies = [retry_new_strategy, retry_failed_strategy, no_retries_strategy]
           @mutex = Mutex.new
         end
 
-        def configure(library_settings)
-          @retry_failed_tests_enabled &&= library_settings.flaky_test_retries_enabled?
+        def configure(library_settings, test_session)
+          # let all strategies configure themselves
+          @retry_strategies.each do |strategy|
+            strategy.configure(library_settings, test_session)
+          end
         end
 
         def with_retries(&block)
-          # @type var retry_strategy: Strategy::Base
-          retry_strategy = nil
-
-          test_finished_callback = lambda do |test_span|
-            if retry_strategy.nil?
-              # we always run test at least once and after first pass create a correct retry strategy
-              retry_strategy = build_strategy(test_span)
-            else
-              # after each retry we record the result, strategy will decide if we should retry again
-              retry_strategy&.record_retry(test_span)
-            end
-          end
-
-          test_visibility_component.set_test_finished_callback(test_finished_callback)
+          reset_retries!
 
           loop do
             yield
 
-            break unless retry_strategy&.should_retry?
+            break unless should_retry?
           end
         ensure
-          test_visibility_component.remove_test_finished_callback
+          reset_retries!
         end
 
-        def build_strategy(test_span)
+        def build_driver(test_span)
           @mutex.synchronize do
-            if should_retry_failed_test?(test_span)
-              Datadog.logger.debug("Failed test retry starts")
-              @retry_failed_tests_count += 1
-
-              Strategy::RetryFailed.new(max_attempts: @retry_failed_tests_max_attempts)
-            else
-              Strategy::NoRetry.new
-            end
+            # find the first strategy that covers the test span and let it build the driver
+            strategy = @retry_strategies.find { |strategy| strategy.covers?(test_span) }
+
+            raise "No retry strategy found for test span: #{test_span.name}" if strategy.nil?
+
+            strategy.build_driver(test_span)
           end
         end
 
+        def record_test_finished(test_span)
+          if current_retry_driver.nil?
+            # we always run test at least once and after the first pass create a correct retry driver
+            self.current_retry_driver = build_driver(test_span)
+          else
+            # after each retry we record the result, the driver will decide if we should retry again
+            current_retry_driver&.record_retry(test_span)
+          end
+        end
+
+        def record_test_span_duration(tracer_span)
+          current_retry_driver&.record_duration(tracer_span.duration)
+        end
+
+        # this API is targeted on Cucumber instrumentation or any other that cannot leverage #with_retries method
+        def reset_retries!
+          self.current_retry_driver = nil
+        end
+
+        def should_retry?
+          !!current_retry_driver&.should_retry?
+        end
+
         private
 
-        def should_retry_failed_test?(test_span)
-          @retry_failed_tests_enabled && !!test_span&.failed? && @retry_failed_tests_count < @retry_failed_tests_total_limit
+        def current_retry_driver
+          Thread.current[FIBER_LOCAL_CURRENT_RETRY_DRIVER_KEY]
         end
 
-        def test_visibility_component
-          Datadog.send(:components).test_visibility
+        def current_retry_driver=(driver)
+          Thread.current[FIBER_LOCAL_CURRENT_RETRY_DRIVER_KEY] = driver
         end
       end
     end

data/lib/datadog/ci/test_retries/driver/base.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Datadog
+  module CI
+    module TestRetries
+      module Driver
+        # Driver is the class responsible for the current test retry mechanism.
+        # It receives signals about each retry execution and steers the current retry strategy.
+        class Base
+          def should_retry?
+            false
+          end
+
+          def record_retry(test_span)
+            test_span&.set_tag(Ext::Test::TAG_IS_RETRY, "true")
+          end
+
+          # duration in float seconds
+          def record_duration(duration)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/test_retries/driver/no_retry.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+require_relative "base"
+
+module Datadog
+  module CI
+    module TestRetries
+      module Driver
+        class NoRetry < Base
+          def record_retry(test_span)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/test_retries/driver/retry_failed.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require_relative "base"
+
+require_relative "../../ext/test"
+
+module Datadog
+  module CI
+    module TestRetries
+      module Driver
+        class RetryFailed < Base
+          attr_reader :max_attempts
+
+          def initialize(max_attempts:)
+            @max_attempts = max_attempts
+
+            @attempts = 0
+            @passed_once = false
+          end
+
+          def should_retry?
+            @attempts < @max_attempts && !@passed_once
+          end
+
+          def record_retry(test_span)
+            super
+
+            @attempts += 1
+            @passed_once = true if test_span&.passed?
+
+            Datadog.logger.debug { "Retry Attempts [#{@attempts} / #{@max_attempts}], Passed: [#{@passed_once}]" }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/test_retries/driver/retry_new.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+require_relative "base"
+
+require_relative "../../ext/test"
+
+module Datadog
+  module CI
+    module TestRetries
+      module Driver
+        # retry every new test up to 10 times (early flake detection)
+        class RetryNew < Base
+          def initialize(test_span, max_attempts_thresholds:)
+            @max_attempts_thresholds = max_attempts_thresholds
+            @attempts = 0
+            # will be changed based on test span duration
+            @max_attempts = 10
+
+            mark_new_test(test_span)
+          end
+
+          def should_retry?
+            @attempts < @max_attempts
+          end
+
+          def record_retry(test_span)
+            super
+
+            @attempts += 1
+            mark_new_test(test_span)
+
+            Datadog.logger.debug { "Retry Attempts [#{@attempts} / #{@max_attempts}]" }
+          end
+
+          def record_duration(duration)
+            @max_attempts = @max_attempts_thresholds.max_attempts_for_duration(duration)
+
+            Datadog.logger.debug { "Recorded test duration of [#{duration}], new Max Attempts value is [#{@max_attempts}]" }
+          end
+
+          private
+
+          def mark_new_test(test_span)
+            test_span.set_tag(Ext::Test::TAG_IS_NEW, "true")
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/test_retries/null_component.rb

@@ -6,13 +6,7 @@ module Datadog
   module CI
     module TestRetries
       class NullComponent < Component
-        attr_reader :retry_failed_tests_enabled, :retry_failed_tests_max_attempts, :retry_failed_tests_total_limit
-
         def initialize
-          # enabled only by remote settings
-          @retry_failed_tests_enabled = false
-          @retry_failed_tests_max_attempts = 0
-          @retry_failed_tests_total_limit = 0
         end
 
         def configure(library_settings)
@@ -22,6 +16,13 @@ module Datadog
           no_action = proc {}
           yield no_action
         end
+
+        def reset_retries!
+        end
+
+        def should_retry?
+          false
+        end
       end
     end
   end

data/lib/datadog/ci/test_retries/strategy/base.rb

@@ -1,16 +1,23 @@
 # frozen_string_literal: true
 
+require_relative "../driver/no_retry"
+
 module Datadog
   module CI
     module TestRetries
       module Strategy
+        # Strategies are subcomponents of the retry mechanism. They are responsible for
+        # determining which tests should be retried and how.
         class Base
-          def should_retry?
-            false
+          def covers?(test_span)
+            true
+          end
+
+          def configure(_library_settings, _test_session)
           end
 
-          def record_retry(test_span)
-            test_span&.set_tag(Ext::Test::TAG_IS_RETRY, "true")
+          def build_driver(_test_span)
+            Driver::NoRetry.new
           end
         end
       end

data/lib/datadog/ci/test_retries/strategy/no_retry.rb

@@ -7,8 +7,6 @@ module Datadog
     module TestRetries
       module Strategy
         class NoRetry < Base
-          def record_retry(test_span)
-          end
         end
       end
     end

data/lib/datadog/ci/test_retries/strategy/retry_failed.rb

@@ -2,33 +2,50 @@
 
 require_relative "base"
 
-require_relative "../../ext/test"
+require_relative "../driver/retry_failed"
 
 module Datadog
   module CI
     module TestRetries
       module Strategy
         class RetryFailed < Base
-          attr_reader :max_attempts
-
-          def initialize(max_attempts:)
+          attr_reader :enabled, :max_attempts,
+            :total_limit, :retried_count
+
+          def initialize(
+            enabled:,
+            max_attempts:,
+            total_limit:
+          )
+            @enabled = enabled
             @max_attempts = max_attempts
+            @total_limit = total_limit
+            @retried_count = 0
+          end
+
+          def covers?(test_span)
+            return false unless @enabled
+
+            if @retried_count >= @total_limit
+              Datadog.logger.debug do
+                "Retry failed tests limit reached: [#{@retried_count}] out of [#{@total_limit}]"
+              end
+              @enabled = false
+            end
 
-            @attempts = 0
-            @passed_once = false
+            @enabled && !!test_span&.failed?
           end
 
-          def should_retry?
-            @attempts < @max_attempts && !@passed_once
+          def configure(library_settings, test_session)
+            @enabled &&= library_settings.flaky_test_retries_enabled?
           end
 
-          def record_retry(test_span)
-            super
+          def build_driver(test_span)
+            Datadog.logger.debug { "#{test_span.name} failed, will be retried" }
 
-            @attempts += 1
-            @passed_once = true if test_span&.passed?
+            @retried_count += 1
 
-            Datadog.logger.debug { "Retry Attempts [#{@attempts} / #{@max_attempts}], Passed: [#{@passed_once}]" }
+            Driver::RetryFailed.new(max_attempts: max_attempts)
           end
         end
       end

data/lib/datadog/ci/test_retries/strategy/retry_new.rb

@@ -0,0 +1,132 @@
+# frozen_string_literal: true
+
+require_relative "base"
+
+require_relative "../driver/retry_new"
+
+module Datadog
+  module CI
+    module TestRetries
+      module Strategy
+        class RetryNew < Base
+          DEFAULT_TOTAL_TESTS_COUNT = 100
+
+          attr_reader :enabled, :max_attempts_thresholds, :unique_tests_set, :total_limit, :retried_count
+
+          def initialize(
+            enabled:,
+            unique_tests_client:
+          )
+            @enabled = enabled
+            @unique_tests_set = Set.new
+            # total maximum number of new tests to retry (will be set based on the total number of tests in the session)
+            @total_limit = 0
+            @retried_count = 0
+
+            @unique_tests_client = unique_tests_client
+          end
+
+          def covers?(test_span)
+            return false unless @enabled
+
+            if @retried_count >= @total_limit
+              Datadog.logger.debug do
+                "Retry new tests limit reached: [#{@retried_count}] out of [#{@total_limit}]"
+              end
+              @enabled = false
+              mark_test_session_faulty(Datadog::CI.active_test_session)
+            end
+
+            @enabled && !test_span.skipped? && is_new_test?(test_span)
+          end
+
+          def configure(library_settings, test_session)
+            @enabled &&= library_settings.early_flake_detection_enabled?
+
+            return unless @enabled
+
+            # mark early flake detection enabled for test session
+            test_session.set_tag(Ext::Test::TAG_EARLY_FLAKE_ENABLED, "true")
+
+            set_max_attempts_thresholds(library_settings)
+            calculate_total_retries_limit(library_settings, test_session)
+            fetch_known_unique_tests(test_session)
+          end
+
+          def build_driver(test_span)
+            Datadog.logger.debug do
+              "#{test_span.name} is new, will be retried"
+            end
+            @retried_count += 1
+
+            Driver::RetryNew.new(test_span, max_attempts_thresholds: @max_attempts_thresholds)
+          end
+
+          private
+
+          def mark_test_session_faulty(test_session)
+            test_session&.set_tag(Ext::Test::TAG_EARLY_FLAKE_ABORT_REASON, Ext::Test::EARLY_FLAKE_FAULTY)
+          end
+
+          def is_new_test?(test_span)
+            test_id = Utils::TestRun.datadog_test_id(test_span.name, test_span.test_suite_name)
+
+            result = !@unique_tests_set.include?(test_id)
+
+            if result
+              Datadog.logger.debug do
+                "#{test_id} is not found in the unique tests set, it is a new test"
+              end
+            end
+
+            result
+          end
+
+          def set_max_attempts_thresholds(library_settings)
+            @max_attempts_thresholds = library_settings.slow_test_retries
+            Datadog.logger.debug do
+              "Slow test retries thresholds: #{@max_attempts_thresholds.entries}"
+            end
+          end
+
+          def calculate_total_retries_limit(library_settings, test_session)
+            percentage_limit = library_settings.faulty_session_threshold
+            tests_count = test_session.total_tests_count.to_i
+            if tests_count.zero?
+              Datadog.logger.debug do
+                "Total tests count is zero, using default value for the total number of tests: [#{DEFAULT_TOTAL_TESTS_COUNT}]"
+              end
+
+              tests_count = DEFAULT_TOTAL_TESTS_COUNT
+            end
+            @total_limit = (tests_count * percentage_limit / 100.0).ceil
+            Datadog.logger.debug do
+              "Retry new tests total limit is [#{@total_limit}] (#{percentage_limit}%) of #{tests_count}"
+            end
+          end
+
+          def fetch_known_unique_tests(test_session)
+            @unique_tests_set = @unique_tests_client.fetch_unique_tests(test_session)
+            if @unique_tests_set.empty?
+              @enabled = false
+              mark_test_session_faulty(test_session)
+
+              Datadog.logger.warn(
+                "Disabling early flake detection because there are no known tests (possible reason: no test runs in default branch)"
+              )
+            end
+
+            # report how many unique tests were found
+            Datadog.logger.debug do
+              "Found [#{@unique_tests_set.size}] known unique tests"
+            end
+            Utils::Telemetry.distribution(
+              Ext::Telemetry::METRIC_EFD_UNIQUE_TESTS_RESPONSE_TESTS,
+              @unique_tests_set.size.to_f
+            )
+          end
+        end
+      end
+    end
+  end
+end