datadog-ci 1.3.0 → 1.4.0

data/lib/datadog/ci/remote/component.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module Datadog
+  module CI
+    module Remote
+      # Remote configuration component.
+      # Responsible for fetching library settings and configuring the library accordingly.
+      class Component
+        def initialize(library_settings_client:)
+          @library_settings_client = library_settings_client
+        end
+
+        # called on test session start, uses test session info to send configuration request to the backend
+        def configure(test_session)
+          library_configuration = @library_settings_client.fetch(test_session)
+          # sometimes we can skip code coverage for default branch if there are no changes in the repository
+          # backend needs git metadata uploaded for this test session to check if we can skip code coverage
+          if library_configuration.require_git?
+            Datadog.logger.debug { "Library configuration endpoint requires git upload to be finished, waiting..." }
+            git_tree_upload_worker.wait_until_done
+
+            Datadog.logger.debug { "Requesting library configuration again..." }
+            library_configuration = @library_settings_client.fetch(test_session)
+
+            if library_configuration.require_git?
+              Datadog.logger.debug { "git metadata upload did not complete in time when configuring library" }
+            end
+          end
+
+          test_optimisation.configure(library_configuration, test_session)
+          test_retries.configure(library_configuration)
+        end
+
+        private
+
+        def test_optimisation
+          Datadog.send(:components).test_optimisation
+        end
+
+        def test_retries
+          Datadog.send(:components).test_retries
+        end
+
+        def git_tree_upload_worker
+          Datadog.send(:components).git_tree_upload_worker
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/remote/library_settings.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+require "json"
+
+require_relative "../ext/telemetry"
+require_relative "../ext/transport"
+require_relative "../transport/telemetry"
+require_relative "../utils/parsing"
+
+module Datadog
+  module CI
+    module Remote
+      # Wrapper around the settings HTTP response
+      class LibrarySettings
+        def initialize(http_response)
+          @http_response = http_response
+          @json = nil
+        end
+
+        def ok?
+          resp = @http_response
+          !resp.nil? && resp.ok?
+        end
+
+        def payload
+          cached = @json
+          return cached unless cached.nil?
+
+          resp = @http_response
+          return @json = default_payload if resp.nil? || !ok?
+
+          begin
+            @json = JSON.parse(resp.payload).dig(*Ext::Transport::DD_API_SETTINGS_RESPONSE_DIG_KEYS) ||
+              default_payload
+          rescue JSON::ParserError => e
+            Datadog.logger.error("Failed to parse settings response payload: #{e}. Payload was: #{resp.payload}")
+
+            Transport::Telemetry.api_requests_errors(
+              Ext::Telemetry::METRIC_GIT_REQUESTS_SETTINGS_ERRORS,
+              1,
+              error_type: "invalid_json",
+              status_code: nil
+            )
+
+            @json = default_payload
+          end
+        end
+
+        def require_git?
+          return @require_git if defined?(@require_git)
+
+          @require_git = bool(Ext::Transport::DD_API_SETTINGS_RESPONSE_REQUIRE_GIT_KEY)
+        end
+
+        def itr_enabled?
+          return @itr_enabled if defined?(@itr_enabled)
+
+          @itr_enabled = bool(Ext::Transport::DD_API_SETTINGS_RESPONSE_ITR_ENABLED_KEY)
+        end
+
+        def code_coverage_enabled?
+          return @code_coverage_enabled if defined?(@code_coverage_enabled)
+
+          @code_coverage_enabled = bool(Ext::Transport::DD_API_SETTINGS_RESPONSE_CODE_COVERAGE_KEY)
+        end
+
+        def tests_skipping_enabled?
+          return @tests_skipping_enabled if defined?(@tests_skipping_enabled)
+
+          @tests_skipping_enabled = bool(Ext::Transport::DD_API_SETTINGS_RESPONSE_TESTS_SKIPPING_KEY)
+        end
+
+        def flaky_test_retries_enabled?
+          return @flaky_test_retries_enabled if defined?(@flaky_test_retries_enabled)
+
+          @flaky_test_retries_enabled = bool(Ext::Transport::DD_API_SETTINGS_RESPONSE_FLAKY_TEST_RETRIES_KEY)
+        end
+
+        private
+
+        def bool(key)
+          Utils::Parsing.convert_to_bool(payload.fetch(key, false))
+        end
+
+        def default_payload
+          Ext::Transport::DD_API_SETTINGS_RESPONSE_DEFAULT
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/{transport/remote_settings_api.rb → remote/library_settings_client.rb}

@@ -4,73 +4,28 @@ require "json"
 
 require "datadog/core/environment/identity"
 
+require_relative "library_settings"
+
+require_relative "../ext/test"
 require_relative "../ext/telemetry"
 require_relative "../ext/transport"
 require_relative "../transport/telemetry"
-require_relative "../utils/parsing"
 require_relative "../utils/telemetry"
 
 module Datadog
   module CI
-    module Transport
-      # Datadog API client
+    module Remote
       # Calls settings endpoint to fetch library settings for given service and env
-      class RemoteSettingsApi
-        class Response
-          def initialize(http_response)
-            @http_response = http_response
-            @json = nil
-          end
-
-          def ok?
-            resp = @http_response
-            !resp.nil? && resp.ok?
-          end
-
-          def payload
-            cached = @json
-            return cached unless cached.nil?
-
-            resp = @http_response
-            return @json = default_payload if resp.nil? || !ok?
-
-            begin
-              @json = JSON.parse(resp.payload).dig(*Ext::Transport::DD_API_SETTINGS_RESPONSE_DIG_KEYS) ||
-                default_payload
-            rescue JSON::ParserError => e
-              Datadog.logger.error("Failed to parse settings response payload: #{e}. Payload was: #{resp.payload}")
-
-              Transport::Telemetry.api_requests_errors(
-                Ext::Telemetry::METRIC_GIT_REQUESTS_SETTINGS_ERRORS,
-                1,
-                error_type: "invalid_json",
-                status_code: nil
-              )
-
-              @json = default_payload
-            end
-          end
-
-          def require_git?
-            Utils::Parsing.convert_to_bool(payload[Ext::Transport::DD_API_SETTINGS_RESPONSE_REQUIRE_GIT_KEY])
-          end
-
-          private
-
-          def default_payload
-            Ext::Transport::DD_API_SETTINGS_RESPONSE_DEFAULT
-          end
-        end
-
+      class LibrarySettingsClient
         def initialize(dd_env:, api: nil, config_tags: {})
           @api = api
           @dd_env = dd_env
           @config_tags = config_tags || {}
         end
 
-        def fetch_library_settings(test_session)
+        def fetch(test_session)
           api = @api
-          return Response.new(nil) unless api
+          return LibrarySettings.new(nil) unless api
 
           request_payload = payload(test_session)
           Datadog.logger.debug("Fetching library settings with request: #{request_payload}")
@@ -96,18 +51,18 @@ module Datadog
             )
           end
 
-          response = Response.new(http_response)
+          library_settings = LibrarySettings.new(http_response)
 
           Utils::Telemetry.inc(
             Ext::Telemetry::METRIC_GIT_REQUESTS_SETTINGS_RESPONSE,
             1,
             {
-              Ext::Telemetry::TAG_COVERAGE_ENABLED => response.payload[Ext::Transport::DD_API_SETTINGS_RESPONSE_CODE_COVERAGE_KEY],
-              Ext::Telemetry::TAG_ITR_SKIP_ENABLED => response.payload[Ext::Transport::DD_API_SETTINGS_RESPONSE_TESTS_SKIPPING_KEY]
+              Ext::Telemetry::TAG_COVERAGE_ENABLED => library_settings.code_coverage_enabled?.to_s,
+              Ext::Telemetry::TAG_ITR_SKIP_ENABLED => library_settings.tests_skipping_enabled?.to_s
             }
           )
 
-          response
+          library_settings
         end
 
         private

data/lib/datadog/ci/test.rb

@@ -142,7 +142,14 @@ module Datadog
       private
 
       def record_test_result(datadog_status)
-        test_suite&.record_test_result(datadog_status)
+        test_id = Utils::TestRun.skippable_test_id(name, test_suite_name, parameters)
+
+        # if this test was already executed in this test suite, mark it as retried
+        if test_suite&.test_executed?(test_id)
+          set_tag(Ext::Test::TAG_IS_RETRY, "true")
+        end
+
+        test_suite&.record_test_result(test_id, datadog_status)
       end
     end
   end

data/lib/datadog/ci/test_optimisation/component.rb

@@ -6,7 +6,6 @@ require "datadog/core/utils/forking"
 
 require_relative "../ext/test"
 require_relative "../ext/telemetry"
-require_relative "../ext/transport"
 
 require_relative "../git/local_repository"
 
@@ -65,24 +64,17 @@ module Datadog
           Datadog.logger.debug("TestOptimisation initialized with enabled: #{@enabled}")
         end
 
-        def configure(remote_configuration, test_session:, git_tree_upload_worker:)
+        def configure(remote_configuration, test_session)
+          return unless enabled?
+
           Datadog.logger.debug("Configuring TestOptimisation with remote configuration: #{remote_configuration}")
 
-          @enabled = Utils::Parsing.convert_to_bool(
-            remote_configuration.fetch(Ext::Transport::DD_API_SETTINGS_RESPONSE_ITR_ENABLED_KEY, false)
-          )
-          @test_skipping_enabled = @enabled && Utils::Parsing.convert_to_bool(
-            remote_configuration.fetch(Ext::Transport::DD_API_SETTINGS_RESPONSE_TESTS_SKIPPING_KEY, false)
-          )
-          @code_coverage_enabled = @enabled && Utils::Parsing.convert_to_bool(
-            remote_configuration.fetch(Ext::Transport::DD_API_SETTINGS_RESPONSE_CODE_COVERAGE_KEY, false)
-          )
+          @enabled = remote_configuration.itr_enabled?
+          @test_skipping_enabled = @enabled && remote_configuration.tests_skipping_enabled?
+          @code_coverage_enabled = @enabled && remote_configuration.code_coverage_enabled?
 
           test_session.set_tag(Ext::Test::TAG_ITR_TEST_SKIPPING_ENABLED, @test_skipping_enabled)
-          # currently we set this tag when ITR requires collecting code coverage
-          # this will change as soon as we implement total code coverage support in this library
           test_session.set_tag(Ext::Test::TAG_CODE_COVERAGE_ENABLED, @code_coverage_enabled)
-
           # we skip tests, not suites
           test_session.set_tag(Ext::Test::TAG_ITR_TEST_SKIPPING_TYPE, Ext::Test::ITR_TEST_SKIPPING_MODE)
 
@@ -90,7 +82,7 @@ module Datadog
 
           Datadog.logger.debug("Configured TestOptimisation with enabled: #{@enabled}, skipping_tests: #{@test_skipping_enabled}, code_coverage: #{@code_coverage_enabled}")
 
-          fetch_skippable_tests(test_session: test_session, git_tree_upload_worker: git_tree_upload_worker)
+          fetch_skippable_tests(test_session)
         end
 
         def enabled?
@@ -225,7 +217,7 @@ module Datadog
           coverage[absolute_test_source_file_path] = true
         end
 
-        def fetch_skippable_tests(test_session:, git_tree_upload_worker:)
+        def fetch_skippable_tests(test_session)
           return unless skipping_tests?
 
           # we can only request skippable tests if git metadata is already uploaded
@@ -248,6 +240,10 @@ module Datadog
         def code_coverage_mode
           @use_single_threaded_coverage ? :single : :multi
         end
+
+        def git_tree_upload_worker
+          Datadog.send(:components).git_tree_upload_worker
+        end
       end
     end
   end

data/lib/datadog/ci/test_retries/component.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+require_relative "strategy/no_retry"
+require_relative "strategy/retry_failed"
+
+module Datadog
+  module CI
+    module TestRetries
+      # Encapsulates the logic to enable test retries, including:
+      # - retrying failed tests - improve success rate of CI pipelines
+      # - retrying new tests - detect flaky tests as early as possible to prevent them from being merged
+      class Component
+        attr_reader :retry_failed_tests_enabled, :retry_failed_tests_max_attempts,
+          :retry_failed_tests_total_limit, :retry_failed_tests_count
+
+        def initialize(
+          retry_failed_tests_enabled:,
+          retry_failed_tests_max_attempts:,
+          retry_failed_tests_total_limit:
+        )
+          @retry_failed_tests_enabled = retry_failed_tests_enabled
+          @retry_failed_tests_max_attempts = retry_failed_tests_max_attempts
+          @retry_failed_tests_total_limit = retry_failed_tests_total_limit
+          # counter that store the current number of failed tests retried
+          @retry_failed_tests_count = 0
+
+          @mutex = Mutex.new
+        end
+
+        def configure(library_settings)
+          @retry_failed_tests_enabled &&= library_settings.flaky_test_retries_enabled?
+        end
+
+        def with_retries(&block)
+          # @type var retry_strategy: Strategy::Base
+          retry_strategy = nil
+
+          test_finished_callback = lambda do |test_span|
+            if retry_strategy.nil?
+              # we always run test at least once and after first pass create a correct retry strategy
+              retry_strategy = build_strategy(test_span)
+            else
+              # after each retry we record the result, strategy will decide if we should retry again
+              retry_strategy&.record_retry(test_span)
+            end
+          end
+
+          test_visibility_component.set_test_finished_callback(test_finished_callback)
+
+          loop do
+            yield
+
+            break unless retry_strategy&.should_retry?
+          end
+        ensure
+          test_visibility_component.remove_test_finished_callback
+        end
+
+        def build_strategy(test_span)
+          @mutex.synchronize do
+            if should_retry_failed_test?(test_span)
+              Datadog.logger.debug("Failed test retry starts")
+              @retry_failed_tests_count += 1
+
+              Strategy::RetryFailed.new(max_attempts: @retry_failed_tests_max_attempts)
+            else
+              Strategy::NoRetry.new
+            end
+          end
+        end
+
+        private
+
+        def should_retry_failed_test?(test_span)
+          @retry_failed_tests_enabled && !!test_span&.failed? && @retry_failed_tests_count < @retry_failed_tests_total_limit
+        end
+
+        def test_visibility_component
+          Datadog.send(:components).test_visibility
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/test_retries/null_component.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require_relative "component"
+
+module Datadog
+  module CI
+    module TestRetries
+      class NullComponent < Component
+        attr_reader :retry_failed_tests_enabled, :retry_failed_tests_max_attempts, :retry_failed_tests_total_limit
+
+        def initialize
+          # enabled only by remote settings
+          @retry_failed_tests_enabled = false
+          @retry_failed_tests_max_attempts = 0
+          @retry_failed_tests_total_limit = 0
+        end
+
+        def configure(library_settings)
+        end
+
+        def with_retries(&block)
+          no_action = proc {}
+          yield no_action
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/test_retries/strategy/base.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Datadog
+  module CI
+    module TestRetries
+      module Strategy
+        class Base
+          def should_retry?
+            false
+          end
+
+          def record_retry(test_span)
+            test_span&.set_tag(Ext::Test::TAG_IS_RETRY, "true")
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/test_retries/strategy/no_retry.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+require_relative "base"
+
+module Datadog
+  module CI
+    module TestRetries
+      module Strategy
+        class NoRetry < Base
+          def record_retry(test_span)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/test_retries/strategy/retry_failed.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require_relative "base"
+
+require_relative "../../ext/test"
+
+module Datadog
+  module CI
+    module TestRetries
+      module Strategy
+        class RetryFailed < Base
+          attr_reader :max_attempts
+
+          def initialize(max_attempts:)
+            @max_attempts = max_attempts
+
+            @attempts = 0
+            @passed_once = false
+          end
+
+          def should_retry?
+            @attempts < @max_attempts && !@passed_once
+          end
+
+          def record_retry(test_span)
+            super
+
+            @attempts += 1
+            @passed_once = true if test_span&.passed?
+
+            Datadog.logger.debug { "Retry Attempts [#{@attempts} / #{@max_attempts}], Passed: [#{@passed_once}]" }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/test_suite.rb

@@ -16,7 +16,9 @@ module Datadog
       def initialize(tracer_span)
         super
 
-        @test_suite_stats = Hash.new(0)
+        # counts how many times every test in this suite was executed with each status:
+        #   { "MySuite.mytest.a:1" => { "pass" => 3, "fail" => 2 } }
+        @execution_stats_per_test = {}
       end
 
       # Finishes this test suite.
@@ -33,42 +35,61 @@ module Datadog
       end
 
       # @internal
-      def record_test_result(datadog_test_status)
+      def record_test_result(test_id, datadog_test_status)
         synchronize do
-          @test_suite_stats[datadog_test_status] += 1
+          @execution_stats_per_test[test_id] ||= Hash.new(0)
+          @execution_stats_per_test[test_id][datadog_test_status] += 1
         end
       end
 
       # @internal
-      def passed_tests_count
+      def any_passed?
         synchronize do
-          @test_suite_stats[Ext::Test::Status::PASS]
+          @execution_stats_per_test.any? do |_, stats|
+            stats[Ext::Test::Status::PASS] > 0
+          end
         end
       end
 
       # @internal
-      def skipped_tests_count
+      def test_executed?(test_id)
         synchronize do
-          @test_suite_stats[Ext::Test::Status::SKIP]
+          @execution_stats_per_test.key?(test_id)
         end
       end
 
-      # @internal
-      def failed_tests_count
+      private
+
+      def set_status_from_stats!
         synchronize do
-          @test_suite_stats[Ext::Test::Status::FAIL]
+          # count how many tests passed, failed and skipped
+          test_suite_stats = @execution_stats_per_test.each_with_object(Hash.new(0)) do |(_test_id, stats), acc|
+            acc[derive_test_status_from_execution_stats(stats)] += 1
+          end
+
+          # test suite is considered failed if at least one test failed
+          if test_suite_stats[Ext::Test::Status::FAIL] > 0
+            failed!
+          # if there are no failures and no passes, it is skipped
+          elsif test_suite_stats[Ext::Test::Status::PASS] == 0
+            skipped!
+          # otherwise we consider it passed
+          else
+            passed!
+          end
         end
       end
 
-      private
-
-      def set_status_from_stats!
-        if failed_tests_count > 0
-          failed!
-        elsif passed_tests_count == 0
-          skipped!
+      def derive_test_status_from_execution_stats(test_execution_stats)
+        # test is passed if it passed at least once
+        if test_execution_stats[Ext::Test::Status::PASS] > 0
+          Ext::Test::Status::PASS
+        # if test was never passed, it is failed if it failed at least once
+        elsif test_execution_stats[Ext::Test::Status::FAIL] > 0
+          Ext::Test::Status::FAIL
+        # otherwise it is skipped
         else
-          passed!
+          Ext::Test::Status::SKIP
         end
       end
     end