datadog-ci 1.11.0 → 1.13.0

data/lib/datadog/ci/test_retries/component.rb

@@ -7,6 +7,7 @@ require_relative "driver/retry_new"
 require_relative "strategy/no_retry"
 require_relative "strategy/retry_failed"
 require_relative "strategy/retry_new"
+require_relative "strategy/retry_flaky_fixed"
 
 require_relative "../ext/telemetry"
 require_relative "../utils/telemetry"
@@ -25,23 +26,33 @@ module Datadog
           retry_failed_tests_max_attempts:,
           retry_failed_tests_total_limit:,
           retry_new_tests_enabled:,
-          unique_tests_client:
+          retry_flaky_fixed_tests_enabled:,
+          retry_flaky_fixed_tests_max_attempts:
         )
           no_retries_strategy = Strategy::NoRetry.new
 
-          retry_failed_strategy = Strategy::RetryFailed.new(
+          @retry_failed_strategy = Strategy::RetryFailed.new(
             enabled: retry_failed_tests_enabled,
             max_attempts: retry_failed_tests_max_attempts,
             total_limit: retry_failed_tests_total_limit
           )
 
-          retry_new_strategy = Strategy::RetryNew.new(
-            enabled: retry_new_tests_enabled,
-            unique_tests_client: unique_tests_client
+          @retry_new_strategy = Strategy::RetryNew.new(
+            enabled: retry_new_tests_enabled
           )
 
-          # order is important, we should try to retry new tests first
-          @retry_strategies = [retry_new_strategy, retry_failed_strategy, no_retries_strategy]
+          retry_flaky_fixed_strategy = Strategy::RetryFlakyFixed.new(
+            enabled: retry_flaky_fixed_tests_enabled,
+            max_attempts: retry_flaky_fixed_tests_max_attempts
+          )
+
+          # order is important, we apply the first matching strategy
+          @retry_strategies = [
+            retry_flaky_fixed_strategy,
+            @retry_new_strategy,
+            @retry_failed_strategy,
+            no_retries_strategy
+          ]
           @mutex = Mutex.new
         end
 
@@ -82,6 +93,8 @@ module Datadog
           else
             # after each retry we record the result, the driver will decide if we should retry again
             current_retry_driver&.record_retry(test_span)
+
+            tag_last_retry(test_span) unless should_retry?
           end
         end
 
@@ -94,10 +107,27 @@ module Datadog
           self.current_retry_driver = nil
         end
 
+        def tag_last_retry(test_span)
+          test_span&.set_tag(Ext::Test::TAG_HAS_FAILED_ALL_RETRIES, "true") if test_span&.all_executions_failed?
+
+          # if we are attempting to fix the test and all retries passed, we indicate that the fix might have worked
+          if test_span&.attempt_to_fix? && test_span.all_executions_passed?
+            test_span&.set_tag(Ext::Test::TAG_ATTEMPT_TO_FIX_PASSED, "true")
+          end
+        end
+
         def should_retry?
           !!current_retry_driver&.should_retry?
         end
 
+        def auto_test_retries_feature_enabled
+          @retry_failed_strategy.enabled
+        end
+
+        def early_flake_detection_feature_enabled
+          @retry_new_strategy.enabled
+        end
+
         private
 
         def current_retry_driver

data/lib/datadog/ci/test_retries/driver/base.rb

@@ -13,11 +13,16 @@ module Datadog
 
           def record_retry(test_span)
             test_span&.set_tag(Ext::Test::TAG_IS_RETRY, "true")
+            test_span&.set_tag(Ext::Test::TAG_RETRY_REASON, retry_reason)
           end
 
           # duration in float seconds
           def record_duration(duration)
           end
+
+          def retry_reason
+            "unknown"
+          end
         end
       end
     end

data/lib/datadog/ci/test_retries/driver/retry_failed.rb

@@ -30,6 +30,10 @@ module Datadog
 
             Datadog.logger.debug { "Retry Attempts [#{@attempts} / #{@max_attempts}], Passed: [#{@passed_once}]" }
           end
+
+          def retry_reason
+            Ext::Test::RetryReason::RETRY_FAILED
+          end
         end
       end
     end

data/lib/datadog/ci/test_retries/driver/retry_flaky_fixed.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require_relative "base"
+
+require_relative "../../ext/test"
+
+module Datadog
+  module CI
+    module TestRetries
+      module Driver
+        class RetryFlakyFixed < Base
+          attr_reader :max_attempts
+
+          def initialize(max_attempts:)
+            @attempts = 0
+            @max_attempts = max_attempts
+          end
+
+          def should_retry?
+            @attempts < @max_attempts
+          end
+
+          def record_retry(test_span)
+            super
+
+            @attempts += 1
+
+            Datadog.logger.debug { "Retry Attempts [#{@attempts} / #{@max_attempts}]" }
+          end
+
+          def retry_reason
+            Ext::Test::RetryReason::RETRY_FLAKY_FIXED
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/test_retries/driver/retry_new.rb

@@ -15,8 +15,6 @@ module Datadog
             @attempts = 0
             # will be changed based on test span duration
             @max_attempts = 10
-
-            mark_new_test(test_span)
           end
 
           def should_retry?
@@ -27,7 +25,6 @@ module Datadog
             super
 
             @attempts += 1
-            mark_new_test(test_span)
 
             Datadog.logger.debug { "Retry Attempts [#{@attempts} / #{@max_attempts}]" }
           end
@@ -38,10 +35,8 @@ module Datadog
             Datadog.logger.debug { "Recorded test duration of [#{duration}], new Max Attempts value is [#{@max_attempts}]" }
           end
 
-          private
-
-          def mark_new_test(test_span)
-            test_span.set_tag(Ext::Test::TAG_IS_NEW, "true")
+          def retry_reason
+            Ext::Test::RetryReason::RETRY_NEW
           end
         end
       end

data/lib/datadog/ci/test_retries/strategy/retry_flaky_fixed.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require_relative "base"
+
+require_relative "../driver/retry_flaky_fixed"
+
+module Datadog
+  module CI
+    module TestRetries
+      module Strategy
+        # This strategy retries tests that are flaky and were marked as attempted to be fixed in Datadog Test Management UI.
+        class RetryFlakyFixed < Base
+          attr_reader :enabled, :max_attempts
+
+          def initialize(
+            enabled:,
+            max_attempts:
+          )
+            @enabled = enabled
+            @max_attempts = max_attempts
+          end
+
+          def configure(library_settings, test_session)
+            @enabled &&= library_settings.test_management_enabled?
+            @max_attempts = library_settings.attempt_to_fix_retries_count || @max_attempts
+          end
+
+          def covers?(test_span)
+            return false unless @enabled
+
+            !!test_span&.attempt_to_fix?
+          end
+
+          def build_driver(test_span)
+            Datadog.logger.debug { "#{test_span.name} is attempt_to_fix, will be retried" }
+
+            Driver::RetryFlakyFixed.new(max_attempts: max_attempts)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/test_retries/strategy/retry_new.rb

@@ -11,19 +11,13 @@ module Datadog
         class RetryNew < Base
           DEFAULT_TOTAL_TESTS_COUNT = 100
 
-          attr_reader :enabled, :max_attempts_thresholds, :unique_tests_set, :total_limit, :retried_count
+          attr_reader :enabled, :max_attempts_thresholds, :total_limit, :retried_count
 
-          def initialize(
-            enabled:,
-            unique_tests_client:
-          )
+          def initialize(enabled:)
             @enabled = enabled
-            @unique_tests_set = Set.new
             # total maximum number of new tests to retry (will be set based on the total number of tests in the session)
             @total_limit = 0
             @retried_count = 0
-
-            @unique_tests_client = unique_tests_client
           end
 
           def covers?(test_span)
@@ -37,11 +31,11 @@ module Datadog
               mark_test_session_faulty(Datadog::CI.active_test_session)
             end
 
-            @enabled && !test_span.skipped? && is_new_test?(test_span)
+            @enabled && !test_span.skipped? && test_span.is_new?
           end
 
           def configure(library_settings, test_session)
-            @enabled &&= library_settings.early_flake_detection_enabled?
+            @enabled &&= library_settings.early_flake_detection_enabled? && library_settings.known_tests_enabled?
 
             return unless @enabled
 
@@ -50,7 +44,6 @@ module Datadog
 
             set_max_attempts_thresholds(library_settings)
             calculate_total_retries_limit(library_settings, test_session)
-            fetch_known_unique_tests(test_session)
           end
 
           def build_driver(test_span)
@@ -68,20 +61,6 @@ module Datadog
             test_session&.set_tag(Ext::Test::TAG_EARLY_FLAKE_ABORT_REASON, Ext::Test::EARLY_FLAKE_FAULTY)
           end
 
-          def is_new_test?(test_span)
-            test_id = Utils::TestRun.datadog_test_id(test_span.name, test_span.test_suite_name)
-
-            result = !@unique_tests_set.include?(test_id)
-
-            if result
-              Datadog.logger.debug do
-                "#{test_id} is not found in the unique tests set, it is a new test"
-              end
-            end
-
-            result
-          end
-
           def set_max_attempts_thresholds(library_settings)
             @max_attempts_thresholds = library_settings.slow_test_retries
             Datadog.logger.debug do
@@ -104,27 +83,6 @@ module Datadog
               "Retry new tests total limit is [#{@total_limit}] (#{percentage_limit}% of #{tests_count})"
             end
           end
-
-          def fetch_known_unique_tests(test_session)
-            @unique_tests_set = @unique_tests_client.fetch_unique_tests(test_session)
-            if @unique_tests_set.empty?
-              @enabled = false
-              mark_test_session_faulty(test_session)
-
-              Datadog.logger.warn(
-                "Disabling early flake detection because there are no known tests (possible reason: no test runs in default branch)"
-              )
-            end
-
-            # report how many unique tests were found
-            Datadog.logger.debug do
-              "Found [#{@unique_tests_set.size}] known unique tests"
-            end
-            Utils::Telemetry.distribution(
-              Ext::Telemetry::METRIC_EFD_UNIQUE_TESTS_RESPONSE_TESTS,
-              @unique_tests_set.size.to_f
-            )
-          end
         end
       end
     end

data/lib/datadog/ci/test_suite.rb

@@ -59,6 +59,24 @@ module Datadog
         end
       end
 
+      # @internal
+      def all_executions_failed?(test_id)
+        synchronize do
+          stats = @execution_stats_per_test[test_id]
+          stats && (stats[Ext::Test::Status::FAIL] > 0 || stats[Ext::Test::ExecutionStatsStatus::FAIL_IGNORED] > 0) &&
+            stats[Ext::Test::Status::PASS] == 0
+        end
+      end
+
+      # @internal
+      def all_executions_passed?(test_id)
+        synchronize do
+          stats = @execution_stats_per_test[test_id]
+          stats && stats[Ext::Test::Status::PASS] > 0 && stats[Ext::Test::Status::FAIL] == 0 &&
+            stats[Ext::Test::ExecutionStatsStatus::FAIL_IGNORED] == 0
+        end
+      end
+
       # @internal
       def test_executed?(test_id)
         synchronize do
@@ -89,8 +107,9 @@ module Datadog
       end
 
       def derive_test_status_from_execution_stats(test_execution_stats)
-        # test is passed if it passed at least once
-        if test_execution_stats[Ext::Test::Status::PASS] > 0
+        # test is passed if it passed at least once or it failed but fail was ignored
+        if test_execution_stats[Ext::Test::Status::PASS] > 0 ||
+            test_execution_stats[Ext::Test::ExecutionStatsStatus::FAIL_IGNORED] > 0
           Ext::Test::Status::PASS
         # if test was never passed, it is failed if it failed at least once
         elsif test_execution_stats[Ext::Test::Status::FAIL] > 0

data/lib/datadog/ci/test_visibility/capabilities.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require_relative "../ext/test"
+
+module Datadog
+  module CI
+    module TestVisibility
+      # Generates internal tags for library capabilities
+      module Capabilities
+        def self.tags
+          tags = {}
+
+          test_optimisation = Datadog::CI.send(:test_optimisation)
+          tags[Ext::Test::LibraryCapabilities::TAG_TEST_IMPACT_ANALYSIS] = test_optimisation.enabled.to_s
+
+          test_management = Datadog::CI.send(:test_management)
+          test_management_tag_value = test_management.enabled.to_s
+
+          [
+            Ext::Test::LibraryCapabilities::TAG_TEST_MANAGEMENT_ATTEMPT_TO_FIX,
+            Ext::Test::LibraryCapabilities::TAG_TEST_MANAGEMENT_QUARANTINE,
+            Ext::Test::LibraryCapabilities::TAG_TEST_MANAGEMENT_DISABLE
+          ].each do |tag|
+            tags[tag] = test_management_tag_value
+          end
+
+          test_retries = Datadog::CI.send(:test_retries)
+          tags[Ext::Test::LibraryCapabilities::TAG_AUTO_TEST_RETRIES] = test_retries.auto_test_retries_feature_enabled.to_s
+          tags[Ext::Test::LibraryCapabilities::TAG_EARLY_FLAKE_DETECTION] = test_retries.early_flake_detection_feature_enabled.to_s
+
+          tags
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/test_visibility/component.rb

@@ -18,9 +18,10 @@ module Datadog
     module TestVisibility
       # Common behavior for CI tests
       class Component
-        attr_reader :test_suite_level_visibility_enabled, :logical_test_session_name
+        attr_reader :test_suite_level_visibility_enabled, :logical_test_session_name, :known_tests, :known_tests_enabled
 
         def initialize(
+          known_tests_client:,
           test_suite_level_visibility_enabled: false,
           codeowners: Codeowners::Parser.new(Git::LocalRepository.root).parse,
           logical_test_session_name: nil
@@ -29,6 +30,21 @@ module Datadog
           @context = Context.new
           @codeowners = codeowners
           @logical_test_session_name = logical_test_session_name
+
+          # "Known tests" feature fetches a list of all tests known to Datadog for this repository
+          # and uses this list to determine if a test is new or not. New tests are marked with "test.is_new" tag.
+          @known_tests_enabled = false
+          @known_tests_client = known_tests_client
+          @known_tests = Set.new
+        end
+
+        def configure(library_configuration, test_session)
+          return unless test_suite_level_visibility_enabled
+
+          if library_configuration.known_tests_enabled?
+            @known_tests_enabled = true
+            fetch_known_tests(test_session)
+          end
         end
 
         def start_test_session(service: nil, tags: {}, total_tests_count: 0)
@@ -158,7 +174,8 @@ module Datadog
           # sets logical test session name if none provided by the user
           override_logical_test_session_name!(test_session) if logical_test_session_name.nil?
 
-          # signal Remote::Component to configure the library
+          # Signal Remote::Component to configure the library.
+          # Note that it will call this component back (unfortunate circular dependency).
           remote.configure(test_session)
         end
 
@@ -182,6 +199,10 @@ module Datadog
 
           Telemetry.event_created(test)
 
+          mark_test_as_new(test) if new_test?(test)
+
+          test_management.tag_test_from_properties(test)
+
           test_optimisation.mark_if_skippable(test)
           test_optimisation.start_coverage(test)
         end
@@ -287,6 +308,49 @@ module Datadog
           end
         end
 
+        def new_test?(test_span)
+          return false unless @known_tests_enabled
+
+          test_id = Utils::TestRun.datadog_test_id(test_span.name, test_span.test_suite_name)
+          result = !@known_tests.include?(test_id)
+
+          if result
+            Datadog.logger.debug do
+              "#{test_id} is not found in the known tests set, it is a new test"
+            end
+          end
+
+          result
+        end
+
+        def fetch_known_tests(test_session)
+          @known_tests = @known_tests_client.fetch(test_session)
+
+          if @known_tests.empty?
+            @known_tests_enabled = false
+
+            # this adds unfortunate knowledge on EFD from Testvisibility, rethink this
+            test_session&.set_tag(Ext::Test::TAG_EARLY_FLAKE_ABORT_REASON, Ext::Test::EARLY_FLAKE_FAULTY)
+
+            Datadog.logger.warn("Empty set of tests known to Datadog")
+          end
+
+          # report how many known tests were found
+          Datadog.logger.debug do
+            "Found [#{@known_tests.size}] known tests"
+          end
+          Utils::Telemetry.distribution(
+            Ext::Telemetry::METRIC_KNOWN_TESTS_RESPONSE_TESTS,
+            @known_tests.size.to_f
+          )
+
+          @known_tests
+        end
+
+        def mark_test_as_new(test_span)
+          test_span.set_tag(Ext::Test::TAG_IS_NEW, "true")
+        end
+
         def test_optimisation
           Datadog.send(:components).test_optimisation
         end
@@ -302,6 +366,10 @@ module Datadog
         def remote
           Datadog.send(:components).ci_remote
         end
+
+        def test_management
+          Datadog.send(:components).test_management
+        end
       end
     end
   end

data/lib/datadog/ci/test_visibility/context.rb

@@ -208,10 +208,6 @@ module Datadog
           ci_span.set_tags(@environment_tags)
 
           ci_span.set_metric(Ext::Test::METRIC_CPU_COUNT, Utils::TestRun.virtual_cpu_count)
-          ci_span.set_tag(
-            Ext::Test::TAG_USER_PROVIDED_TEST_SERVICE,
-            Utils::Configuration.service_name_provided_by_user?.to_s
-          )
         end
 
         # PROPAGATING CONTEXT FROM TOP-LEVEL TO THE LOWER LEVELS

data/lib/datadog/ci/{test_retries/unique_tests_client.rb → test_visibility/known_tests.rb}

@@ -10,9 +10,9 @@ require_relative "../utils/test_run"
 
 module Datadog
   module CI
-    module TestRetries
-      # fetch a list of unique known tests from the backend
-      class UniqueTestsClient
+    module TestVisibility
+      # fetches and stores a list of known tests from the backend
+      class KnownTests
         class Response
           def initialize(http_response)
             @http_response = http_response
@@ -66,7 +66,7 @@ module Datadog
           @config_tags = config_tags
         end
 
-        def fetch_unique_tests(test_session)
+        def fetch(test_session)
           api = @api
           return Set.new unless api
 
@@ -78,21 +78,21 @@ module Datadog
             payload: request_payload
           )
 
-          Transport::Telemetry.api_requests(
-            Ext::Telemetry::METRIC_EFD_UNIQUE_TESTS_REQUEST,
+          CI::Transport::Telemetry.api_requests(
+            Ext::Telemetry::METRIC_KNOWN_TESTS_REQUEST,
             1,
             compressed: http_response.request_compressed
           )
-          Utils::Telemetry.distribution(Ext::Telemetry::METRIC_EFD_UNIQUE_TESTS_REQUEST_MS, http_response.duration_ms)
+          Utils::Telemetry.distribution(Ext::Telemetry::METRIC_KNOWN_TESTS_REQUEST_MS, http_response.duration_ms)
           Utils::Telemetry.distribution(
-            Ext::Telemetry::METRIC_EFD_UNIQUE_TESTS_RESPONSE_BYTES,
+            Ext::Telemetry::METRIC_KNOWN_TESTS_RESPONSE_BYTES,
             http_response.response_size.to_f,
             {Ext::Telemetry::TAG_RESPONSE_COMPRESSED => http_response.gzipped_content?.to_s}
           )
 
           unless http_response.ok?
-            Transport::Telemetry.api_requests_errors(
-              Ext::Telemetry::METRIC_EFD_UNIQUE_TESTS_REQUEST_ERRORS,
+            CI::Transport::Telemetry.api_requests_errors(
+              Ext::Telemetry::METRIC_KNOWN_TESTS_REQUEST_ERRORS,
               1,
               error_type: http_response.telemetry_error_type,
               status_code: http_response.code

data/lib/datadog/ci/test_visibility/null_component.rb

@@ -5,6 +5,9 @@ module Datadog
     module TestVisibility
       # Special test visibility component that does not record anything
       class NullComponent
+        def configure(_, _)
+        end
+
         def start_test_session(service: nil, tags: {}, total_tests_count: 0)
           skip_tracing
         end

data/lib/datadog/ci/test_visibility/serializers/factories/test_level.rb

@@ -9,7 +9,7 @@ module Datadog
       module Serializers
         module Factories
           # This factory takes care of creating msgpack serializers when test-level visibility is enabled
-          # NOTE: citestcycle is a protocol Datadog uses to submit test execution tracing information to CI visibility
+          # NOTE: citestcycle is a protocol Datadog uses to submit test execution tracing information to Test Optimization
           # backend
           module TestLevel
             module_function

data/lib/datadog/ci/test_visibility/telemetry.rb

@@ -54,8 +54,11 @@ module Datadog
           # codeowner tag
           tags[Ext::Telemetry::TAG_HAS_CODEOWNER] = "true" if span.get_tag(Ext::Test::TAG_CODEOWNERS)
 
-          # set is_retry tag if span represents a retried test
-          tags[Ext::Telemetry::TAG_IS_RETRY] = "true" if span.get_tag(Ext::Test::TAG_IS_RETRY)
+          # set is_retry and retry_reason tags if span represents a retried test
+          if span.get_tag(Ext::Test::TAG_IS_RETRY)
+            tags[Ext::Telemetry::TAG_IS_RETRY] = "true"
+            tags[Ext::Telemetry::TAG_RETRY_REASON] = span.get_tag(Ext::Test::TAG_RETRY_REASON)
+          end
 
           # is_new
           tags[Ext::Telemetry::TAG_IS_NEW] = "true" if span.get_tag(Ext::Test::TAG_IS_NEW)
@@ -66,6 +69,12 @@ module Datadog
             tags[Ext::Telemetry::TAG_EARLY_FLAKE_DETECTION_ABORT_REASON] = early_flake_detection_abort_reason
           end
 
+          # test management tags
+          tags[Ext::Telemetry::TAG_IS_ATTEMPT_TO_FIX] = "true" if span.get_tag(Ext::Test::TAG_IS_ATTEMPT_TO_FIX)
+          tags[Ext::Telemetry::TAG_IS_QUARANTINED] = "true" if span.get_tag(Ext::Test::TAG_IS_QUARANTINED)
+          tags[Ext::Telemetry::TAG_IS_TEST_DISABLED] = "true" if span.get_tag(Ext::Test::TAG_IS_TEST_DISABLED)
+          tags[Ext::Telemetry::TAG_HAS_FAILED_ALL_RETRIES] = "true" if span.get_tag(Ext::Test::TAG_HAS_FAILED_ALL_RETRIES)
+
           tags
         end