datadog-ci 1.12.0 → 1.13.0

data/lib/datadog/ci/test_management/component.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+require_relative "../ext/telemetry"
+require_relative "../ext/test"
+require_relative "../utils/telemetry"
+require_relative "../utils/test_run"
+
+module Datadog
+  module CI
+    module TestManagement
+      # Test management is a feature that lets people manage their flaky tests in Datadog.
+      # It includes:
+      # - marking test as quarantined causes test to continue running but not failing the build
+      # - marking test as disabled causes test to be skipped
+      # - marking test as "attempted to fix" causes test to be retried many times to confirm that fix worked
+      class Component
+        attr_reader :enabled, :tests_properties
+
+        def initialize(enabled:, tests_properties_client:)
+          @enabled = enabled
+
+          @tests_properties_client = tests_properties_client
+          @tests_properties = {}
+        end
+
+        def configure(library_settings, test_session)
+          @enabled &&= library_settings.test_management_enabled?
+
+          return unless @enabled
+
+          test_session.set_tag(Ext::Test::TAG_TEST_MANAGEMENT_ENABLED, "true")
+
+          @tests_properties = @tests_properties_client.fetch(test_session)
+
+          Utils::Telemetry.distribution(
+            Ext::Telemetry::METRIC_TEST_MANAGEMENT_TESTS_RESPONSE_TESTS,
+            @tests_properties.count.to_f
+          )
+        end
+
+        def tag_test_from_properties(test_span)
+          return unless @enabled
+
+          datadog_test_id = Utils::TestRun.datadog_test_id(test_span.name, test_span.test_suite_name)
+          test_properties = @tests_properties[datadog_test_id]
+
+          if test_properties.nil?
+            Datadog.logger.debug { "Test properties not found for test: #{datadog_test_id}" }
+            return
+          end
+
+          Datadog.logger.debug { "Test properties for test #{datadog_test_id} are: [#{test_properties}]" }
+
+          test_span.set_tag(Ext::Test::TAG_IS_QUARANTINED, "true") if test_properties["quarantined"]
+          test_span.set_tag(Ext::Test::TAG_IS_TEST_DISABLED, "true") if test_properties["disabled"]
+          test_span.set_tag(Ext::Test::TAG_IS_ATTEMPT_TO_FIX, "true") if test_properties["attempt_to_fix"]
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/test_management/null_component.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+require_relative "../ext/telemetry"
+require_relative "../utils/telemetry"
+
+module Datadog
+  module CI
+    module TestManagement
+      class NullComponent
+        attr_reader :enabled, :tests_properties
+
+        def initialize
+          @enabled = false
+          @tests_properties = {}
+        end
+
+        def configure(_, _)
+        end
+
+        def tag_test_from_properties(_)
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/test_management/tests_properties.rb

@@ -0,0 +1,128 @@
+# frozen_string_literal: true
+
+require "json"
+
+require_relative "../ext/telemetry"
+require_relative "../ext/transport"
+require_relative "../transport/telemetry"
+require_relative "../utils/parsing"
+require_relative "../utils/telemetry"
+require_relative "../utils/test_run"
+
+module Datadog
+  module CI
+    module TestManagement
+      # fetches and stores a map of tests to their test management properties from the backend
+      class TestsProperties
+        class Response
+          def initialize(http_response)
+            @http_response = http_response
+            @json = nil
+          end
+
+          def ok?
+            resp = @http_response
+            !resp.nil? && resp.ok?
+          end
+
+          def tests
+            tests_map = {}
+
+            payload
+              .fetch("data", {})
+              .fetch("attributes", {})
+              .fetch("modules", {})
+              .each do |_test_module, module_hash|
+                module_hash
+                  .fetch("suites", {})
+                  .each do |test_suite, suite_hash|
+                    suite_hash.fetch("tests", {})
+                      .each do |test_name, properties_hash|
+                        properties = properties_hash.fetch("properties", {})
+                        properties.transform_values! { |v| Utils::Parsing.convert_to_bool(v) }
+
+                        tests_map[Utils::TestRun.datadog_test_id(test_name, test_suite)] = properties
+                      end
+                  end
+              end
+
+            tests_map
+          end
+
+          private
+
+          def payload
+            cached = @json
+            return cached unless cached.nil?
+
+            resp = @http_response
+            return @json = {} if resp.nil? || !ok?
+
+            begin
+              @json = JSON.parse(resp.payload)
+            rescue JSON::ParserError => e
+              Datadog.logger.error(
+                "Failed to parse test management tests response payload: #{e}. Payload was: #{resp.payload}"
+              )
+              @json = {}
+            end
+          end
+        end
+
+        def initialize(api: nil)
+          @api = api
+        end
+
+        def fetch(test_session)
+          api = @api
+          return {} unless api
+
+          request_payload = payload(test_session)
+          Datadog.logger.debug("Fetching test management tests with request: #{request_payload}")
+
+          http_response = api.api_request(
+            path: Ext::Transport::DD_API_TEST_MANAGEMENT_TESTS_PATH,
+            payload: request_payload
+          )
+
+          CI::Transport::Telemetry.api_requests(
+            Ext::Telemetry::METRIC_TEST_MANAGEMENT_TESTS_REQUEST,
+            1,
+            compressed: http_response.request_compressed
+          )
+          Utils::Telemetry.distribution(Ext::Telemetry::METRIC_TEST_MANAGEMENT_TESTS_REQUEST_MS, http_response.duration_ms)
+          Utils::Telemetry.distribution(
+            Ext::Telemetry::METRIC_TEST_MANAGEMENT_TESTS_RESPONSE_BYTES,
+            http_response.response_size.to_f,
+            {Ext::Telemetry::TAG_RESPONSE_COMPRESSED => http_response.gzipped_content?.to_s}
+          )
+
+          unless http_response.ok?
+            CI::Transport::Telemetry.api_requests_errors(
+              Ext::Telemetry::METRIC_TEST_MANAGEMENT_TESTS_REQUEST_ERRORS,
+              1,
+              error_type: http_response.telemetry_error_type,
+              status_code: http_response.code
+            )
+          end
+
+          Response.new(http_response).tests
+        end
+
+        private
+
+        def payload(test_session)
+          {
+            "data" => {
+              "id" => Datadog::Core::Environment::Identity.id,
+              "type" => Ext::Transport::DD_API_TEST_MANAGEMENT_TESTS_TYPE,
+              "attributes" => {
+                "repository_url" => test_session.git_repository_url
+              }
+            }
+          }.to_json
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/test_optimisation/component.rb

@@ -20,7 +20,7 @@ require_relative "telemetry"
 module Datadog
   module CI
     module TestOptimisation
-      # Intelligent test runner implementation
+      # Test Impact Analysis implementation
       # Integrates with backend to provide test impact analysis data and
       # skip tests that are not impacted by the changes
       class Component
@@ -156,7 +156,7 @@ module Datadog
 
           if skippable?(test)
             if forked?
-              Datadog.logger.warn { "Intelligent test runner is not supported for forking test runners yet" }
+              Datadog.logger.warn { "Test Impact Analysis is not supported for forking test runners yet" }
               return
             end
 
@@ -172,7 +172,7 @@ module Datadog
           @mutex.synchronize do
             @total_tests_count += 1
 
-            return if !test.skipped? || !test.skipped_by_itr?
+            return if !test.skipped? || !test.skipped_by_test_impact_analysis?
 
             if forked?
               Datadog.logger.warn { "ITR is not supported for forking test runners yet" }

data/lib/datadog/ci/test_retries/component.rb

@@ -7,6 +7,7 @@ require_relative "driver/retry_new"
 require_relative "strategy/no_retry"
 require_relative "strategy/retry_failed"
 require_relative "strategy/retry_new"
+require_relative "strategy/retry_flaky_fixed"
 
 require_relative "../ext/telemetry"
 require_relative "../utils/telemetry"
@@ -25,23 +26,33 @@ module Datadog
           retry_failed_tests_max_attempts:,
           retry_failed_tests_total_limit:,
           retry_new_tests_enabled:,
-          unique_tests_client:
+          retry_flaky_fixed_tests_enabled:,
+          retry_flaky_fixed_tests_max_attempts:
         )
           no_retries_strategy = Strategy::NoRetry.new
 
-          retry_failed_strategy = Strategy::RetryFailed.new(
+          @retry_failed_strategy = Strategy::RetryFailed.new(
             enabled: retry_failed_tests_enabled,
             max_attempts: retry_failed_tests_max_attempts,
             total_limit: retry_failed_tests_total_limit
           )
 
-          retry_new_strategy = Strategy::RetryNew.new(
-            enabled: retry_new_tests_enabled,
-            unique_tests_client: unique_tests_client
+          @retry_new_strategy = Strategy::RetryNew.new(
+            enabled: retry_new_tests_enabled
           )
 
-          # order is important, we should try to retry new tests first
-          @retry_strategies = [retry_new_strategy, retry_failed_strategy, no_retries_strategy]
+          retry_flaky_fixed_strategy = Strategy::RetryFlakyFixed.new(
+            enabled: retry_flaky_fixed_tests_enabled,
+            max_attempts: retry_flaky_fixed_tests_max_attempts
+          )
+
+          # order is important, we apply the first matching strategy
+          @retry_strategies = [
+            retry_flaky_fixed_strategy,
+            @retry_new_strategy,
+            @retry_failed_strategy,
+            no_retries_strategy
+          ]
           @mutex = Mutex.new
         end
 
@@ -82,6 +93,8 @@ module Datadog
           else
             # after each retry we record the result, the driver will decide if we should retry again
             current_retry_driver&.record_retry(test_span)
+
+            tag_last_retry(test_span) unless should_retry?
           end
         end
 
@@ -94,10 +107,27 @@ module Datadog
           self.current_retry_driver = nil
         end
 
+        def tag_last_retry(test_span)
+          test_span&.set_tag(Ext::Test::TAG_HAS_FAILED_ALL_RETRIES, "true") if test_span&.all_executions_failed?
+
+          # if we are attempting to fix the test and all retries passed, we indicate that the fix might have worked
+          if test_span&.attempt_to_fix? && test_span.all_executions_passed?
+            test_span&.set_tag(Ext::Test::TAG_ATTEMPT_TO_FIX_PASSED, "true")
+          end
+        end
+
         def should_retry?
           !!current_retry_driver&.should_retry?
         end
 
+        def auto_test_retries_feature_enabled
+          @retry_failed_strategy.enabled
+        end
+
+        def early_flake_detection_feature_enabled
+          @retry_new_strategy.enabled
+        end
+
         private
 
         def current_retry_driver

data/lib/datadog/ci/test_retries/driver/base.rb

@@ -13,11 +13,16 @@ module Datadog
 
           def record_retry(test_span)
             test_span&.set_tag(Ext::Test::TAG_IS_RETRY, "true")
+            test_span&.set_tag(Ext::Test::TAG_RETRY_REASON, retry_reason)
           end
 
           # duration in float seconds
           def record_duration(duration)
           end
+
+          def retry_reason
+            "unknown"
+          end
         end
       end
     end

data/lib/datadog/ci/test_retries/driver/retry_failed.rb

@@ -30,6 +30,10 @@ module Datadog
 
             Datadog.logger.debug { "Retry Attempts [#{@attempts} / #{@max_attempts}], Passed: [#{@passed_once}]" }
           end
+
+          def retry_reason
+            Ext::Test::RetryReason::RETRY_FAILED
+          end
         end
       end
     end

data/lib/datadog/ci/test_retries/driver/retry_flaky_fixed.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require_relative "base"
+
+require_relative "../../ext/test"
+
+module Datadog
+  module CI
+    module TestRetries
+      module Driver
+        class RetryFlakyFixed < Base
+          attr_reader :max_attempts
+
+          def initialize(max_attempts:)
+            @attempts = 0
+            @max_attempts = max_attempts
+          end
+
+          def should_retry?
+            @attempts < @max_attempts
+          end
+
+          def record_retry(test_span)
+            super
+
+            @attempts += 1
+
+            Datadog.logger.debug { "Retry Attempts [#{@attempts} / #{@max_attempts}]" }
+          end
+
+          def retry_reason
+            Ext::Test::RetryReason::RETRY_FLAKY_FIXED
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/test_retries/driver/retry_new.rb

@@ -15,8 +15,6 @@ module Datadog
             @attempts = 0
             # will be changed based on test span duration
             @max_attempts = 10
-
-            mark_new_test(test_span)
           end
 
           def should_retry?
@@ -27,7 +25,6 @@ module Datadog
             super
 
             @attempts += 1
-            mark_new_test(test_span)
 
             Datadog.logger.debug { "Retry Attempts [#{@attempts} / #{@max_attempts}]" }
           end
@@ -38,10 +35,8 @@ module Datadog
             Datadog.logger.debug { "Recorded test duration of [#{duration}], new Max Attempts value is [#{@max_attempts}]" }
           end
 
-          private
-
-          def mark_new_test(test_span)
-            test_span.set_tag(Ext::Test::TAG_IS_NEW, "true")
+          def retry_reason
+            Ext::Test::RetryReason::RETRY_NEW
           end
         end
       end

data/lib/datadog/ci/test_retries/strategy/retry_flaky_fixed.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require_relative "base"
+
+require_relative "../driver/retry_flaky_fixed"
+
+module Datadog
+  module CI
+    module TestRetries
+      module Strategy
+        # This strategy retries tests that are flaky and were marked as attempted to be fixed in Datadog Test Management UI.
+        class RetryFlakyFixed < Base
+          attr_reader :enabled, :max_attempts
+
+          def initialize(
+            enabled:,
+            max_attempts:
+          )
+            @enabled = enabled
+            @max_attempts = max_attempts
+          end
+
+          def configure(library_settings, test_session)
+            @enabled &&= library_settings.test_management_enabled?
+            @max_attempts = library_settings.attempt_to_fix_retries_count || @max_attempts
+          end
+
+          def covers?(test_span)
+            return false unless @enabled
+
+            !!test_span&.attempt_to_fix?
+          end
+
+          def build_driver(test_span)
+            Datadog.logger.debug { "#{test_span.name} is attempt_to_fix, will be retried" }
+
+            Driver::RetryFlakyFixed.new(max_attempts: max_attempts)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/test_retries/strategy/retry_new.rb

@@ -11,19 +11,13 @@ module Datadog
         class RetryNew < Base
           DEFAULT_TOTAL_TESTS_COUNT = 100
 
-          attr_reader :enabled, :max_attempts_thresholds, :unique_tests_set, :total_limit, :retried_count
+          attr_reader :enabled, :max_attempts_thresholds, :total_limit, :retried_count
 
-          def initialize(
-            enabled:,
-            unique_tests_client:
-          )
+          def initialize(enabled:)
             @enabled = enabled
-            @unique_tests_set = Set.new
             # total maximum number of new tests to retry (will be set based on the total number of tests in the session)
             @total_limit = 0
             @retried_count = 0
-
-            @unique_tests_client = unique_tests_client
           end
 
           def covers?(test_span)
@@ -37,11 +31,11 @@ module Datadog
               mark_test_session_faulty(Datadog::CI.active_test_session)
             end
 
-            @enabled && !test_span.skipped? && is_new_test?(test_span)
+            @enabled && !test_span.skipped? && test_span.is_new?
           end
 
           def configure(library_settings, test_session)
-            @enabled &&= library_settings.early_flake_detection_enabled?
+            @enabled &&= library_settings.early_flake_detection_enabled? && library_settings.known_tests_enabled?
 
             return unless @enabled
 
@@ -50,7 +44,6 @@ module Datadog
 
             set_max_attempts_thresholds(library_settings)
             calculate_total_retries_limit(library_settings, test_session)
-            fetch_known_unique_tests(test_session)
           end
 
           def build_driver(test_span)
@@ -68,20 +61,6 @@ module Datadog
             test_session&.set_tag(Ext::Test::TAG_EARLY_FLAKE_ABORT_REASON, Ext::Test::EARLY_FLAKE_FAULTY)
           end
 
-          def is_new_test?(test_span)
-            test_id = Utils::TestRun.datadog_test_id(test_span.name, test_span.test_suite_name)
-
-            result = !@unique_tests_set.include?(test_id)
-
-            if result
-              Datadog.logger.debug do
-                "#{test_id} is not found in the unique tests set, it is a new test"
-              end
-            end
-
-            result
-          end
-
           def set_max_attempts_thresholds(library_settings)
             @max_attempts_thresholds = library_settings.slow_test_retries
             Datadog.logger.debug do
@@ -104,27 +83,6 @@ module Datadog
               "Retry new tests total limit is [#{@total_limit}] (#{percentage_limit}% of #{tests_count})"
             end
           end
-
-          def fetch_known_unique_tests(test_session)
-            @unique_tests_set = @unique_tests_client.fetch_unique_tests(test_session)
-            if @unique_tests_set.empty?
-              @enabled = false
-              mark_test_session_faulty(test_session)
-
-              Datadog.logger.warn(
-                "Disabling early flake detection because there are no known tests (possible reason: no test runs in default branch)"
-              )
-            end
-
-            # report how many unique tests were found
-            Datadog.logger.debug do
-              "Found [#{@unique_tests_set.size}] known unique tests"
-            end
-            Utils::Telemetry.distribution(
-              Ext::Telemetry::METRIC_EFD_UNIQUE_TESTS_RESPONSE_TESTS,
-              @unique_tests_set.size.to_f
-            )
-          end
         end
       end
     end

data/lib/datadog/ci/test_suite.rb

@@ -59,6 +59,24 @@ module Datadog
         end
       end
 
+      # @internal
+      def all_executions_failed?(test_id)
+        synchronize do
+          stats = @execution_stats_per_test[test_id]
+          stats && (stats[Ext::Test::Status::FAIL] > 0 || stats[Ext::Test::ExecutionStatsStatus::FAIL_IGNORED] > 0) &&
+            stats[Ext::Test::Status::PASS] == 0
+        end
+      end
+
+      # @internal
+      def all_executions_passed?(test_id)
+        synchronize do
+          stats = @execution_stats_per_test[test_id]
+          stats && stats[Ext::Test::Status::PASS] > 0 && stats[Ext::Test::Status::FAIL] == 0 &&
+            stats[Ext::Test::ExecutionStatsStatus::FAIL_IGNORED] == 0
+        end
+      end
+
       # @internal
       def test_executed?(test_id)
         synchronize do
@@ -89,8 +107,9 @@ module Datadog
       end
 
       def derive_test_status_from_execution_stats(test_execution_stats)
-        # test is passed if it passed at least once
-        if test_execution_stats[Ext::Test::Status::PASS] > 0
+        # test is passed if it passed at least once or it failed but fail was ignored
+        if test_execution_stats[Ext::Test::Status::PASS] > 0 ||
+            test_execution_stats[Ext::Test::ExecutionStatsStatus::FAIL_IGNORED] > 0
           Ext::Test::Status::PASS
         # if test was never passed, it is failed if it failed at least once
         elsif test_execution_stats[Ext::Test::Status::FAIL] > 0

data/lib/datadog/ci/test_visibility/capabilities.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require_relative "../ext/test"
+
+module Datadog
+  module CI
+    module TestVisibility
+      # Generates internal tags for library capabilities
+      module Capabilities
+        def self.tags
+          tags = {}
+
+          test_optimisation = Datadog::CI.send(:test_optimisation)
+          tags[Ext::Test::LibraryCapabilities::TAG_TEST_IMPACT_ANALYSIS] = test_optimisation.enabled.to_s
+
+          test_management = Datadog::CI.send(:test_management)
+          test_management_tag_value = test_management.enabled.to_s
+
+          [
+            Ext::Test::LibraryCapabilities::TAG_TEST_MANAGEMENT_ATTEMPT_TO_FIX,
+            Ext::Test::LibraryCapabilities::TAG_TEST_MANAGEMENT_QUARANTINE,
+            Ext::Test::LibraryCapabilities::TAG_TEST_MANAGEMENT_DISABLE
+          ].each do |tag|
+            tags[tag] = test_management_tag_value
+          end
+
+          test_retries = Datadog::CI.send(:test_retries)
+          tags[Ext::Test::LibraryCapabilities::TAG_AUTO_TEST_RETRIES] = test_retries.auto_test_retries_feature_enabled.to_s
+          tags[Ext::Test::LibraryCapabilities::TAG_EARLY_FLAKE_DETECTION] = test_retries.early_flake_detection_feature_enabled.to_s
+
+          tags
+        end
+      end
+    end
+  end
+end