datadog-ci 1.12.0 → 1.14.0

data/lib/datadog/ci/remote/library_settings_client.rb

@@ -62,7 +62,8 @@ module Datadog
               Ext::Telemetry::TAG_EARLY_FLAKE_DETECTION_ENABLED => library_settings.early_flake_detection_enabled?.to_s,
               Ext::Telemetry::TAG_FLAKY_TEST_RETRIES_ENABLED => library_settings.flaky_test_retries_enabled?.to_s,
               Ext::Telemetry::TAG_ITR_ENABLED => library_settings.itr_enabled?.to_s,
-              Ext::Telemetry::TAG_REQUIRE_GIT => library_settings.require_git?.to_s
+              Ext::Telemetry::TAG_REQUIRE_GIT => library_settings.require_git?.to_s,
+              Ext::Telemetry::TAG_KNOWN_TESTS_ENABLED => library_settings.known_tests_enabled?.to_s
             }
           )
 

data/lib/datadog/ci/span.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "drb"
+
 require "datadog/core/environment/platform"
 
 require_relative "ext/test"
@@ -12,6 +14,8 @@ module Datadog
     #
     # @public_api
     class Span
+      include DRb::DRbUndumped
+
       attr_reader :tracer_span
 
       def initialize(tracer_span)

data/lib/datadog/ci/test.rb

@@ -62,19 +62,37 @@ module Datadog
         get_tag(Ext::Test::TAG_TEST_SESSION_ID)
       end
 
-      # Returns "true" if the test is skipped by the intelligent test runner.
-      # @return [Boolean] true if the test is skipped by the intelligent test runner, false otherwise.
-      def skipped_by_itr?
-        get_tag(Ext::Test::TAG_ITR_SKIPPED_BY_ITR) == "true"
-      end
-
       # Returns "true" if test span represents a retry.
       # @return [Boolean] true if this test is a retry, false otherwise.
       def is_retry?
         get_tag(Ext::Test::TAG_IS_RETRY) == "true"
       end
 
-      # Marks this test as unskippable by the intelligent test runner.
+      # Returns "true" if this span represents a test that wasn't known to Datadog before.
+      # @return [Boolean] true if this test is a new one, false otherwise.
+      def is_new?
+        get_tag(Ext::Test::TAG_IS_NEW) == "true"
+      end
+
+      # Returns "true" if this test is quarantined by Datadog test management.
+      # @return [Boolean] true if this test is quarantined, false otherwise.
+      def quarantined?
+        get_tag(Ext::Test::TAG_IS_QUARANTINED) == "true"
+      end
+
+      # Returns "true" if this test is disabled by Datadog test management.
+      # @return [Boolean] true if this test is disabled, false otherwise.
+      def disabled?
+        get_tag(Ext::Test::TAG_IS_TEST_DISABLED) == "true"
+      end
+
+      # Returns "true" if this flaky test has fixing attempts (determined by Datadog backend).
+      # @return [Boolean] true if this test is attempted to be fixed.
+      def attempt_to_fix?
+        get_tag(Ext::Test::TAG_IS_ATTEMPT_TO_FIX) == "true"
+      end
+
+      # Marks this test as unskippable by the Test Impact Analysis.
       # This must be done before the test execution starts.
       #
       # Examples of tests that should be unskippable:
@@ -88,7 +106,7 @@ module Datadog
         TestOptimisation::Telemetry.itr_unskippable
         set_tag(Ext::Test::TAG_ITR_UNSKIPPABLE, "true")
 
-        if skipped_by_itr?
+        if skipped_by_test_impact_analysis?
           clear_tag(Ext::Test::TAG_ITR_SKIPPED_BY_ITR)
 
           TestOptimisation::Telemetry.itr_forced_run
@@ -110,7 +128,13 @@ module Datadog
       def failed!(exception: nil)
         super
 
-        record_test_result(Ext::Test::Status::FAIL)
+        # if we should ignore failures, we consider this test to be passed
+        if should_ignore_failures?
+          # use a special "fail_ignored" status to mark this test as failed but ignored
+          record_test_result(Ext::Test::ExecutionStatsStatus::FAIL_IGNORED)
+        else
+          record_test_result(Ext::Test::Status::FAIL)
+        end
       end
 
       # Sets the status of the span to "skip".
@@ -148,6 +172,40 @@ module Datadog
         !!test_suite&.any_test_retry_passed?(datadog_test_id)
       end
 
+      # @internal
+      def all_executions_failed?
+        !!test_suite&.all_executions_failed?(datadog_test_id)
+      end
+
+      # @internal
+      def all_executions_passed?
+        !!test_suite&.all_executions_passed?(datadog_test_id)
+      end
+
+      # @internal
+      def datadog_skip_reason
+        if skipped_by_test_impact_analysis?
+          Ext::Test::SkipReason::TEST_IMPACT_ANALYSIS
+        elsif disabled? || quarantined?
+          Ext::Test::SkipReason::TEST_MANAGEMENT_DISABLED
+        end
+      end
+
+      # @internal
+      def should_skip?
+        skipped_by_test_impact_analysis? || (disabled? && !attempt_to_fix?)
+      end
+
+      # @internal
+      def should_ignore_failures?
+        quarantined? || disabled? || any_retry_passed?
+      end
+
+      # @internal
+      def skipped_by_test_impact_analysis?
+        get_tag(Ext::Test::TAG_ITR_SKIPPED_BY_ITR) == "true"
+      end
+
       private
 
       def record_test_result(datadog_status)

data/lib/datadog/ci/test_management/component.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+require_relative "../ext/telemetry"
+require_relative "../ext/test"
+require_relative "../utils/telemetry"
+require_relative "../utils/test_run"
+
+module Datadog
+  module CI
+    module TestManagement
+      # Test management is a feature that lets people manage their flaky tests in Datadog.
+      # It includes:
+      # - marking test as quarantined causes test to continue running but not failing the build
+      # - marking test as disabled causes test to be skipped
+      # - marking test as "attempted to fix" causes test to be retried many times to confirm that fix worked
+      class Component
+        attr_reader :enabled, :tests_properties
+
+        def initialize(enabled:, tests_properties_client:)
+          @enabled = enabled
+
+          @tests_properties_client = tests_properties_client
+          @tests_properties = {}
+        end
+
+        def configure(library_settings, test_session)
+          @enabled &&= library_settings.test_management_enabled?
+
+          return unless @enabled
+
+          test_session.set_tag(Ext::Test::TAG_TEST_MANAGEMENT_ENABLED, "true")
+
+          @tests_properties = @tests_properties_client.fetch(test_session)
+
+          Utils::Telemetry.distribution(
+            Ext::Telemetry::METRIC_TEST_MANAGEMENT_TESTS_RESPONSE_TESTS,
+            @tests_properties.count.to_f
+          )
+        end
+
+        def tag_test_from_properties(test_span)
+          return unless @enabled
+
+          datadog_test_id = Utils::TestRun.datadog_test_id(test_span.name, test_span.test_suite_name)
+          test_properties = @tests_properties[datadog_test_id]
+
+          if test_properties.nil?
+            Datadog.logger.debug { "Test properties not found for test: #{datadog_test_id}" }
+            return
+          end
+
+          Datadog.logger.debug { "Test properties for test #{datadog_test_id} are: [#{test_properties}]" }
+
+          test_span.set_tag(Ext::Test::TAG_IS_QUARANTINED, "true") if test_properties["quarantined"]
+          test_span.set_tag(Ext::Test::TAG_IS_TEST_DISABLED, "true") if test_properties["disabled"]
+          test_span.set_tag(Ext::Test::TAG_IS_ATTEMPT_TO_FIX, "true") if test_properties["attempt_to_fix"]
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/test_management/null_component.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+require_relative "../ext/telemetry"
+require_relative "../utils/telemetry"
+
+module Datadog
+  module CI
+    module TestManagement
+      class NullComponent
+        attr_reader :enabled, :tests_properties
+
+        def initialize
+          @enabled = false
+          @tests_properties = {}
+        end
+
+        def configure(_, _)
+        end
+
+        def tag_test_from_properties(_)
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/test_management/tests_properties.rb

@@ -0,0 +1,128 @@
+# frozen_string_literal: true
+
+require "json"
+
+require_relative "../ext/telemetry"
+require_relative "../ext/transport"
+require_relative "../transport/telemetry"
+require_relative "../utils/parsing"
+require_relative "../utils/telemetry"
+require_relative "../utils/test_run"
+
+module Datadog
+  module CI
+    module TestManagement
+      # fetches and stores a map of tests to their test management properties from the backend
+      class TestsProperties
+        class Response
+          def initialize(http_response)
+            @http_response = http_response
+            @json = nil
+          end
+
+          def ok?
+            resp = @http_response
+            !resp.nil? && resp.ok?
+          end
+
+          def tests
+            tests_map = {}
+
+            payload
+              .fetch("data", {})
+              .fetch("attributes", {})
+              .fetch("modules", {})
+              .each do |_test_module, module_hash|
+                module_hash
+                  .fetch("suites", {})
+                  .each do |test_suite, suite_hash|
+                    suite_hash.fetch("tests", {})
+                      .each do |test_name, properties_hash|
+                        properties = properties_hash.fetch("properties", {})
+                        properties.transform_values! { |v| Utils::Parsing.convert_to_bool(v) }
+
+                        tests_map[Utils::TestRun.datadog_test_id(test_name, test_suite)] = properties
+                      end
+                  end
+              end
+
+            tests_map
+          end
+
+          private
+
+          def payload
+            cached = @json
+            return cached unless cached.nil?
+
+            resp = @http_response
+            return @json = {} if resp.nil? || !ok?
+
+            begin
+              @json = JSON.parse(resp.payload)
+            rescue JSON::ParserError => e
+              Datadog.logger.error(
+                "Failed to parse test management tests response payload: #{e}. Payload was: #{resp.payload}"
+              )
+              @json = {}
+            end
+          end
+        end
+
+        def initialize(api: nil)
+          @api = api
+        end
+
+        def fetch(test_session)
+          api = @api
+          return {} unless api
+
+          request_payload = payload(test_session)
+          Datadog.logger.debug("Fetching test management tests with request: #{request_payload}")
+
+          http_response = api.api_request(
+            path: Ext::Transport::DD_API_TEST_MANAGEMENT_TESTS_PATH,
+            payload: request_payload
+          )
+
+          CI::Transport::Telemetry.api_requests(
+            Ext::Telemetry::METRIC_TEST_MANAGEMENT_TESTS_REQUEST,
+            1,
+            compressed: http_response.request_compressed
+          )
+          Utils::Telemetry.distribution(Ext::Telemetry::METRIC_TEST_MANAGEMENT_TESTS_REQUEST_MS, http_response.duration_ms)
+          Utils::Telemetry.distribution(
+            Ext::Telemetry::METRIC_TEST_MANAGEMENT_TESTS_RESPONSE_BYTES,
+            http_response.response_size.to_f,
+            {Ext::Telemetry::TAG_RESPONSE_COMPRESSED => http_response.gzipped_content?.to_s}
+          )
+
+          unless http_response.ok?
+            CI::Transport::Telemetry.api_requests_errors(
+              Ext::Telemetry::METRIC_TEST_MANAGEMENT_TESTS_REQUEST_ERRORS,
+              1,
+              error_type: http_response.telemetry_error_type,
+              status_code: http_response.code
+            )
+          end
+
+          Response.new(http_response).tests
+        end
+
+        private
+
+        def payload(test_session)
+          {
+            "data" => {
+              "id" => Datadog::Core::Environment::Identity.id,
+              "type" => Ext::Transport::DD_API_TEST_MANAGEMENT_TESTS_TYPE,
+              "attributes" => {
+                "repository_url" => test_session.git_repository_url
+              }
+            }
+          }.to_json
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/test_optimisation/component.rb

@@ -3,7 +3,6 @@
 require "pp"
 
 require "datadog/core/telemetry/logging"
-require "datadog/core/utils/forking"
 
 require_relative "../ext/test"
 require_relative "../ext/telemetry"
@@ -20,14 +19,11 @@ require_relative "telemetry"
 module Datadog
   module CI
     module TestOptimisation
-      # Intelligent test runner implementation
+      # Test Impact Analysis implementation
       # Integrates with backend to provide test impact analysis data and
       # skip tests that are not impacted by the changes
       class Component
-        include Core::Utils::Forking
-
         attr_reader :correlation_id, :skippable_tests, :skippable_tests_fetch_error,
-          :skipped_tests_count, :total_tests_count,
           :enabled, :test_skipping_enabled, :code_coverage_enabled
 
         def initialize(
@@ -61,9 +57,6 @@ module Datadog
           @correlation_id = nil
           @skippable_tests = Set.new
 
-          @total_tests_count = 0
-          @skipped_tests_count = 0
-
           @mutex = Mutex.new
 
           Datadog.logger.debug("TestOptimisation initialized with enabled: #{@enabled}")
@@ -155,11 +148,6 @@ module Datadog
           return if !enabled? || !skipping_tests?
 
           if skippable?(test)
-            if forked?
-              Datadog.logger.warn { "Intelligent test runner is not supported for forking test runners yet" }
-              return
-            end
-
             test.set_tag(Ext::Test::TAG_ITR_SKIPPED_BY_ITR, "true")
 
             Datadog.logger.debug { "Marked test as skippable: #{test.datadog_test_id}" }
@@ -168,31 +156,22 @@ module Datadog
           end
         end
 
-        def count_skipped_test(test)
-          @mutex.synchronize do
-            @total_tests_count += 1
+        def on_test_finished(test, context)
+          return if !test.skipped? || !test.skipped_by_test_impact_analysis?
 
-            return if !test.skipped? || !test.skipped_by_itr?
+          Telemetry.itr_skipped
 
-            if forked?
-              Datadog.logger.warn { "ITR is not supported for forking test runners yet" }
-              return
-            end
-
-            Telemetry.itr_skipped
-
-            @skipped_tests_count += 1
-          end
+          context.incr_tests_skipped_by_tia_count
         end
 
-        def write_test_session_tags(test_session)
+        def write_test_session_tags(test_session, skipped_tests_count)
           return if !enabled?
 
           Datadog.logger.debug { "Finished optimised session with test skipping enabled: #{@test_skipping_enabled}" }
-          Datadog.logger.debug { "#{@skipped_tests_count} tests were skipped" }
+          Datadog.logger.debug { "#{skipped_tests_count} tests were skipped" }
 
-          test_session.set_tag(Ext::Test::TAG_ITR_TESTS_SKIPPED, @skipped_tests_count.positive?.to_s)
-          test_session.set_tag(Ext::Test::TAG_ITR_TEST_SKIPPING_COUNT, @skipped_tests_count)
+          test_session.set_tag(Ext::Test::TAG_ITR_TESTS_SKIPPED, skipped_tests_count.positive?.to_s)
+          test_session.set_tag(Ext::Test::TAG_ITR_TEST_SKIPPING_COUNT, skipped_tests_count)
         end
 
         def skippable_tests_count

data/lib/datadog/ci/test_optimisation/skippable_percentage/base.rb

@@ -44,6 +44,10 @@ module Datadog
           def test_optimisation
             Datadog.send(:components).test_optimisation
           end
+
+          def test_visibility
+            Datadog.send(:components).test_visibility
+          end
         end
       end
     end

data/lib/datadog/ci/test_optimisation/skippable_percentage/calculator.rb

@@ -34,11 +34,11 @@ module Datadog
               return 0.0
             end
 
-            log("Total tests count: #{test_optimisation.total_tests_count}")
-            log("Skipped tests count: #{test_optimisation.skipped_tests_count}")
+            log("Total tests count: #{test_visibility.total_tests_count}")
+            log("Skipped tests count: #{test_visibility.tests_skipped_by_tia_count}")
             validate_test_optimisation_state!
 
-            (test_optimisation.skipped_tests_count.to_f / test_optimisation.total_tests_count.to_f).floor(2)
+            (test_visibility.tests_skipped_by_tia_count.to_f / test_visibility.total_tests_count.to_f).floor(2)
           end
 
           private

data/lib/datadog/ci/test_retries/component.rb

@@ -7,6 +7,7 @@ require_relative "driver/retry_new"
 require_relative "strategy/no_retry"
 require_relative "strategy/retry_failed"
 require_relative "strategy/retry_new"
+require_relative "strategy/retry_flaky_fixed"
 
 require_relative "../ext/telemetry"
 require_relative "../utils/telemetry"
@@ -25,23 +26,33 @@ module Datadog
           retry_failed_tests_max_attempts:,
           retry_failed_tests_total_limit:,
           retry_new_tests_enabled:,
-          unique_tests_client:
+          retry_flaky_fixed_tests_enabled:,
+          retry_flaky_fixed_tests_max_attempts:
         )
           no_retries_strategy = Strategy::NoRetry.new
 
-          retry_failed_strategy = Strategy::RetryFailed.new(
+          @retry_failed_strategy = Strategy::RetryFailed.new(
             enabled: retry_failed_tests_enabled,
             max_attempts: retry_failed_tests_max_attempts,
             total_limit: retry_failed_tests_total_limit
           )
 
-          retry_new_strategy = Strategy::RetryNew.new(
-            enabled: retry_new_tests_enabled,
-            unique_tests_client: unique_tests_client
+          @retry_new_strategy = Strategy::RetryNew.new(
+            enabled: retry_new_tests_enabled
           )
 
-          # order is important, we should try to retry new tests first
-          @retry_strategies = [retry_new_strategy, retry_failed_strategy, no_retries_strategy]
+          retry_flaky_fixed_strategy = Strategy::RetryFlakyFixed.new(
+            enabled: retry_flaky_fixed_tests_enabled,
+            max_attempts: retry_flaky_fixed_tests_max_attempts
+          )
+
+          # order is important, we apply the first matching strategy
+          @retry_strategies = [
+            retry_flaky_fixed_strategy,
+            @retry_new_strategy,
+            @retry_failed_strategy,
+            no_retries_strategy
+          ]
           @mutex = Mutex.new
         end
 
@@ -82,6 +93,8 @@ module Datadog
           else
             # after each retry we record the result, the driver will decide if we should retry again
             current_retry_driver&.record_retry(test_span)
+
+            tag_last_retry(test_span) unless should_retry?
           end
         end
 
@@ -94,10 +107,27 @@ module Datadog
           self.current_retry_driver = nil
         end
 
+        def tag_last_retry(test_span)
+          test_span&.set_tag(Ext::Test::TAG_HAS_FAILED_ALL_RETRIES, "true") if test_span&.all_executions_failed?
+
+          # if we are attempting to fix the test and all retries passed, we indicate that the fix might have worked
+          if test_span&.attempt_to_fix? && test_span.all_executions_passed?
+            test_span&.set_tag(Ext::Test::TAG_ATTEMPT_TO_FIX_PASSED, "true")
+          end
+        end
+
         def should_retry?
           !!current_retry_driver&.should_retry?
         end
 
+        def auto_test_retries_feature_enabled
+          @retry_failed_strategy.enabled
+        end
+
+        def early_flake_detection_feature_enabled
+          @retry_new_strategy.enabled
+        end
+
         private
 
         def current_retry_driver

data/lib/datadog/ci/test_retries/driver/base.rb

@@ -13,11 +13,16 @@ module Datadog
 
           def record_retry(test_span)
             test_span&.set_tag(Ext::Test::TAG_IS_RETRY, "true")
+            test_span&.set_tag(Ext::Test::TAG_RETRY_REASON, retry_reason)
           end
 
           # duration in float seconds
           def record_duration(duration)
           end
+
+          def retry_reason
+            "unknown"
+          end
         end
       end
     end

data/lib/datadog/ci/test_retries/driver/retry_failed.rb

@@ -30,6 +30,10 @@ module Datadog
 
             Datadog.logger.debug { "Retry Attempts [#{@attempts} / #{@max_attempts}], Passed: [#{@passed_once}]" }
           end
+
+          def retry_reason
+            Ext::Test::RetryReason::RETRY_FAILED
+          end
         end
       end
     end

data/lib/datadog/ci/test_retries/driver/retry_flaky_fixed.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require_relative "base"
+
+require_relative "../../ext/test"
+
+module Datadog
+  module CI
+    module TestRetries
+      module Driver
+        class RetryFlakyFixed < Base
+          attr_reader :max_attempts
+
+          def initialize(max_attempts:)
+            @attempts = 0
+            @max_attempts = max_attempts
+          end
+
+          def should_retry?
+            @attempts < @max_attempts
+          end
+
+          def record_retry(test_span)
+            super
+
+            @attempts += 1
+
+            Datadog.logger.debug { "Retry Attempts [#{@attempts} / #{@max_attempts}]" }
+          end
+
+          def retry_reason
+            Ext::Test::RetryReason::RETRY_FLAKY_FIXED
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/test_retries/driver/retry_new.rb

@@ -15,8 +15,6 @@ module Datadog
             @attempts = 0
             # will be changed based on test span duration
             @max_attempts = 10
-
-            mark_new_test(test_span)
           end
 
           def should_retry?
@@ -27,7 +25,6 @@ module Datadog
             super
 
             @attempts += 1
-            mark_new_test(test_span)
 
             Datadog.logger.debug { "Retry Attempts [#{@attempts} / #{@max_attempts}]" }
           end
@@ -38,10 +35,8 @@ module Datadog
             Datadog.logger.debug { "Recorded test duration of [#{duration}], new Max Attempts value is [#{@max_attempts}]" }
           end
 
-          private
-
-          def mark_new_test(test_span)
-            test_span.set_tag(Ext::Test::TAG_IS_NEW, "true")
+          def retry_reason
+            Ext::Test::RetryReason::RETRY_NEW
           end
         end
       end