datadog-ci 1.12.0 → 1.14.0

data/lib/datadog/ci/test_retries/strategy/retry_flaky_fixed.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require_relative "base"
+
+require_relative "../driver/retry_flaky_fixed"
+
+module Datadog
+  module CI
+    module TestRetries
+      module Strategy
+        # This strategy retries tests that are flaky and were marked as attempted to be fixed in Datadog Test Management UI.
+        class RetryFlakyFixed < Base
+          attr_reader :enabled, :max_attempts
+
+          def initialize(
+            enabled:,
+            max_attempts:
+          )
+            @enabled = enabled
+            @max_attempts = max_attempts
+          end
+
+          def configure(library_settings, test_session)
+            @enabled &&= library_settings.test_management_enabled?
+            @max_attempts = library_settings.attempt_to_fix_retries_count || @max_attempts
+          end
+
+          def covers?(test_span)
+            return false unless @enabled
+
+            !!test_span&.attempt_to_fix?
+          end
+
+          def build_driver(test_span)
+            Datadog.logger.debug { "#{test_span.name} is attempt_to_fix, will be retried" }
+
+            Driver::RetryFlakyFixed.new(max_attempts: max_attempts)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/test_retries/strategy/retry_new.rb

@@ -11,19 +11,13 @@ module Datadog
         class RetryNew < Base
           DEFAULT_TOTAL_TESTS_COUNT = 100
 
-          attr_reader :enabled, :max_attempts_thresholds, :unique_tests_set, :total_limit, :retried_count
+          attr_reader :enabled, :max_attempts_thresholds, :total_limit, :retried_count
 
-          def initialize(
-            enabled:,
-            unique_tests_client:
-          )
+          def initialize(enabled:)
             @enabled = enabled
-            @unique_tests_set = Set.new
             # total maximum number of new tests to retry (will be set based on the total number of tests in the session)
             @total_limit = 0
             @retried_count = 0
-
-            @unique_tests_client = unique_tests_client
           end
 
           def covers?(test_span)
@@ -37,11 +31,11 @@ module Datadog
               mark_test_session_faulty(Datadog::CI.active_test_session)
             end
 
-            @enabled && !test_span.skipped? && is_new_test?(test_span)
+            @enabled && !test_span.skipped? && test_span.is_new?
           end
 
           def configure(library_settings, test_session)
-            @enabled &&= library_settings.early_flake_detection_enabled?
+            @enabled &&= library_settings.early_flake_detection_enabled? && library_settings.known_tests_enabled?
 
             return unless @enabled
 
@@ -50,7 +44,6 @@ module Datadog
 
             set_max_attempts_thresholds(library_settings)
             calculate_total_retries_limit(library_settings, test_session)
-            fetch_known_unique_tests(test_session)
           end
 
           def build_driver(test_span)
@@ -68,20 +61,6 @@ module Datadog
             test_session&.set_tag(Ext::Test::TAG_EARLY_FLAKE_ABORT_REASON, Ext::Test::EARLY_FLAKE_FAULTY)
           end
 
-          def is_new_test?(test_span)
-            test_id = Utils::TestRun.datadog_test_id(test_span.name, test_span.test_suite_name)
-
-            result = !@unique_tests_set.include?(test_id)
-
-            if result
-              Datadog.logger.debug do
-                "#{test_id} is not found in the unique tests set, it is a new test"
-              end
-            end
-
-            result
-          end
-
           def set_max_attempts_thresholds(library_settings)
             @max_attempts_thresholds = library_settings.slow_test_retries
             Datadog.logger.debug do
@@ -91,7 +70,7 @@ module Datadog
 
           def calculate_total_retries_limit(library_settings, test_session)
             percentage_limit = library_settings.faulty_session_threshold
-            tests_count = test_session.total_tests_count.to_i
+            tests_count = test_session.estimated_total_tests_count.to_i
             if tests_count.zero?
               Datadog.logger.debug do
                 "Total tests count is zero, using default value for the total number of tests: [#{DEFAULT_TOTAL_TESTS_COUNT}]"
@@ -104,27 +83,6 @@ module Datadog
               "Retry new tests total limit is [#{@total_limit}] (#{percentage_limit}% of #{tests_count})"
             end
           end
-
-          def fetch_known_unique_tests(test_session)
-            @unique_tests_set = @unique_tests_client.fetch_unique_tests(test_session)
-            if @unique_tests_set.empty?
-              @enabled = false
-              mark_test_session_faulty(test_session)
-
-              Datadog.logger.warn(
-                "Disabling early flake detection because there are no known tests (possible reason: no test runs in default branch)"
-              )
-            end
-
-            # report how many unique tests were found
-            Datadog.logger.debug do
-              "Found [#{@unique_tests_set.size}] known unique tests"
-            end
-            Utils::Telemetry.distribution(
-              Ext::Telemetry::METRIC_EFD_UNIQUE_TESTS_RESPONSE_TESTS,
-              @unique_tests_set.size.to_f
-            )
-          end
         end
       end
     end

data/lib/datadog/ci/test_session.rb

@@ -12,7 +12,7 @@ module Datadog
     #
     # @public_api
     class TestSession < ConcurrentSpan
-      attr_accessor :total_tests_count
+      attr_accessor :estimated_total_tests_count
 
       # Finishes the current test session.
       # @return [void]

data/lib/datadog/ci/test_suite.rb

@@ -59,6 +59,24 @@ module Datadog
         end
       end
 
+      # @internal
+      def all_executions_failed?(test_id)
+        synchronize do
+          stats = @execution_stats_per_test[test_id]
+          stats && (stats[Ext::Test::Status::FAIL] > 0 || stats[Ext::Test::ExecutionStatsStatus::FAIL_IGNORED] > 0) &&
+            stats[Ext::Test::Status::PASS] == 0
+        end
+      end
+
+      # @internal
+      def all_executions_passed?(test_id)
+        synchronize do
+          stats = @execution_stats_per_test[test_id]
+          stats && stats[Ext::Test::Status::PASS] > 0 && stats[Ext::Test::Status::FAIL] == 0 &&
+            stats[Ext::Test::ExecutionStatsStatus::FAIL_IGNORED] == 0
+        end
+      end
+
       # @internal
       def test_executed?(test_id)
         synchronize do
@@ -66,6 +84,24 @@ module Datadog
         end
       end
 
+      # @internal
+      def set_expected_tests!(expected_tests)
+        synchronize do
+          return if @expected_tests_set
+
+          @expected_tests_set = Set.new(expected_tests)
+        end
+      end
+
+      # @internal
+      def expected_test_done!(test_name)
+        synchronize do
+          @expected_tests_set.delete(test_name)
+
+          finish if @expected_tests_set.empty?
+        end
+      end
+
       private
 
       def set_status_from_stats!
@@ -89,8 +125,9 @@ module Datadog
       end
 
       def derive_test_status_from_execution_stats(test_execution_stats)
-        # test is passed if it passed at least once
-        if test_execution_stats[Ext::Test::Status::PASS] > 0
+        # test is passed if it passed at least once or it failed but fail was ignored
+        if test_execution_stats[Ext::Test::Status::PASS] > 0 ||
+            test_execution_stats[Ext::Test::ExecutionStatsStatus::FAIL_IGNORED] > 0
           Ext::Test::Status::PASS
         # if test was never passed, it is failed if it failed at least once
         elsif test_execution_stats[Ext::Test::Status::FAIL] > 0

data/lib/datadog/ci/test_visibility/capabilities.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require_relative "../ext/test"
+
+module Datadog
+  module CI
+    module TestVisibility
+      # Generates internal tags for library capabilities
+      module Capabilities
+        def self.tags
+          tags = {}
+
+          test_optimisation = Datadog::CI.send(:test_optimisation)
+          tags[Ext::Test::LibraryCapabilities::TAG_TEST_IMPACT_ANALYSIS] = test_optimisation.enabled.to_s
+
+          test_management = Datadog::CI.send(:test_management)
+          test_management_tag_value = test_management.enabled.to_s
+
+          [
+            Ext::Test::LibraryCapabilities::TAG_TEST_MANAGEMENT_ATTEMPT_TO_FIX,
+            Ext::Test::LibraryCapabilities::TAG_TEST_MANAGEMENT_QUARANTINE,
+            Ext::Test::LibraryCapabilities::TAG_TEST_MANAGEMENT_DISABLE
+          ].each do |tag|
+            tags[tag] = test_management_tag_value
+          end
+
+          test_retries = Datadog::CI.send(:test_retries)
+          tags[Ext::Test::LibraryCapabilities::TAG_AUTO_TEST_RETRIES] = test_retries.auto_test_retries_feature_enabled.to_s
+          tags[Ext::Test::LibraryCapabilities::TAG_EARLY_FLAKE_DETECTION] = test_retries.early_flake_detection_feature_enabled.to_s
+
+          tags
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/test_visibility/component.rb

@@ -1,7 +1,10 @@
 # frozen_string_literal: true
 
+require "drb"
 require "rbconfig"
 
+require "datadog/core/utils/forking"
+
 require_relative "context"
 require_relative "telemetry"
 require_relative "total_coverage"
@@ -16,26 +19,49 @@ require_relative "../worker"
 module Datadog
   module CI
     module TestVisibility
-      # Common behavior for CI tests
+      # Core functionality of the library: tracing tests' execution
       class Component
-        attr_reader :test_suite_level_visibility_enabled, :logical_test_session_name
+        include Core::Utils::Forking
+
+        attr_reader :test_suite_level_visibility_enabled, :logical_test_session_name,
+          :known_tests, :known_tests_enabled
 
         def initialize(
+          known_tests_client:,
           test_suite_level_visibility_enabled: false,
           codeowners: Codeowners::Parser.new(Git::LocalRepository.root).parse,
           logical_test_session_name: nil
         )
           @test_suite_level_visibility_enabled = test_suite_level_visibility_enabled
+
           @context = Context.new
+
           @codeowners = codeowners
           @logical_test_session_name = logical_test_session_name
+
+          # "Known tests" feature fetches a list of all tests known to Datadog for this repository
+          # and uses this list to determine if a test is new or not. New tests are marked with "test.is_new" tag.
+          @known_tests_enabled = false
+          @known_tests_client = known_tests_client
+          @known_tests = Set.new
         end
 
-        def start_test_session(service: nil, tags: {}, total_tests_count: 0)
+        def configure(library_configuration, test_session)
+          return unless test_suite_level_visibility_enabled
+
+          if library_configuration.known_tests_enabled?
+            @known_tests_enabled = true
+            fetch_known_tests(test_session)
+          end
+        end
+
+        def start_test_session(service: nil, tags: {}, estimated_total_tests_count: 0)
           return skip_tracing unless test_suite_level_visibility_enabled
 
+          start_drb_service
+
           test_session = @context.start_test_session(service: service, tags: tags)
-          test_session.total_tests_count = total_tests_count
+          test_session.estimated_total_tests_count = estimated_total_tests_count
 
           on_test_session_started(test_session)
           test_session
@@ -52,14 +78,17 @@ module Datadog
         def start_test_suite(test_suite_name, service: nil, tags: {})
           return skip_tracing unless test_suite_level_visibility_enabled
 
-          test_suite = @context.start_test_suite(test_suite_name, service: service, tags: tags)
+          test_suite = maybe_remote_context.start_test_suite(test_suite_name, service: service, tags: tags)
           on_test_suite_started(test_suite)
           test_suite
         end
 
         def trace_test(test_name, test_suite_name, service: nil, tags: {}, &block)
+          test_suite = maybe_remote_context.active_test_suite(test_suite_name)
+          tags[Ext::Test::TAG_SUITE] ||= test_suite_name
+
           if block
-            @context.trace_test(test_name, test_suite_name, service: service, tags: tags) do |test|
+            @context.trace_test(test_name, test_suite, service: service, tags: tags) do |test|
               subscribe_to_after_stop_event(test.tracer_span)
 
               on_test_started(test)
@@ -68,7 +97,7 @@ module Datadog
               res
             end
           else
-            test = @context.trace_test(test_name, test_suite_name, service: service, tags: tags)
+            test = @context.trace_test(test_name, test_suite, service: service, tags: tags)
             subscribe_to_after_stop_event(test.tracer_span)
             on_test_started(test)
             test
@@ -102,7 +131,7 @@ module Datadog
         end
 
         def active_test_suite(test_suite_name)
-          @context.active_test_suite(test_suite_name)
+          maybe_remote_context.active_test_suite(test_suite_name)
         end
 
         def deactivate_test
@@ -130,7 +159,15 @@ module Datadog
           test_suite = active_test_suite(test_suite_name)
           on_test_suite_finished(test_suite) if test_suite
 
-          @context.deactivate_test_suite(test_suite_name)
+          maybe_remote_context.deactivate_test_suite(test_suite_name)
+        end
+
+        def total_tests_count
+          maybe_remote_context.total_tests_count
+        end
+
+        def tests_skipped_by_tia_count
+          maybe_remote_context.tests_skipped_by_tia_count
         end
 
         def itr_enabled?
@@ -158,7 +195,8 @@ module Datadog
           # sets logical test session name if none provided by the user
           override_logical_test_session_name!(test_session) if logical_test_session_name.nil?
 
-          # signal Remote::Component to configure the library
+          # Signal Remote::Component to configure the library.
+          # Note that it will call this component back (unfortunate circular dependency).
           remote.configure(test_session)
         end
 
@@ -173,6 +211,8 @@ module Datadog
         end
 
         def on_test_started(test)
+          maybe_remote_context.incr_total_tests_count
+
           # sometimes test suite is not being assigned correctly
           # fix it by fetching the one single running test suite from the global context
           fix_test_suite!(test) if test.test_suite_id.nil?
@@ -182,12 +222,16 @@ module Datadog
 
           Telemetry.event_created(test)
 
+          mark_test_as_new(test) if new_test?(test)
+
+          test_management.tag_test_from_properties(test)
+
           test_optimisation.mark_if_skippable(test)
           test_optimisation.start_coverage(test)
         end
 
         def on_test_session_finished(test_session)
-          test_optimisation.write_test_session_tags(test_session)
+          test_optimisation.write_test_session_tags(test_session, maybe_remote_context.tests_skipped_by_tia_count)
 
           TotalCoverage.extract_lines_pct(test_session)
 
@@ -195,6 +239,8 @@ module Datadog
         end
 
         def on_test_module_finished(test_module)
+          @context.stop_all_test_suites
+
           Telemetry.event_finished(test_module)
         end
 
@@ -204,7 +250,7 @@ module Datadog
 
         def on_test_finished(test)
           test_optimisation.stop_coverage(test)
-          test_optimisation.count_skipped_test(test)
+          test_optimisation.on_test_finished(test, maybe_remote_context)
 
           Telemetry.event_finished(test)
 
@@ -237,7 +283,7 @@ module Datadog
         def fix_test_suite!(test)
           return unless test_suite_level_visibility_enabled
 
-          test_suite = @context.single_active_test_suite
+          test_suite = maybe_remote_context.single_active_test_suite
           unless test_suite
             Datadog.logger.debug do
               "Trying to fix test suite for test [#{test.name}] but no single test suite is running."
@@ -287,6 +333,49 @@ module Datadog
           end
         end
 
+        def new_test?(test_span)
+          return false unless @known_tests_enabled
+
+          test_id = Utils::TestRun.datadog_test_id(test_span.name, test_span.test_suite_name)
+          result = !@known_tests.include?(test_id)
+
+          if result
+            Datadog.logger.debug do
+              "#{test_id} is not found in the known tests set, it is a new test"
+            end
+          end
+
+          result
+        end
+
+        def fetch_known_tests(test_session)
+          @known_tests = @known_tests_client.fetch(test_session)
+
+          if @known_tests.empty?
+            @known_tests_enabled = false
+
+            # this adds unfortunate knowledge on EFD from Testvisibility, rethink this
+            test_session&.set_tag(Ext::Test::TAG_EARLY_FLAKE_ABORT_REASON, Ext::Test::EARLY_FLAKE_FAULTY)
+
+            Datadog.logger.warn("Empty set of tests known to Datadog")
+          end
+
+          # report how many known tests were found
+          Datadog.logger.debug do
+            "Found [#{@known_tests.size}] known tests"
+          end
+          Utils::Telemetry.distribution(
+            Ext::Telemetry::METRIC_KNOWN_TESTS_RESPONSE_TESTS,
+            @known_tests.size.to_f
+          )
+
+          @known_tests
+        end
+
+        def mark_test_as_new(test_span)
+          test_span.set_tag(Ext::Test::TAG_IS_NEW, "true")
+        end
+
         def test_optimisation
           Datadog.send(:components).test_optimisation
         end
@@ -302,6 +391,31 @@ module Datadog
         def remote
           Datadog.send(:components).ci_remote
         end
+
+        def test_management
+          Datadog.send(:components).test_management
+        end
+
+        # DISTRIBUTED RUBY CONTEXT
+        def start_drb_service
+          return if @context_service_uri
+          return if forked?
+
+          @context_service = DRb.start_service("drbunix:", @context)
+          @context_service_uri = @context_service.uri
+        end
+
+        # depending on whether we are in a forked process or not, returns either the global context or its DRbObject
+        def maybe_remote_context
+          return @context unless forked?
+          return @context_client if defined?(@context_client)
+
+          # once per fork we must stop the running DRb server that was copied from the parent process
+          # otherwise, client will be confused thinking it's server which leads to terrible bugs
+          @context_service.stop_service
+
+          @context_client = DRbObject.new_with_uri(@context_service_uri)
+        end
       end
     end
   end

data/lib/datadog/ci/test_visibility/context.rb

@@ -27,9 +27,16 @@ module Datadog
       # Its responsibility includes building domain models for test visibility as well.
       # Internally it uses Datadog::Tracing module to create spans.
       class Context
+        attr_reader :total_tests_count, :tests_skipped_by_tia_count
+
         def initialize
           @local_context = Store::Local.new
           @global_context = Store::Global.new
+
+          @mutex = Mutex.new
+
+          @total_tests_count = 0
+          @tests_skipped_by_tia_count = 0
         end
 
         def start_test_session(service: nil, tags: {})
@@ -66,17 +73,17 @@ module Datadog
             tracer_span = start_datadog_tracer_span(
               test_suite_name, build_tracing_span_options(service, Ext::AppTypes::TYPE_TEST_SUITE)
             )
-            set_suite_context(tags, span: tracer_span)
+            set_suite_context(tags, test_suite: tracer_span)
 
             build_test_suite(tracer_span, tags)
           end
         end
 
-        def trace_test(test_name, test_suite_name, service: nil, tags: {}, &block)
+        def trace_test(test_name, test_suite, service: nil, tags: {}, &block)
           set_inherited_globals(tags)
           set_session_context(tags)
           set_module_context(tags)
-          set_suite_context(tags, name: test_suite_name)
+          set_suite_context(tags, test_suite: test_suite)
 
           tags[Ext::Test::TAG_NAME] = test_name
           tags[Ext::Test::TAG_TYPE] ||= Ext::Test::Type::TEST
@@ -148,6 +155,10 @@ module Datadog
           @global_context.fetch_single_test_suite
         end
 
+        def stop_all_test_suites
+          @global_context.stop_all_test_suites
+        end
+
         def deactivate_test
           @local_context.deactivate_test
         end
@@ -164,6 +175,14 @@ module Datadog
           @global_context.deactivate_test_suite!(test_suite_name)
         end
 
+        def incr_total_tests_count
+          @mutex.synchronize { @total_tests_count += 1 }
+        end
+
+        def incr_tests_skipped_by_tia_count
+          @mutex.synchronize { @tests_skipped_by_tia_count += 1 }
+        end
+
         private
 
         # BUILDING DOMAIN MODELS
@@ -208,10 +227,6 @@ module Datadog
           ci_span.set_tags(@environment_tags)
 
           ci_span.set_metric(Ext::Test::METRIC_CPU_COUNT, Utils::TestRun.virtual_cpu_count)
-          ci_span.set_tag(
-            Ext::Test::TAG_USER_PROVIDED_TEST_SERVICE,
-            Utils::Configuration.service_name_provided_by_user?.to_s
-          )
         end
 
         # PROPAGATING CONTEXT FROM TOP-LEVEL TO THE LOWER LEVELS
@@ -236,17 +251,11 @@ module Datadog
           end
         end
 
-        def set_suite_context(tags, span: nil, name: nil)
-          return if span.nil? && name.nil?
+        def set_suite_context(tags, test_suite: nil)
+          return if test_suite.nil?
 
-          test_suite = span || active_test_suite(name)
-
-          if test_suite
-            tags[Ext::Test::TAG_TEST_SUITE_ID] = test_suite.id.to_s
-            tags[Ext::Test::TAG_SUITE] = test_suite.name
-          else
-            tags[Ext::Test::TAG_SUITE] = name
-          end
+          tags[Ext::Test::TAG_TEST_SUITE_ID] = test_suite.id.to_s
+          tags[Ext::Test::TAG_SUITE] = test_suite.name
         end
 
         # INTERACTIONS WITH TRACING

data/lib/datadog/ci/{test_retries/unique_tests_client.rb → test_visibility/known_tests.rb}

@@ -10,9 +10,9 @@ require_relative "../utils/test_run"
 
 module Datadog
   module CI
-    module TestRetries
-      # fetch a list of unique known tests from the backend
-      class UniqueTestsClient
+    module TestVisibility
+      # fetches and stores a list of known tests from the backend
+      class KnownTests
         class Response
           def initialize(http_response)
             @http_response = http_response
@@ -66,7 +66,7 @@ module Datadog
           @config_tags = config_tags
         end
 
-        def fetch_unique_tests(test_session)
+        def fetch(test_session)
           api = @api
           return Set.new unless api
 
@@ -78,21 +78,21 @@ module Datadog
             payload: request_payload
           )
 
-          Transport::Telemetry.api_requests(
-            Ext::Telemetry::METRIC_EFD_UNIQUE_TESTS_REQUEST,
+          CI::Transport::Telemetry.api_requests(
+            Ext::Telemetry::METRIC_KNOWN_TESTS_REQUEST,
             1,
             compressed: http_response.request_compressed
           )
-          Utils::Telemetry.distribution(Ext::Telemetry::METRIC_EFD_UNIQUE_TESTS_REQUEST_MS, http_response.duration_ms)
+          Utils::Telemetry.distribution(Ext::Telemetry::METRIC_KNOWN_TESTS_REQUEST_MS, http_response.duration_ms)
           Utils::Telemetry.distribution(
-            Ext::Telemetry::METRIC_EFD_UNIQUE_TESTS_RESPONSE_BYTES,
+            Ext::Telemetry::METRIC_KNOWN_TESTS_RESPONSE_BYTES,
             http_response.response_size.to_f,
             {Ext::Telemetry::TAG_RESPONSE_COMPRESSED => http_response.gzipped_content?.to_s}
           )
 
           unless http_response.ok?
-            Transport::Telemetry.api_requests_errors(
-              Ext::Telemetry::METRIC_EFD_UNIQUE_TESTS_REQUEST_ERRORS,
+            CI::Transport::Telemetry.api_requests_errors(
+              Ext::Telemetry::METRIC_KNOWN_TESTS_REQUEST_ERRORS,
               1,
               error_type: http_response.telemetry_error_type,
               status_code: http_response.code

data/lib/datadog/ci/test_visibility/null_component.rb

@@ -5,7 +5,10 @@ module Datadog
     module TestVisibility
       # Special test visibility component that does not record anything
       class NullComponent
-        def start_test_session(service: nil, tags: {}, total_tests_count: 0)
+        def configure(_, _)
+        end
+
+        def start_test_session(service: nil, tags: {}, estimated_total_tests_count: 0)
           skip_tracing
         end
 

data/lib/datadog/ci/test_visibility/serializers/factories/test_level.rb

@@ -9,7 +9,7 @@ module Datadog
       module Serializers
         module Factories
           # This factory takes care of creating msgpack serializers when test-level visibility is enabled
-          # NOTE: citestcycle is a protocol Datadog uses to submit test execution tracing information to CI visibility
+          # NOTE: citestcycle is a protocol Datadog uses to submit test execution tracing information to Test Optimization
           # backend
           module TestLevel
             module_function