datadog-ci 1.14.0 → 1.16.0

data/lib/datadog/ci/readonly_test_module.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require_relative "test_module"
+
+module Datadog
+  module CI
+    # @internal_api
+    class ReadonlyTestModule < TestModule
+      def initialize(test_module)
+        @id = test_module.id
+        @name = test_module.name
+      end
+      attr_reader :id, :name
+
+      def finish
+        raise "ReadonlyTestModule cannot be finished"
+      end
+
+      def set_tag(key, value)
+        raise "ReadonlyTestModule cannot be modified"
+      end
+
+      def set_metric(key, value)
+        raise "ReadonlyTestModule cannot be modified"
+      end
+    end
+  end
+end

data/lib/datadog/ci/readonly_test_session.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require_relative "test_session"
+
+module Datadog
+  module CI
+    # @internal_api
+    class ReadonlyTestSession < TestSession
+      def initialize(test_session)
+        @id = test_session.id
+        @name = test_session.name
+        @inheritable_tags = test_session.inheritable_tags
+        @service = test_session.service
+      end
+
+      attr_reader :id, :name, :inheritable_tags, :service
+
+      def finish
+        raise "ReadonlyTestSession cannot be finished"
+      end
+
+      def set_tag(key, value)
+        raise "ReadonlyTestSession cannot be modified"
+      end
+
+      def set_metric(key, value)
+        raise "ReadonlyTestSession cannot be modified"
+      end
+    end
+  end
+end

data/lib/datadog/ci/remote/component.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require_relative "../worker"
+require_relative "../utils/stateful"
 
 module Datadog
   module CI
@@ -8,33 +9,44 @@ module Datadog
       # Remote configuration component.
       # Responsible for fetching library settings and configuring the library accordingly.
       class Component
+        include Datadog::CI::Utils::Stateful
+
+        FILE_STORAGE_KEY = "remote_component_state"
+
         def initialize(library_settings_client:)
           @library_settings_client = library_settings_client
+          @library_configuration = nil
         end
 
         # called on test session start, uses test session info to send configuration request to the backend
         def configure(test_session)
-          library_configuration = @library_settings_client.fetch(test_session)
-          # sometimes we can skip code coverage for default branch if there are no changes in the repository
-          # backend needs git metadata uploaded for this test session to check if we can skip code coverage
-          if library_configuration.require_git?
-            Datadog.logger.debug { "Library configuration endpoint requires git upload to be finished, waiting..." }
-            git_tree_upload_worker.wait_until_done
-
-            Datadog.logger.debug { "Requesting library configuration again..." }
-            library_configuration = @library_settings_client.fetch(test_session)
-
-            if library_configuration.require_git?
-              Datadog.logger.debug { "git metadata upload did not complete in time when configuring library" }
+          # If component state is loaded successfully, skip fetching library configuration
+          unless load_component_state
+            @library_configuration = @library_settings_client.fetch(test_session)
+            # sometimes we can skip code coverage for default branch if there are no changes in the repository
+            # backend needs git metadata uploaded for this test session to check if we can skip code coverage
+            if @library_configuration.require_git?
+              Datadog.logger.debug { "Library configuration endpoint requires git upload to be finished, waiting..." }
+              git_tree_upload_worker.wait_until_done
+
+              Datadog.logger.debug { "Requesting library configuration again..." }
+              @library_configuration = @library_settings_client.fetch(test_session)
+
+              if @library_configuration.require_git?
+                Datadog.logger.debug { "git metadata upload did not complete in time when configuring library" }
+              end
             end
+
+            # Store component state for distributed test runs
+            store_component_state if test_session.distributed
           end
 
           # configure different components in parallel because they might block on HTTP requests
           configuration_workers = [
-            Worker.new { test_optimisation.configure(library_configuration, test_session) },
-            Worker.new { test_retries.configure(library_configuration, test_session) },
-            Worker.new { test_visibility.configure(library_configuration, test_session) },
-            Worker.new { test_management.configure(library_configuration, test_session) }
+            Worker.new { test_optimisation.configure(@library_configuration, test_session) },
+            Worker.new { test_retries.configure(@library_configuration, test_session) },
+            Worker.new { test_visibility.configure(@library_configuration, test_session) },
+            Worker.new { test_management.configure(@library_configuration, test_session) }
           ]
 
           # launch configuration workers
@@ -44,6 +56,21 @@ module Datadog
           configuration_workers.each(&:wait_until_done)
         end
 
+        # Implementation of Stateful interface
+        def serialize_state
+          {
+            library_configuration: @library_configuration
+          }
+        end
+
+        def restore_state(state)
+          @library_configuration = state[:library_configuration]
+        end
+
+        def storage_key
+          FILE_STORAGE_KEY
+        end
+
         private
 
         def test_management

data/lib/datadog/ci/test.rb

@@ -25,6 +25,10 @@ module Datadog
       # Finishes the current test.
       # @return [void]
       def finish
+        if is_retry? && retry_reason.nil?
+          set_tag(Ext::Test::TAG_RETRY_REASON, Ext::Test::RetryReason::RETRY_EXTERNAL)
+        end
+
         test_visibility.deactivate_test
 
         super
@@ -68,6 +72,12 @@ module Datadog
         get_tag(Ext::Test::TAG_IS_RETRY) == "true"
       end
 
+      # Returns string with a reason why test was retried
+      # @return [String] retry reason
+      def retry_reason
+        get_tag(Ext::Test::TAG_RETRY_REASON)
+      end
+
       # Returns "true" if this span represents a test that wasn't known to Datadog before.
       # @return [Boolean] true if this test is a new one, false otherwise.
       def is_new?

data/lib/datadog/ci/test_management/component.rb

@@ -2,6 +2,7 @@
 
 require_relative "../ext/telemetry"
 require_relative "../ext/test"
+require_relative "../utils/stateful"
 require_relative "../utils/telemetry"
 require_relative "../utils/test_run"
 
@@ -14,6 +15,10 @@ module Datadog
       # - marking test as disabled causes test to be skipped
       # - marking test as "attempted to fix" causes test to be retried many times to confirm that fix worked
       class Component
+        include Datadog::CI::Utils::Stateful
+
+        FILE_STORAGE_KEY = "test_management_component_state"
+
         attr_reader :enabled, :tests_properties
 
         def initialize(enabled:, tests_properties_client:)
@@ -30,7 +35,11 @@ module Datadog
 
           test_session.set_tag(Ext::Test::TAG_TEST_MANAGEMENT_ENABLED, "true")
 
-          @tests_properties = @tests_properties_client.fetch(test_session)
+          # Load component state first, and if successful, skip fetching tests properties
+          if !load_component_state
+            @tests_properties = @tests_properties_client.fetch(test_session)
+            store_component_state if test_session.distributed
+          end
 
           Utils::Telemetry.distribution(
             Ext::Telemetry::METRIC_TEST_MANAGEMENT_TESTS_RESPONSE_TESTS,
@@ -55,6 +64,30 @@ module Datadog
           test_span.set_tag(Ext::Test::TAG_IS_TEST_DISABLED, "true") if test_properties["disabled"]
           test_span.set_tag(Ext::Test::TAG_IS_ATTEMPT_TO_FIX, "true") if test_properties["attempt_to_fix"]
         end
+
+        def attempt_to_fix?(datadog_fqn_test_id)
+          return false unless @enabled
+
+          test_properties = @tests_properties[datadog_fqn_test_id]
+          return false if test_properties.nil?
+
+          test_properties.fetch("attempt_to_fix", false)
+        end
+
+        # Implementation of Stateful interface
+        def serialize_state
+          {
+            tests_properties: @tests_properties
+          }
+        end
+
+        def restore_state(state)
+          @tests_properties = state[:tests_properties]
+        end
+
+        def storage_key
+          FILE_STORAGE_KEY
+        end
       end
     end
   end

data/lib/datadog/ci/test_management/tests_properties.rb

@@ -117,7 +117,8 @@ module Datadog
               "id" => Datadog::Core::Environment::Identity.id,
               "type" => Ext::Transport::DD_API_TEST_MANAGEMENT_TESTS_TYPE,
               "attributes" => {
-                "repository_url" => test_session.git_repository_url
+                "repository_url" => test_session.git_repository_url,
+                "commit_message" => test_session.git_commit_message
               }
             }
           }.to_json

data/lib/datadog/ci/test_optimisation/component.rb

@@ -10,6 +10,7 @@ require_relative "../ext/telemetry"
 require_relative "../git/local_repository"
 
 require_relative "../utils/parsing"
+require_relative "../utils/stateful"
 require_relative "../utils/telemetry"
 
 require_relative "coverage/event"
@@ -23,6 +24,10 @@ module Datadog
       # Integrates with backend to provide test impact analysis data and
       # skip tests that are not impacted by the changes
       class Component
+        include Datadog::CI::Utils::Stateful
+
+        FILE_STORAGE_KEY = "test_optimisation_component_state"
+
         attr_reader :correlation_id, :skippable_tests, :skippable_tests_fetch_error,
           :enabled, :test_skipping_enabled, :code_coverage_enabled
 
@@ -78,9 +83,13 @@ module Datadog
 
           load_datadog_cov! if @code_coverage_enabled
 
-          Datadog.logger.debug("Configured TestOptimisation with enabled: #{@enabled}, skipping_tests: #{@test_skipping_enabled}, code_coverage: #{@code_coverage_enabled}")
+          # Load component state first, and if successful, skip fetching skippable tests
+          if skipping_tests? && !load_component_state
+            fetch_skippable_tests(test_session)
+            store_component_state if test_session.distributed
+          end
 
-          fetch_skippable_tests(test_session)
+          Datadog.logger.debug("Configured TestOptimisation with enabled: #{@enabled}, skipping_tests: #{@test_skipping_enabled}, code_coverage: #{@code_coverage_enabled}")
         end
 
         def enabled?
@@ -138,16 +147,16 @@ module Datadog
           event
         end
 
-        def skippable?(test)
+        def skippable?(datadog_test_id)
           return false if !enabled? || !skipping_tests?
 
-          @skippable_tests.include?(test.datadog_test_id)
+          @skippable_tests.include?(datadog_test_id)
         end
 
         def mark_if_skippable(test)
           return if !enabled? || !skipping_tests?
 
-          if skippable?(test)
+          if skippable?(test.datadog_test_id) && !test.attempt_to_fix?
             test.set_tag(Ext::Test::TAG_ITR_SKIPPED_BY_ITR, "true")
 
             Datadog.logger.debug { "Marked test as skippable: #{test.datadog_test_id}" }
@@ -182,6 +191,23 @@ module Datadog
           @coverage_writer&.stop
         end
 
+        # Implementation of Stateful interface
+        def serialize_state
+          {
+            correlation_id: @correlation_id,
+            skippable_tests: @skippable_tests
+          }
+        end
+
+        def restore_state(state)
+          @correlation_id = state[:correlation_id]
+          @skippable_tests = state[:skippable_tests]
+        end
+
+        def storage_key
+          FILE_STORAGE_KEY
+        end
+
         private
 
         def write(event)

data/lib/datadog/ci/test_retries/driver/retry_new.rb

@@ -36,7 +36,7 @@ module Datadog
           end
 
           def retry_reason
-            Ext::Test::RetryReason::RETRY_NEW
+            Ext::Test::RetryReason::RETRY_DETECT_FLAKY
           end
         end
       end

data/lib/datadog/ci/test_session.rb

@@ -12,7 +12,7 @@ module Datadog
     #
     # @public_api
     class TestSession < ConcurrentSpan
-      attr_accessor :estimated_total_tests_count
+      attr_accessor :estimated_total_tests_count, :distributed
 
       # Finishes the current test session.
       # @return [void]
@@ -46,6 +46,12 @@ module Datadog
         get_tag(Ext::Environment::TAG_JOB_NAME)
       end
 
+      # Returns the git commit message extracted from the environment.
+      # @return [String] the commit message.
+      def git_commit_message
+        get_tag(Ext::Git::TAG_COMMIT_MESSAGE)
+      end
+
       def skipping_tests?
         get_tag(Ext::Test::TAG_ITR_TEST_SKIPPING_ENABLED) == "true"
       end

data/lib/datadog/ci/test_visibility/component.rb

@@ -13,6 +13,8 @@ require_relative "../codeowners/parser"
 require_relative "../contrib/instrumentation"
 require_relative "../ext/test"
 require_relative "../git/local_repository"
+require_relative "../utils/file_storage"
+require_relative "../utils/stateful"
 
 require_relative "../worker"
 
@@ -22,19 +24,23 @@ module Datadog
       # Core functionality of the library: tracing tests' execution
       class Component
         include Core::Utils::Forking
+        include Datadog::CI::Utils::Stateful
+
+        FILE_STORAGE_KEY = "test_visibility_component_state"
 
         attr_reader :test_suite_level_visibility_enabled, :logical_test_session_name,
-          :known_tests, :known_tests_enabled
+          :known_tests, :known_tests_enabled, :context_service_uri, :local_test_suites_mode
 
         def initialize(
           known_tests_client:,
           test_suite_level_visibility_enabled: false,
           codeowners: Codeowners::Parser.new(Git::LocalRepository.root).parse,
-          logical_test_session_name: nil
+          logical_test_session_name: nil,
+          context_service_uri: nil
         )
           @test_suite_level_visibility_enabled = test_suite_level_visibility_enabled
 
-          @context = Context.new
+          @context = Context.new(test_visibility_component: self)
 
           @codeowners = codeowners
           @logical_test_session_name = logical_test_session_name
@@ -44,47 +50,72 @@ module Datadog
           @known_tests_enabled = false
           @known_tests_client = known_tests_client
           @known_tests = Set.new
+
+          # this is used for parallel test runners such as parallel_tests
+          if context_service_uri
+            @context_service_uri = context_service_uri
+            @is_client_process = true
+          end
+
+          # This is used for parallel test runners such as parallel_tests.
+          # If true, then test suites are created in the worker process, not the parent.
+          #
+          # The only test runner that requires creating test suites in the remote process is rails test runner because
+          # it splits workload by test, not by test suite.
+          #
+          # Another test runner that splits workload by test is knapsack_pro, but we lack distributed test sessions/test suties
+          # support for that one (as of 2025-03).
+          @local_test_suites_mode = true
         end
 
         def configure(library_configuration, test_session)
           return unless test_suite_level_visibility_enabled
+          return unless library_configuration.known_tests_enabled?
 
-          if library_configuration.known_tests_enabled?
-            @known_tests_enabled = true
-            fetch_known_tests(test_session)
-          end
+          @known_tests_enabled = true
+          return if load_component_state
+
+          fetch_known_tests(test_session)
+          store_component_state if test_session.distributed
         end
 
-        def start_test_session(service: nil, tags: {}, estimated_total_tests_count: 0)
+        def start_test_session(service: nil, tags: {}, estimated_total_tests_count: 0, distributed: false, local_test_suites_mode: true)
           return skip_tracing unless test_suite_level_visibility_enabled
 
+          @local_test_suites_mode = local_test_suites_mode
+
           start_drb_service
 
-          test_session = @context.start_test_session(service: service, tags: tags)
+          test_session = maybe_remote_context.start_test_session(service: service, tags: tags)
           test_session.estimated_total_tests_count = estimated_total_tests_count
+          test_session.distributed = distributed
 
           on_test_session_started(test_session)
+
           test_session
         end
 
         def start_test_module(test_module_name, service: nil, tags: {})
           return skip_tracing unless test_suite_level_visibility_enabled
 
-          test_module = @context.start_test_module(test_module_name, service: service, tags: tags)
+          test_module = maybe_remote_context.start_test_module(test_module_name, service: service, tags: tags)
           on_test_module_started(test_module)
+
           test_module
         end
 
         def start_test_suite(test_suite_name, service: nil, tags: {})
           return skip_tracing unless test_suite_level_visibility_enabled
 
-          test_suite = maybe_remote_context.start_test_suite(test_suite_name, service: service, tags: tags)
+          context = @local_test_suites_mode ? @context : maybe_remote_context
+
+          test_suite = context.start_test_suite(test_suite_name, service: service, tags: tags)
           on_test_suite_started(test_suite)
           test_suite
         end
 
         def trace_test(test_name, test_suite_name, service: nil, tags: {}, &block)
-          test_suite = maybe_remote_context.active_test_suite(test_suite_name)
+          test_suite = active_test_suite(test_suite_name)
           tags[Ext::Test::TAG_SUITE] ||= test_suite_name
 
           if block
@@ -123,14 +154,18 @@ module Datadog
         end
 
         def active_test_session
-          @context.active_test_session
+          maybe_remote_context.active_test_session
         end
 
         def active_test_module
-          @context.active_test_module
+          maybe_remote_context.active_test_module
         end
 
         def active_test_suite(test_suite_name)
+          # when fetching test_suite to use as test's context, try local context instance first
+          local_test_suite = @context.active_test_suite(test_suite_name)
+          return local_test_suite if local_test_suite
+
           maybe_remote_context.active_test_suite(test_suite_name)
         end
 
@@ -159,7 +194,8 @@ module Datadog
           test_suite = active_test_suite(test_suite_name)
           on_test_suite_finished(test_suite) if test_suite
 
-          maybe_remote_context.deactivate_test_suite(test_suite_name)
+          # deactivation always happens on the same process where test suite is located
+          @context.deactivate_test_suite(test_suite_name)
         end
 
         def total_tests_count
@@ -178,6 +214,10 @@ module Datadog
           # noop, there is no thread owned by test visibility component
         end
 
+        def client_process?
+          forked? || @is_client_process
+        end
+
         private
 
         # DOMAIN EVENTS
@@ -188,10 +228,6 @@ module Datadog
           # finds and instruments additional test libraries that we support (ex: selenium-webdriver)
           Contrib::Instrumentation.instrument_on_session_start
 
-          # sends internal telemetry events
-          Telemetry.test_session_started(test_session)
-          Telemetry.event_created(test_session)
-
           # sets logical test session name if none provided by the user
           override_logical_test_session_name!(test_session) if logical_test_session_name.nil?
 
@@ -200,21 +236,22 @@ module Datadog
           remote.configure(test_session)
         end
 
+        # intentionally empty
         def on_test_module_started(test_module)
-          Telemetry.event_created(test_module)
         end
 
         def on_test_suite_started(test_suite)
           set_codeowners(test_suite)
-
-          Telemetry.event_created(test_suite)
         end
 
         def on_test_started(test)
           maybe_remote_context.incr_total_tests_count
 
-          # sometimes test suite is not being assigned correctly
-          # fix it by fetching the one single running test suite from the global context
+          # Sometimes test suite is not being assigned correctly.
+          # Fix it by fetching the one single running test suite from the process context.
+          #
+          # This is a hack to fix some edge cases that come from some minitest plugins,
+          # especially thoughtbot/shoulda-context.
           fix_test_suite!(test) if test.test_suite_id.nil?
           validate_test_suite_level_visibility_correctness(test)
 
@@ -236,6 +273,8 @@ module Datadog
           TotalCoverage.extract_lines_pct(test_session)
 
           Telemetry.event_finished(test_session)
+
+          Utils::FileStorage.cleanup
         end
 
         def on_test_module_finished(test_module)
@@ -399,7 +438,7 @@ module Datadog
         # DISTRIBUTED RUBY CONTEXT
         def start_drb_service
           return if @context_service_uri
-          return if forked?
+          return if client_process?
 
           @context_service = DRb.start_service("drbunix:", @context)
           @context_service_uri = @context_service.uri
@@ -407,15 +446,30 @@ module Datadog
 
         # depending on whether we are in a forked process or not, returns either the global context or its DRbObject
         def maybe_remote_context
-          return @context unless forked?
+          return @context unless client_process?
           return @context_client if defined?(@context_client)
 
-          # once per fork we must stop the running DRb server that was copied from the parent process
+          # at least once per fork we must stop the running DRb server that was copied from the parent process
           # otherwise, client will be confused thinking it's server which leads to terrible bugs
-          @context_service.stop_service
+          @context_service&.stop_service
 
           @context_client = DRbObject.new_with_uri(@context_service_uri)
         end
+
+        # Implementation of Stateful interface
+        def serialize_state
+          {
+            known_tests: @known_tests
+          }
+        end
+
+        def restore_state(state)
+          @known_tests = state[:known_tests]
+        end
+
+        def storage_key
+          FILE_STORAGE_KEY
+        end
       end
     end
   end