datadog-ci 1.1.0 → 1.3.0

data/lib/datadog/ci/configuration/components.rb

@@ -1,18 +1,23 @@
 # frozen_string_literal: true
 
+require "datadog/core/telemetry/ext"
+
 require_relative "../ext/settings"
 require_relative "../git/tree_uploader"
-require_relative "../itr/runner"
-require_relative "../itr/coverage/transport"
-require_relative "../itr/coverage/writer"
+require_relative "../test_optimisation/component"
+require_relative "../test_optimisation/coverage/transport"
+require_relative "../test_optimisation/coverage/writer"
+require_relative "../test_visibility/component"
 require_relative "../test_visibility/flush"
-require_relative "../test_visibility/recorder"
-require_relative "../test_visibility/null_recorder"
+require_relative "../test_visibility/null_component"
 require_relative "../test_visibility/serializers/factories/test_level"
 require_relative "../test_visibility/serializers/factories/test_suite_level"
 require_relative "../test_visibility/transport"
+require_relative "../transport/adapters/telemetry_webmock_safe_adapter"
 require_relative "../transport/api/builder"
 require_relative "../transport/remote_settings_api"
+require_relative "../utils/identity"
+require_relative "../utils/parsing"
 require_relative "../utils/test_run"
 require_relative "../worker"
 
@@ -21,15 +26,15 @@ module Datadog
     module Configuration
       # Adds CI behavior to Datadog trace components
       module Components
-        attr_reader :ci_recorder, :itr
+        attr_reader :test_visibility, :test_optimisation
 
         def initialize(settings)
+          @test_optimisation = nil
+          @test_visibility = TestVisibility::NullComponent.new
+
           # Activate CI mode if enabled
           if settings.ci.enabled
             activate_ci!(settings)
-          else
-            @itr = nil
-            @ci_recorder = TestVisibility::NullRecorder.new
           end
 
           super
@@ -38,8 +43,8 @@ module Datadog
         def shutdown!(replacement = nil)
           super
 
-          @ci_recorder&.shutdown!
-          @itr&.shutdown!
+          @test_visibility&.shutdown!
+          @test_optimisation&.shutdown!
         end
 
         def activate_ci!(settings)
@@ -53,22 +58,19 @@ module Datadog
             return
           end
 
-          # Configure ddtrace library for CI visibility mode
-          # Deactivate telemetry
-          settings.telemetry.enabled = false
-
-          # Deactivate remote configuration
-          settings.remote.enabled = false
+          # Builds test visibility API layer in agentless or EvP proxy mode
+          test_visibility_api = build_test_visibility_api(settings)
+          # bail out early if api is misconfigured
+          return unless settings.ci.enabled
 
-          # do not use 128-bit trace ids for CI visibility
-          # they are used for OTEL compatibility in Datadog tracer
-          settings.tracing.trace_id_128_bit_generation_enabled = false
+          # Configure datadog gem for test visibility mode
+          configure_telemetry(settings)
 
-          # Activate underlying tracing test mode
-          settings.tracing.test_mode.enabled = true
+          # Test visibility uses its own remote settings
+          settings.remote.enabled = false
 
-          # Choose user defined TraceFlush or default to CI TraceFlush
-          settings.tracing.test_mode.trace_flush = settings.ci.trace_flush || CI::TestVisibility::Flush::Partial.new
+          # startup logs are useless for test visibility and create noise
+          settings.diagnostics.startup_logs.enabled = false
 
           # When timecop is present, Time.now is mocked and .now_without_mock_time is added on Time to
           # get the current time without the mock.
@@ -81,76 +83,70 @@ module Datadog
             end
           end
 
-          # startup logs are useless for CI visibility and create noise
-          settings.diagnostics.startup_logs.enabled = false
+          # Configure Datadog::Tracing module
 
-          # transport creation
-          writer_options = settings.ci.writer_options
-          coverage_writer = nil
-          test_visibility_api = build_test_visibility_api(settings)
+          # No need not use 128-bit trace ids for test visibility,
+          # they are used for OTEL compatibility in Datadog tracer
+          settings.tracing.trace_id_128_bit_generation_enabled = false
 
-          if test_visibility_api
-            # setup writer for code coverage payloads
-            coverage_writer = ITR::Coverage::Writer.new(
-              transport: ITR::Coverage::Transport.new(api: test_visibility_api)
-            )
+          # Activate underlying tracing test mode with async worker
+          settings.tracing.test_mode.enabled = true
+          settings.tracing.test_mode.async = true
+          settings.tracing.test_mode.trace_flush = settings.ci.trace_flush || CI::TestVisibility::Flush::Partial.new
 
-            # configure tracing writer to send traces to CI visibility backend
-            writer_options[:transport] = TestVisibility::Transport.new(
-              api: test_visibility_api,
-              serializers_factory: serializers_factory(settings),
-              dd_env: settings.env
-            )
-            writer_options[:shutdown_timeout] = 60
-            writer_options[:buffer_size] = 10_000
+          trace_writer_options = settings.ci.writer_options
+          trace_writer_options[:shutdown_timeout] = 60
+          trace_writer_options[:buffer_size] = 10_000
+          tracing_transport = build_tracing_transport(settings, test_visibility_api)
+          trace_writer_options[:transport] = tracing_transport if tracing_transport
 
-            settings.tracing.test_mode.async = true
-          else
-            # only legacy APM protocol is supported, so no test suite level visibility
-            settings.ci.force_test_level_visibility = true
+          settings.tracing.test_mode.writer_options = trace_writer_options
 
-            # ITR is not supported with APM protocol
-            settings.ci.itr_enabled = false
-          end
+          # @type ivar @test_optimisation: Datadog::CI::TestOptimisation::Component
+          @test_optimisation = build_test_optimisation(settings, test_visibility_api)
 
-          settings.tracing.test_mode.writer_options = writer_options
+          @test_visibility = TestVisibility::Component.new(
+            test_optimisation: @test_optimisation,
+            test_suite_level_visibility_enabled: !settings.ci.force_test_level_visibility,
+            remote_settings_api: build_remote_settings_client(settings, test_visibility_api),
+            git_tree_upload_worker: build_git_upload_worker(settings, test_visibility_api)
+          )
+        end
 
-          custom_configuration_tags = Utils::TestRun.custom_configuration(settings.tags)
+        def build_test_optimisation(settings, test_visibility_api)
+          if settings.ci.itr_code_coverage_use_single_threaded_mode &&
+              settings.ci.itr_test_impact_analysis_use_allocation_tracing
+            Datadog.logger.warn(
+              "Intelligent test runner: Single threaded coverage mode is incompatible with allocation tracing. " \
+              "Allocation tracing will be disabled. It means that test impact analysis will not be able to detect " \
+              "instantiations of objects in your code, which is important for ActiveRecord models. " \
+              "Please add your app/model folder to the list of tracked files or disable single threaded coverage mode."
+            )
 
-          remote_settings_api = Transport::RemoteSettingsApi.new(
-            api: test_visibility_api,
-            dd_env: settings.env,
-            config_tags: custom_configuration_tags
-          )
+            settings.ci.itr_test_impact_analysis_use_allocation_tracing = false
+          end
 
-          itr = ITR::Runner.new(
+          if RUBY_VERSION.start_with?("3.2.") && RUBY_VERSION < "3.2.3" &&
+              settings.ci.itr_test_impact_analysis_use_allocation_tracing
+            Datadog.logger.warn(
+              "Intelligent test runner: Allocation tracing is not supported in Ruby versions 3.2.0, 3.2.1 and 3.2.2 and will be forcibly " \
+              "disabled. This is due to a VM bug that can lead to crashes (https://bugs.ruby-lang.org/issues/19482). " \
+              "Please update your Ruby version or add your app/model folder to the list of tracked files." \
+              "Set env variable DD_CIVISIBILITY_ITR_TEST_IMPACT_ANALYSIS_USE_ALLOCATION_TRACING to 0 to disable this warning."
+            )
+            settings.ci.itr_test_impact_analysis_use_allocation_tracing = false
+          end
+
+          TestOptimisation::Component.new(
             api: test_visibility_api,
             dd_env: settings.env,
-            config_tags: custom_configuration_tags,
-            coverage_writer: coverage_writer,
+            config_tags: custom_configuration(settings),
+            coverage_writer: build_coverage_writer(settings, test_visibility_api),
             enabled: settings.ci.enabled && settings.ci.itr_enabled,
             bundle_location: settings.ci.itr_code_coverage_excluded_bundle_path,
-            use_single_threaded_coverage: settings.ci.itr_code_coverage_use_single_threaded_mode
+            use_single_threaded_coverage: settings.ci.itr_code_coverage_use_single_threaded_mode,
+            use_allocation_tracing: settings.ci.itr_test_impact_analysis_use_allocation_tracing
           )
-
-          git_tree_uploader = Git::TreeUploader.new(api: test_visibility_api)
-          git_tree_upload_worker = if settings.ci.git_metadata_upload_enabled
-            Worker.new do |repository_url|
-              git_tree_uploader.call(repository_url)
-            end
-          else
-            DummyWorker.new
-          end
-
-          # CI visibility recorder global instance
-          @ci_recorder = TestVisibility::Recorder.new(
-            test_suite_level_visibility_enabled: !settings.ci.force_test_level_visibility,
-            itr: itr,
-            remote_settings_api: remote_settings_api,
-            git_tree_upload_worker: git_tree_upload_worker
-          )
-
-          @itr = itr
         end
 
         def build_test_visibility_api(settings)
@@ -179,12 +175,61 @@ module Datadog
               Datadog.logger.debug(
                 "Old agent version detected, no evp_proxy support. Forcing test level visibility mode"
               )
+
+              # only legacy APM protocol is supported, so no test suite level visibility
+              settings.ci.force_test_level_visibility = true
+
+              # ITR is not supported with APM protocol
+              settings.ci.itr_enabled = false
             end
           end
 
           api
         end
 
+        def build_tracing_transport(settings, api)
+          return nil if api.nil?
+
+          TestVisibility::Transport.new(
+            api: api,
+            serializers_factory: serializers_factory(settings),
+            dd_env: settings.env
+          )
+        end
+
+        def build_coverage_writer(settings, api)
+          return nil if api.nil?
+
+          TestOptimisation::Coverage::Writer.new(
+            transport: TestOptimisation::Coverage::Transport.new(api: api)
+          )
+        end
+
+        def build_git_upload_worker(settings, api)
+          if settings.ci.git_metadata_upload_enabled
+            git_tree_uploader = Git::TreeUploader.new(api: api)
+            Worker.new do |repository_url|
+              git_tree_uploader.call(repository_url)
+            end
+          else
+            DummyWorker.new
+          end
+        end
+
+        def build_remote_settings_client(settings, api)
+          Transport::RemoteSettingsApi.new(
+            api: api,
+            dd_env: settings.env,
+            config_tags: custom_configuration(settings)
+          )
+        end
+
+        # fetch custom tags provided by the user in DD_TAGS env var
+        # with prefix test.configuration.
+        def custom_configuration(settings)
+          @custom_configuration ||= Utils::TestRun.custom_configuration(settings.tags)
+        end
+
         def serializers_factory(settings)
           if settings.ci.force_test_level_visibility
             TestVisibility::Serializers::Factories::TestLevel
@@ -204,6 +249,30 @@ module Datadog
           end
         end
 
+        def configure_telemetry(settings)
+          # in development environment Datadog's telemetry is disabled by default
+          # for test visibility we want to enable it by default unless explicitly disabled
+          # NOTE: before agentless mode is released, we only enable telemetry when running with Datadog Agent
+          env_telemetry_enabled = ENV[Core::Telemetry::Ext::ENV_ENABLED]
+          settings.telemetry.enabled = !settings.ci.agentless_mode_enabled &&
+            (env_telemetry_enabled.nil? || Utils::Parsing.convert_to_bool(env_telemetry_enabled))
+
+          return unless settings.telemetry.enabled
+
+          begin
+            require "datadog/core/environment/identity"
+            require "datadog/core/telemetry/http/adapters/net"
+
+            # patch gem's identity to report datadog-ci library version instead of datadog gem version
+            Core::Environment::Identity.include(CI::Utils::Identity)
+
+            # patch gem's telemetry transport layer to use Net::HTTP instead of WebMock's Net::HTTP
+            Core::Telemetry::Http::Adapters::Net.include(CI::Transport::Adapters::TelemetryWebmockSafeAdapter)
+          rescue => e
+            Datadog.logger.warn("Failed to patch Datadog gem's telemetry layer: #{e}")
+          end
+        end
+
         def timecop?
           Gem.loaded_specs.key?("timecop") || !!defined?(Timecop)
         end

data/lib/datadog/ci/configuration/settings.rb

@@ -82,6 +82,12 @@ module Datadog
                 o.default false
               end
 
+              option :itr_test_impact_analysis_use_allocation_tracing do |o|
+                o.type :bool
+                o.env CI::Ext::Settings::ENV_ITR_TEST_IMPACT_ANALYSIS_USE_ALLOCATION_TRACING
+                o.default true
+              end
+
               define_method(:instrument) do |integration_name, options = {}, &block|
                 return unless enabled
 

data/lib/datadog/ci/contrib/cucumber/formatter.rb

@@ -35,14 +35,14 @@ module Datadog
           end
 
           def on_test_run_started(event)
-            CI.start_test_session(
+            test_visibility_component.start_test_session(
               tags: {
                 CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK,
                 CI::Ext::Test::TAG_FRAMEWORK_VERSION => CI::Contrib::Cucumber::Integration.version.to_s
               },
               service: configuration[:service_name]
             )
-            CI.start_test_module(Ext::FRAMEWORK)
+            test_visibility_component.start_test_module(Ext::FRAMEWORK)
           end
 
           def on_test_run_finished(event)
@@ -70,7 +70,7 @@ module Datadog
 
             start_test_suite(test_suite_name) unless same_test_suite_as_current?(test_suite_name)
 
-            test_span = CI.start_test(
+            test_span = test_visibility_component.trace_test(
               event.test_case.name,
               test_suite_name,
               tags: tags,
@@ -82,7 +82,7 @@ module Datadog
           end
 
           def on_test_case_finished(event)
-            test_span = CI.active_test
+            test_span = test_visibility_component.active_test
             return if test_span.nil?
 
             finish_span(test_span, event.result)
@@ -90,11 +90,11 @@ module Datadog
           end
 
           def on_test_step_started(event)
-            CI.trace(event.test_step.to_s, type: Ext::STEP_SPAN_TYPE)
+            test_visibility_component.trace(event.test_step.to_s, type: Ext::STEP_SPAN_TYPE)
           end
 
           def on_test_step_finished(event)
-            current_step_span = CI.active_span
+            current_step_span = test_visibility_component.active_span
             return if current_step_span.nil?
 
             finish_span(current_step_span, event.result)
@@ -130,8 +130,8 @@ module Datadog
           def finish_session(result)
             finish_current_test_suite
 
-            test_session = CI.active_test_session
-            test_module = CI.active_test_module
+            test_session = test_visibility_component.active_test_session
+            test_module = test_visibility_component.active_test_module
 
             return unless test_session && test_module
 
@@ -150,7 +150,7 @@ module Datadog
           def start_test_suite(test_suite_name)
             finish_current_test_suite
 
-            @current_test_suite = CI.start_test_suite(test_suite_name)
+            @current_test_suite = test_visibility_component.start_test_suite(test_suite_name)
           end
 
           def finish_current_test_suite
@@ -197,6 +197,10 @@ module Datadog
           def configuration
             Datadog.configuration.ci[:cucumber]
           end
+
+          def test_visibility_component
+            Datadog.send(:components).test_visibility
+          end
         end
       end
     end

data/lib/datadog/ci/contrib/minitest/runnable.rb

@@ -19,7 +19,7 @@ module Datadog
 
               test_suite_name = Helpers.test_suite_name(self, method)
 
-              test_suite = Datadog::CI.start_test_suite(test_suite_name)
+              test_suite = test_visibility_component.start_test_suite(test_suite_name)
 
               results = super
               return results unless test_suite
@@ -34,6 +34,10 @@ module Datadog
             def datadog_configuration
               Datadog.configuration.ci[:minitest]
             end
+
+            def test_visibility_component
+              Datadog.send(:components).test_visibility
+            end
           end
         end
       end

data/lib/datadog/ci/contrib/minitest/runner.rb

@@ -18,14 +18,14 @@ module Datadog
 
               return unless datadog_configuration[:enabled]
 
-              CI.start_test_session(
+              test_visibility_component.start_test_session(
                 tags: {
                   CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK,
                   CI::Ext::Test::TAG_FRAMEWORK_VERSION => CI::Contrib::Minitest::Integration.version.to_s
                 },
                 service: datadog_configuration[:service_name]
               )
-              CI.start_test_module(Ext::FRAMEWORK)
+              test_visibility_component.start_test_module(Ext::FRAMEWORK)
             end
 
             private
@@ -33,6 +33,10 @@ module Datadog
             def datadog_configuration
               Datadog.configuration.ci[:minitest]
             end
+
+            def test_visibility_component
+              Datadog.send(:components).test_visibility
+            end
           end
         end
       end

data/lib/datadog/ci/contrib/minitest/test.rb

@@ -26,12 +26,12 @@ module Datadog
                 test_suite_name = "#{test_suite_name} (#{name} concurrently)"
 
                 # for parallel execution we need to start a new test suite for each test
-                CI.start_test_suite(test_suite_name)
+                test_visibility_component.start_test_suite(test_suite_name)
               end
 
               source_file, line_number = method(name).source_location
 
-              test_span = CI.start_test(
+              test_span = test_visibility_component.trace_test(
                 name,
                 test_suite_name,
                 tags: {
@@ -47,7 +47,7 @@ module Datadog
             end
 
             def after_teardown
-              test_span = CI.active_test
+              test_span = test_visibility_component.active_test
               return super unless test_span
 
               finish_with_result(test_span, result_code)
@@ -77,6 +77,10 @@ module Datadog
             def datadog_configuration
               Datadog.configuration.ci[:minitest]
             end
+
+            def test_visibility_component
+              Datadog.send(:components).test_visibility
+            end
           end
 
           module ClassMethods

data/lib/datadog/ci/contrib/rspec/example.rb

@@ -34,10 +34,10 @@ module Datadog
 
               if ci_queue?
                 suite_name = "#{suite_name} (ci-queue running example [#{test_name}])"
-                test_suite_span = CI.start_test_suite(suite_name)
+                test_suite_span = test_visibility_component.start_test_suite(suite_name)
               end
 
-              CI.trace_test(
+              test_visibility_component.trace_test(
                 test_name,
                 suite_name,
                 tags: {
@@ -99,6 +99,10 @@ module Datadog
               Datadog.configuration.ci[:rspec]
             end
 
+            def test_visibility_component
+              Datadog.send(:components).test_visibility
+            end
+
             def ci_queue?
               !!defined?(::RSpec::Queue::ExampleExtension) &&
                 self.class.ancestors.include?(::RSpec::Queue::ExampleExtension)

data/lib/datadog/ci/contrib/rspec/example_group.rb

@@ -21,7 +21,7 @@ module Datadog
               return super unless top_level?
 
               suite_name = "#{description} at #{file_path}"
-              test_suite = Datadog::CI.start_test_suite(suite_name)
+              test_suite = test_visibility_component.start_test_suite(suite_name)
 
               success = super
               return success unless test_suite
@@ -44,6 +44,10 @@ module Datadog
             def datadog_configuration
               Datadog.configuration.ci[:rspec]
             end
+
+            def test_visibility_component
+              Datadog.send(:components).test_visibility
+            end
           end
         end
       end

data/lib/datadog/ci/contrib/rspec/knapsack_pro/runner.rb

@@ -18,7 +18,7 @@ module Datadog
                 return super if ::RSpec.configuration.dry_run?
                 return super unless datadog_configuration[:enabled]
 
-                test_session = CI.start_test_session(
+                test_session = test_visibility_component.start_test_session(
                   tags: {
                     CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK,
                     CI::Ext::Test::TAG_FRAMEWORK_VERSION => CI::Contrib::RSpec::Integration.version.to_s
@@ -26,7 +26,7 @@ module Datadog
                   service: datadog_configuration[:service_name]
                 )
 
-                test_module = CI.start_test_module(Ext::FRAMEWORK)
+                test_module = test_visibility_component.start_test_module(Ext::FRAMEWORK)
 
                 result = super
                 return result unless test_module && test_session
@@ -49,6 +49,10 @@ module Datadog
               def datadog_configuration
                 Datadog.configuration.ci[:rspec]
               end
+
+              def test_visibility_component
+                Datadog.send(:components).test_visibility
+              end
             end
           end
         end

data/lib/datadog/ci/contrib/rspec/runner.rb

@@ -18,7 +18,7 @@ module Datadog
               return super if ::RSpec.configuration.dry_run?
               return super unless datadog_configuration[:enabled]
 
-              test_session = CI.start_test_session(
+              test_session = test_visibility_component.start_test_session(
                 tags: {
                   CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK,
                   CI::Ext::Test::TAG_FRAMEWORK_VERSION => CI::Contrib::RSpec::Integration.version.to_s
@@ -26,7 +26,7 @@ module Datadog
                 service: datadog_configuration[:service_name]
               )
 
-              test_module = CI.start_test_module(Ext::FRAMEWORK)
+              test_module = test_visibility_component.start_test_module(Ext::FRAMEWORK)
 
               result = super
               return result unless test_module && test_session
@@ -49,6 +49,10 @@ module Datadog
             def datadog_configuration
               Datadog.configuration.ci[:rspec]
             end
+
+            def test_visibility_component
+              Datadog.send(:components).test_visibility
+            end
           end
         end
       end

data/lib/datadog/ci/ext/environment/providers/appveyor.rb

@@ -15,7 +15,7 @@ module Datadog
             end
 
             def provider_name
-              "appveyor"
+              Provider::APPVEYOR
             end
 
             def pipeline_url

data/lib/datadog/ci/ext/environment/providers/aws_code_pipeline.rb

@@ -17,7 +17,7 @@ module Datadog
             end
 
             def provider_name
-              "awscodepipeline"
+              Provider::AWS
             end
 
             def pipeline_id

data/lib/datadog/ci/ext/environment/providers/azure.rb

@@ -17,7 +17,7 @@ module Datadog
             end
 
             def provider_name
-              "azurepipelines"
+              Provider::AZURE
             end
 
             def pipeline_url

data/lib/datadog/ci/ext/environment/providers/bitbucket.rb

@@ -16,7 +16,7 @@ module Datadog
 
             # overridden methods
             def provider_name
-              "bitbucket"
+              Provider::BITBUCKET
             end
 
             def pipeline_id

data/lib/datadog/ci/ext/environment/providers/bitrise.rb

@@ -15,7 +15,7 @@ module Datadog
             end
 
             def provider_name
-              "bitrise"
+              Provider::BITRISE
             end
 
             def pipeline_id

data/lib/datadog/ci/ext/environment/providers/buddy.rb

@@ -15,7 +15,7 @@ module Datadog
             end
 
             def provider_name
-              "buddy"
+              Provider::BUDDYCI
             end
 
             def pipeline_id

data/lib/datadog/ci/ext/environment/providers/buildkite.rb

@@ -17,7 +17,7 @@ module Datadog
             end
 
             def provider_name
-              "buildkite"
+              Provider::BUILDKITE
             end
 
             def job_url