datadog-ci 1.1.0 → 1.3.0

data/lib/datadog/ci/{itr/runner.rb → test_optimisation/component.rb}

@@ -5,22 +5,25 @@ require "pp"
 require "datadog/core/utils/forking"
 
 require_relative "../ext/test"
+require_relative "../ext/telemetry"
 require_relative "../ext/transport"
 
 require_relative "../git/local_repository"
 
 require_relative "../utils/parsing"
+require_relative "../utils/telemetry"
 
 require_relative "coverage/event"
 require_relative "skippable"
+require_relative "telemetry"
 
 module Datadog
   module CI
-    module ITR
+    module TestOptimisation
       # Intelligent test runner implementation
       # Integrates with backend to provide test impact analysis data and
       # skip tests that are not impacted by the changes
-      class Runner
+      class Component
         include Core::Utils::Forking
 
         attr_reader :correlation_id, :skippable_tests, :skipped_tests_count
@@ -32,7 +35,8 @@ module Datadog
           coverage_writer: nil,
           enabled: false,
           bundle_location: nil,
-          use_single_threaded_coverage: false
+          use_single_threaded_coverage: false,
+          use_allocation_tracing: true
         )
           @enabled = enabled
           @api = api
@@ -45,6 +49,7 @@ module Datadog
             bundle_location
           end
           @use_single_threaded_coverage = use_single_threaded_coverage
+          @use_allocation_tracing = use_allocation_tracing
 
           @test_skipping_enabled = false
           @code_coverage_enabled = false
@@ -57,11 +62,11 @@ module Datadog
           @skipped_tests_count = 0
           @mutex = Mutex.new
 
-          Datadog.logger.debug("ITR Runner initialized with enabled: #{@enabled}")
+          Datadog.logger.debug("TestOptimisation initialized with enabled: #{@enabled}")
         end
 
         def configure(remote_configuration, test_session:, git_tree_upload_worker:)
-          Datadog.logger.debug("Configuring ITR Runner with remote configuration: #{remote_configuration}")
+          Datadog.logger.debug("Configuring TestOptimisation with remote configuration: #{remote_configuration}")
 
           @enabled = Utils::Parsing.convert_to_bool(
             remote_configuration.fetch(Ext::Transport::DD_API_SETTINGS_RESPONSE_ITR_ENABLED_KEY, false)
@@ -83,7 +88,7 @@ module Datadog
 
           load_datadog_cov! if @code_coverage_enabled
 
-          Datadog.logger.debug("Configured ITR Runner with enabled: #{@enabled}, skipping_tests: #{@test_skipping_enabled}, code_coverage: #{@code_coverage_enabled}")
+          Datadog.logger.debug("Configured TestOptimisation with enabled: #{@enabled}, skipping_tests: #{@test_skipping_enabled}, code_coverage: #{@code_coverage_enabled}")
 
           fetch_skippable_tests(test_session: test_session, git_tree_upload_worker: git_tree_upload_worker)
         end
@@ -102,15 +107,20 @@ module Datadog
 
         def start_coverage(test)
           return if !enabled? || !code_coverage?
-
+          Telemetry.code_coverage_started(test)
           coverage_collector&.start
         end
 
         def stop_coverage(test)
           return if !enabled? || !code_coverage?
 
+          Telemetry.code_coverage_finished(test)
+
           coverage = coverage_collector&.stop
-          return if coverage.nil? || coverage.empty?
+          if coverage.nil? || coverage.empty?
+            Telemetry.code_coverage_is_empty
+            return
+          end
 
           return if test.skipped?
 
@@ -119,6 +129,8 @@ module Datadog
           # cucumber's gherkin files are not covered by the code coverage collector
           ensure_test_source_covered(test_source_file, coverage) unless test_source_file.nil?
 
+          Telemetry.code_coverage_files(coverage.size)
+
           event = Coverage::Event.new(
             test_id: test.id.to_s,
             test_suite_id: test.test_suite_id.to_s,
@@ -139,7 +151,7 @@ module Datadog
           skippable_test_id = Utils::TestRun.skippable_test_id(test.name, test.test_suite_name, test.parameters)
           if @skippable_tests.include?(skippable_test_id)
             if forked?
-              Datadog.logger.warn { "ITR is not supported for forking test runners yet" }
+              Datadog.logger.warn { "Intelligent test runner is not supported for forking test runners yet" }
               return
             end
 
@@ -155,11 +167,13 @@ module Datadog
           return if !test.skipped? || !test.skipped_by_itr?
 
           if forked?
-            Datadog.logger.warn { "ITR is not supported for forking test runners yet" }
+            Datadog.logger.warn { "Intelligent test runner is not supported for forking test runners yet" }
             return
           end
 
           @mutex.synchronize do
+            Telemetry.itr_skipped
+
             @skipped_tests_count += 1
           end
         end
@@ -167,7 +181,7 @@ module Datadog
         def write_test_session_tags(test_session)
           return if !enabled?
 
-          Datadog.logger.debug { "Finished ITR session with test skipping enabled: #{@test_skipping_enabled}" }
+          Datadog.logger.debug { "Finished optimised session with test skipping enabled: #{@test_skipping_enabled}" }
           Datadog.logger.debug { "#{@skipped_tests_count} tests were skipped" }
 
           test_session.set_tag(Ext::Test::TAG_ITR_TESTS_SKIPPED, @skipped_tests_count.positive?.to_s)
@@ -189,7 +203,8 @@ module Datadog
           Thread.current[:dd_coverage_collector] ||= Coverage::DDCov.new(
             root: Git::LocalRepository.root,
             ignored_path: @bundle_location,
-            threading_mode: code_coverage_mode
+            threading_mode: code_coverage_mode,
+            use_allocation_tracing: @use_allocation_tracing
           )
         end
 
@@ -226,6 +241,8 @@ module Datadog
           Datadog.logger.debug { "Fetched skippable tests: \n #{@skippable_tests}" }
           Datadog.logger.debug { "Found #{@skippable_tests.count} skippable tests." }
           Datadog.logger.debug { "ITR correlation ID: #{@correlation_id}" }
+
+          Utils::Telemetry.inc(Ext::Telemetry::METRIC_ITR_SKIPPABLE_TESTS_RESPONSE_TESTS, @skippable_tests.count)
         end
 
         def code_coverage_mode

data/lib/datadog/ci/{itr → test_optimisation}/coverage/ddcov.rb

@@ -2,7 +2,7 @@
 
 module Datadog
   module CI
-    module ITR
+    module TestOptimisation
       module Coverage
         # Placeholder for code coverage collection
         # Implementation in ext/datadog_cov

data/lib/datadog/ci/{itr → test_optimisation}/coverage/event.rb

@@ -6,7 +6,7 @@ require_relative "../../git/local_repository"
 
 module Datadog
   module CI
-    module ITR
+    module TestOptimisation
       module Coverage
         class Event
           attr_reader :test_id, :test_suite_id, :test_session_id, :coverage

data/lib/datadog/ci/{itr → test_optimisation}/coverage/transport.rb

@@ -1,15 +1,21 @@
 # frozen_string_literal: true
 
 require_relative "event"
+require_relative "../../ext/telemetry"
 require_relative "../../transport/event_platform_transport"
+require_relative "../../transport/telemetry"
 
 module Datadog
   module CI
-    module ITR
+    module TestOptimisation
       module Coverage
         class Transport < Datadog::CI::Transport::EventPlatformTransport
           private
 
+          def telemetry_endpoint_tag
+            Ext::Telemetry::Endpoint::CODE_COVERAGE
+          end
+
           def send_payload(encoded_payload)
             api.citestcov_request(
               path: Ext::Transport::TEST_COVERAGE_INTAKE_PATH,
@@ -19,7 +25,10 @@ module Datadog
 
           def encode_events(events)
             events.filter_map do |event|
-              next unless event.valid?
+              unless event.valid?
+                CI::Transport::Telemetry.endpoint_payload_dropped(1, endpoint: telemetry_endpoint_tag)
+                next
+              end
 
               encoded = encoder.encode(event)
               next if event_too_large?(event, encoded)

data/lib/datadog/ci/{itr → test_optimisation}/coverage/writer.rb

@@ -11,7 +11,7 @@ require "datadog/core/environment/ext"
 
 module Datadog
   module CI
-    module ITR
+    module TestOptimisation
       module Coverage
         class Writer
           include Core::Workers::Queue

data/lib/datadog/ci/{itr → test_optimisation}/skippable.rb

@@ -2,13 +2,16 @@
 
 require "json"
 
+require_relative "../ext/telemetry"
 require_relative "../ext/transport"
 require_relative "../ext/test"
+require_relative "../transport/telemetry"
 require_relative "../utils/test_run"
+require_relative "../utils/telemetry"
 
 module Datadog
   module CI
-    module ITR
+    module TestOptimisation
       class Skippable
         class Response
           def initialize(http_response)
@@ -75,6 +78,27 @@ module Datadog
             payload: request_payload
           )
 
+          Transport::Telemetry.api_requests(
+            Ext::Telemetry::METRIC_ITR_SKIPPABLE_TESTS_REQUEST,
+            1,
+            compressed: http_response.request_compressed
+          )
+          Utils::Telemetry.distribution(Ext::Telemetry::METRIC_ITR_SKIPPABLE_TESTS_REQUEST_MS, http_response.duration_ms)
+          Utils::Telemetry.distribution(
+            Ext::Telemetry::METRIC_ITR_SKIPPABLE_TESTS_RESPONSE_BYTES,
+            http_response.response_size.to_f,
+            {Ext::Telemetry::TAG_RESPONSE_COMPRESSED => http_response.gzipped_content?.to_s}
+          )
+
+          unless http_response.ok?
+            Transport::Telemetry.api_requests_errors(
+              Ext::Telemetry::METRIC_ITR_SKIPPABLE_TESTS_REQUEST_ERRORS,
+              1,
+              error_type: http_response.telemetry_error_type,
+              status_code: http_response.code
+            )
+          end
+
           Response.new(http_response)
         end
 

data/lib/datadog/ci/test_optimisation/telemetry.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require_relative "../ext/telemetry"
+require_relative "../ext/test"
+require_relative "../utils/telemetry"
+require_relative "../test_visibility/telemetry"
+
+module Datadog
+  module CI
+    module TestOptimisation
+      # Telemetry for test optimisation component
+      module Telemetry
+        def self.code_coverage_started(test)
+          Utils::Telemetry.inc(Ext::Telemetry::METRIC_CODE_COVERAGE_STARTED, 1, tags_for_test(test))
+        end
+
+        def self.code_coverage_finished(test)
+          Utils::Telemetry.inc(Ext::Telemetry::METRIC_CODE_COVERAGE_FINISHED, 1, tags_for_test(test))
+        end
+
+        def self.code_coverage_is_empty
+          Utils::Telemetry.inc(Ext::Telemetry::METRIC_CODE_COVERAGE_IS_EMPTY, 1)
+        end
+
+        def self.code_coverage_files(count)
+          Utils::Telemetry.distribution(Ext::Telemetry::METRIC_CODE_COVERAGE_FILES, count.to_f)
+        end
+
+        def self.itr_skipped
+          Utils::Telemetry.inc(Ext::Telemetry::METRIC_ITR_SKIPPED, 1, tags_for_itr_metrics)
+        end
+
+        def self.itr_forced_run
+          Utils::Telemetry.inc(Ext::Telemetry::METRIC_ITR_FORCED_RUN, 1, tags_for_itr_metrics)
+        end
+
+        def self.itr_unskippable
+          Utils::Telemetry.inc(Ext::Telemetry::METRIC_ITR_UNSKIPPABLE, 1, tags_for_itr_metrics)
+        end
+
+        def self.tags_for_test(test)
+          {
+            Ext::Telemetry::TAG_TEST_FRAMEWORK => test.get_tag(Ext::Test::TAG_FRAMEWORK),
+            Ext::Telemetry::TAG_LIBRARY => Ext::Telemetry::Library::CUSTOM
+          }
+        end
+
+        def self.tags_for_itr_metrics
+          {
+            Ext::Telemetry::TAG_EVENT_TYPE => Ext::Telemetry::EventType::TEST
+          }
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/test_session.rb

@@ -15,7 +15,7 @@ module Datadog
       # Finishes the current test session.
       # @return [void]
       def finish
-        recorder.deactivate_test_session
+        test_visibility.deactivate_test_session
 
         super
       end

data/lib/datadog/ci/test_suite.rb

@@ -26,7 +26,7 @@ module Datadog
           # we try to derive test suite status from execution stats if no status was set explicitly
           set_status_from_stats! if undefined?
 
-          recorder.deactivate_test_suite(name)
+          test_visibility.deactivate_test_suite(name)
 
           super
         end

data/lib/datadog/ci/test_visibility/component.rb

@@ -0,0 +1,282 @@
+# frozen_string_literal: true
+
+require "rbconfig"
+
+require_relative "context"
+require_relative "telemetry"
+
+require_relative "../codeowners/parser"
+require_relative "../contrib/contrib"
+require_relative "../ext/test"
+require_relative "../git/local_repository"
+
+require_relative "../worker"
+
+module Datadog
+  module CI
+    module TestVisibility
+      # Common behavior for CI tests
+      class Component
+        attr_reader :test_suite_level_visibility_enabled
+
+        def initialize(
+          test_optimisation:,
+          remote_settings_api:,
+          git_tree_upload_worker: DummyWorker.new,
+          test_suite_level_visibility_enabled: false,
+          codeowners: Codeowners::Parser.new(Git::LocalRepository.root).parse
+        )
+          @test_suite_level_visibility_enabled = test_suite_level_visibility_enabled
+          @context = Context.new
+          @codeowners = codeowners
+          @test_optimisation = test_optimisation
+          @remote_settings_api = remote_settings_api
+          @git_tree_upload_worker = git_tree_upload_worker
+        end
+
+        def shutdown!
+          @git_tree_upload_worker.stop
+        end
+
+        def start_test_session(service: nil, tags: {})
+          return skip_tracing unless test_suite_level_visibility_enabled
+
+          test_session = @context.start_test_session(service: service, tags: tags)
+          on_test_session_started(test_session)
+          test_session
+        end
+
+        def start_test_module(test_module_name, service: nil, tags: {})
+          return skip_tracing unless test_suite_level_visibility_enabled
+
+          test_module = @context.start_test_module(test_module_name, service: service, tags: tags)
+          on_test_module_started(test_module)
+          test_module
+        end
+
+        def start_test_suite(test_suite_name, service: nil, tags: {})
+          return skip_tracing unless test_suite_level_visibility_enabled
+
+          test_suite = @context.start_test_suite(test_suite_name, service: service, tags: tags)
+          on_test_suite_started(test_suite)
+          test_suite
+        end
+
+        def trace_test(test_name, test_suite_name, service: nil, tags: {}, &block)
+          if block
+            @context.trace_test(test_name, test_suite_name, service: service, tags: tags) do |test|
+              on_test_started(test)
+              res = block.call(test)
+              on_test_finished(test)
+              res
+            end
+          else
+            test = @context.trace_test(test_name, test_suite_name, service: service, tags: tags)
+            on_test_started(test)
+            test
+          end
+        end
+
+        def trace(span_name, type: "span", tags: {}, &block)
+          if block
+            @context.trace(span_name, type: type, tags: tags) do |span|
+              block.call(span)
+            end
+          else
+            @context.trace(span_name, type: type, tags: tags)
+          end
+        end
+
+        def active_span
+          @context.active_span
+        end
+
+        def active_test
+          @context.active_test
+        end
+
+        def active_test_session
+          @context.active_test_session
+        end
+
+        def active_test_module
+          @context.active_test_module
+        end
+
+        def active_test_suite(test_suite_name)
+          @context.active_test_suite(test_suite_name)
+        end
+
+        def deactivate_test
+          test = active_test
+          on_test_finished(test) if test
+
+          @context.deactivate_test
+        end
+
+        def deactivate_test_session
+          test_session = active_test_session
+          on_test_session_finished(test_session) if test_session
+
+          @context.deactivate_test_session
+        end
+
+        def deactivate_test_module
+          test_module = active_test_module
+          on_test_module_finished(test_module) if test_module
+
+          @context.deactivate_test_module
+        end
+
+        def deactivate_test_suite(test_suite_name)
+          test_suite = active_test_suite(test_suite_name)
+          on_test_suite_finished(test_suite) if test_suite
+
+          @context.deactivate_test_suite(test_suite_name)
+        end
+
+        def itr_enabled?
+          @test_optimisation.enabled?
+        end
+
+        private
+
+        # DOMAIN EVENTS
+        def on_test_session_started(test_session)
+          Telemetry.event_created(test_session)
+          Telemetry.test_session_started(test_session)
+
+          # finds and instruments additional test libraries that we support (ex: selenium-webdriver)
+          Contrib.auto_instrument_on_session_start!
+
+          @git_tree_upload_worker.perform(test_session.git_repository_url)
+          configure_library(test_session)
+        end
+
+        def on_test_module_started(test_module)
+          Telemetry.event_created(test_module)
+        end
+
+        def on_test_suite_started(test_suite)
+          Telemetry.event_created(test_suite)
+        end
+
+        def on_test_started(test)
+          # sometimes test suite is not being assigned correctly
+          # fix it by fetching the one single running test suite from the global context
+          fix_test_suite!(test) if test.test_suite_id.nil?
+
+          validate_test_suite_level_visibility_correctness(test)
+          set_codeowners(test)
+
+          Telemetry.event_created(test)
+
+          @test_optimisation.mark_if_skippable(test)
+          @test_optimisation.start_coverage(test)
+        end
+
+        def on_test_session_finished(test_session)
+          @test_optimisation.write_test_session_tags(test_session)
+
+          Telemetry.event_finished(test_session)
+        end
+
+        def on_test_module_finished(test_module)
+          Telemetry.event_finished(test_module)
+        end
+
+        def on_test_suite_finished(test_suite)
+          Telemetry.event_finished(test_suite)
+        end
+
+        def on_test_finished(test)
+          @test_optimisation.stop_coverage(test)
+          @test_optimisation.count_skipped_test(test)
+
+          Telemetry.event_finished(test)
+        end
+
+        # TODO: move this to CI::Configuration::Remote
+        def configure_library(test_session)
+          # this will change when EFD is implemented
+          return unless itr_enabled?
+
+          remote_configuration = @remote_settings_api.fetch_library_settings(test_session)
+          # sometimes we can skip code coverage for default branch if there are no changes in the repository
+          # backend needs git metadata uploaded for this test session to check if we can skip code coverage
+          if remote_configuration.require_git?
+            Datadog.logger.debug { "Library configuration endpoint requires git upload to be finished, waiting..." }
+            @git_tree_upload_worker.wait_until_done
+
+            Datadog.logger.debug { "Requesting library configuration again..." }
+            remote_configuration = @remote_settings_api.fetch_library_settings(test_session)
+
+            if remote_configuration.require_git?
+              Datadog.logger.debug { "git metadata upload did not complete in time when configuring library" }
+            end
+          end
+
+          @test_optimisation.configure(
+            remote_configuration.payload,
+            test_session: test_session,
+            git_tree_upload_worker: @git_tree_upload_worker
+          )
+        end
+
+        # HELPERS
+        def skip_tracing(block = nil)
+          block&.call(nil)
+        end
+
+        def set_codeowners(test)
+          source = test.source_file
+          owners = @codeowners.list_owners(source) if source
+          test.set_tag(Ext::Test::TAG_CODEOWNERS, owners) unless owners.nil?
+        end
+
+        def fix_test_suite!(test)
+          test_suite = @context.single_active_test_suite
+          unless test_suite
+            Datadog.logger.debug do
+              "Trying to fix test suite for test [#{test.name}] but no single test suite is running."
+            end
+            return
+          end
+
+          Datadog.logger.debug do
+            "For test [#{test.name}]: expected test suite [#{test.test_suite_name}] to be running, " \
+            "but it was not found. Fixing it by assigning test suite [#{test_suite.name}] to the test."
+          end
+
+          test.set_tag(Ext::Test::TAG_TEST_SUITE_ID, test_suite.id.to_s)
+          test.set_tag(Ext::Test::TAG_SUITE, test_suite.name)
+        end
+
+        def validate_test_suite_level_visibility_correctness(test)
+          return unless test_suite_level_visibility_enabled
+
+          if test.test_suite_id.nil?
+            Datadog.logger.debug do
+              "Test [#{test.name}] does not have a test suite associated with it. " \
+              "Expected test suite [#{test.test_suite_name}] to be running."
+            end
+          end
+
+          if test.test_module_id.nil?
+            Datadog.logger.debug do
+              "Test [#{test.name}] does not have a test module associated with it. " \
+              "Make sure that there is a test module running within a session."
+            end
+          end
+
+          if test.test_session_id.nil?
+            Datadog.logger.debug do
+              "Test [#{test.name}] does not have a test session associated with it. " \
+              "Make sure that there is a test session running."
+            end
+          end
+        end
+      end
+    end
+  end
+end