datadog-ci 1.33.0 → 1.35.0

data/lib/datadog/ci/source_code/method_inspect.rb

@@ -16,11 +16,8 @@ module Datadog
           return nil if target.nil?
           return nil unless LAST_LINE_AVAILABLE
 
-          # Ruby has outdated RBS for RubyVM::InstructionSequence where method `of` is not defined
-          # steep:ignore:start
           iseq = RubyVM::InstructionSequence.of(target)
           return nil unless iseq.is_a?(RubyVM::InstructionSequence)
-          # steep:ignore:end
 
           # this function is implemented in ext/datadog_ci_native/datadog_method_inspect.c
           _native_last_line_from_iseq(iseq)

data/lib/datadog/ci/test_discovery/component.rb

@@ -4,6 +4,7 @@ require "fileutils"
 require "json"
 require_relative "../ext/test"
 require_relative "../ext/test_discovery"
+require_relative "../utils/test_name"
 
 module Datadog
   module CI
@@ -76,6 +77,9 @@ module Datadog
         end
 
         def record_test(name:, suite:, module_name:, parameters:, source_file:)
+          name = name.nil? ? nil : Utils::TestName.normalize(name)
+          suite = suite.nil? ? nil : Utils::TestName.normalize(suite)
+
           test_info = {
             "name" => name,
             "suite" => suite,

data/lib/datadog/ci/test_impact_analysis/component.rb

@@ -30,8 +30,8 @@ module Datadog
 
         FILE_STORAGE_KEY = "test_impact_analysis_component_state"
 
-        attr_reader :correlation_id, :skippable_tests, :skippable_tests_fetch_error,
-          :enabled, :test_skipping_enabled, :code_coverage_enabled
+        attr_reader :correlation_id, :skippable_tests, :skippable_suites, :skippable_tests_fetch_error,
+          :enabled, :test_skipping_enabled, :code_coverage_enabled, :test_skipping_mode
 
         def initialize(
           dd_env:,
@@ -39,6 +39,7 @@ module Datadog
           api: nil,
           coverage_writer: nil,
           enabled: false,
+          test_skipping_mode: Ext::Test::TIATestSkippingMode::TEST,
           bundle_location: nil,
           use_single_threaded_coverage: false,
           use_allocation_tracing: true,
@@ -48,6 +49,7 @@ module Datadog
           @api = api
           @dd_env = dd_env
           @config_tags = config_tags || {}
+          @test_skipping_mode = test_skipping_mode
 
           @bundle_location = if bundle_location && !File.absolute_path?(bundle_location)
             File.join(Git::LocalRepository.root, bundle_location)
@@ -65,6 +67,7 @@ module Datadog
 
           @correlation_id = nil
           @skippable_tests = Set.new
+          @skippable_suites = Set.new
 
           @mutex = Mutex.new
 
@@ -92,8 +95,7 @@ module Datadog
 
           test_session.set_tag(Ext::Test::TAG_ITR_TEST_SKIPPING_ENABLED, @test_skipping_enabled)
           test_session.set_tag(Ext::Test::TAG_CODE_COVERAGE_ENABLED, @code_coverage_enabled)
-          # we skip tests, not suites
-          test_session.set_tag(Ext::Test::TAG_ITR_TEST_SKIPPING_TYPE, Ext::Test::ITR_TEST_SKIPPING_MODE)
+          test_session.set_tag(Ext::Test::TAG_ITR_TEST_SKIPPING_TYPE, @test_skipping_mode)
 
           if @code_coverage_enabled
             load_datadog_cov!
@@ -102,10 +104,10 @@ module Datadog
           end
 
           # Load external cache or component state first, and if successful, skip fetching skippable tests
-          if skipping_tests?
+          if skipping_tests? || skipping_suites?
             return if load_component_state
 
-            fetch_skippable_tests(test_session)
+            fetch_skippables(test_session)
             store_component_state if test_session.distributed
           end
 
@@ -117,7 +119,19 @@ module Datadog
         end
 
         def skipping_tests?
-          @test_skipping_enabled
+          @test_skipping_enabled && test_skipping_mode?
+        end
+
+        def skipping_suites?
+          @test_skipping_enabled && suite_skipping_mode?
+        end
+
+        def test_skipping_mode?
+          @test_skipping_mode == Ext::Test::TIATestSkippingMode::TEST
+        end
+
+        def suite_skipping_mode?
+          @test_skipping_mode == Ext::Test::TIATestSkippingMode::SUITE
         end
 
         def code_coverage?
@@ -174,6 +188,7 @@ module Datadog
         # @return [void]
         def on_test_started(test)
           return if !enabled? || !code_coverage?
+          return if suite_skipping_mode?
 
           # Stop any in-progress context coverage and store it
           stop_context_coverage_and_store
@@ -200,6 +215,7 @@ module Datadog
         # @return [Datadog::CI::TestImpactAnalysis::Coverage::Event, nil] The coverage event or nil
         def on_test_finished(test, context)
           return unless enabled?
+          return if suite_skipping_mode?
 
           # Handle ITR statistics
           if test.skipped_by_test_impact_analysis?
@@ -222,33 +238,13 @@ module Datadog
           context_ids = test.context_ids || []
           merge_context_coverages_into_test(coverage, context_ids)
 
-          if coverage.empty?
-            Telemetry.code_coverage_is_empty
-            return
-          end
-
-          # cucumber's gherkin files are not covered by the code coverage collector - we add them here explicitly
-          test_source_file = test.source_file
-          ensure_test_source_covered(test_source_file, coverage) unless test_source_file.nil?
-
-          # if we have static dependencies tracking enabled then we can make the coverage
-          # more precise by fetching which files we depend on based on constants usage
-          enrich_coverage_with_static_dependencies(coverage)
-
-          Telemetry.code_coverage_files(coverage.size)
-
-          coverage_event = Coverage::Event.new(
+          write_coverage_event(
             test_id: test.id.to_s,
             test_suite_id: test.test_suite_id.to_s,
             test_session_id: test.test_session_id.to_s,
+            source_file: test.source_file,
             coverage: coverage
           )
-
-          Datadog.logger.debug { "Writing coverage event \n #{coverage_event.pretty_inspect}" }
-
-          write(coverage_event)
-
-          coverage_event
         end
 
         # Clears stored context coverage for a specific context.
@@ -271,7 +267,7 @@ module Datadog
         #
         # @return [Boolean]
         def context_coverage_enabled?
-          enabled? && code_coverage? && !@use_single_threaded_coverage
+          enabled? && code_coverage? && !suite_skipping_mode? && !@use_single_threaded_coverage
         end
 
         def skippable?(datadog_test_id)
@@ -280,6 +276,12 @@ module Datadog
           @mutex.synchronize { @skippable_tests.include?(datadog_test_id) }
         end
 
+        def skippable_suite?(test_suite_name)
+          return false if !enabled? || !skipping_suites?
+
+          @mutex.synchronize { @skippable_suites.include?(test_suite_name) }
+        end
+
         def mark_if_skippable(test)
           return if !enabled? || !skipping_tests?
 
@@ -292,6 +294,66 @@ module Datadog
           end
         end
 
+        def mark_if_suite_skippable(test_suite)
+          return if !enabled? || !skipping_suites?
+
+          unskippable = test_suite.itr_unskippable?
+          Telemetry.itr_unskippable if unskippable
+
+          unless skippable_suite?(test_suite.name)
+            Datadog.logger.debug { "Test suite is not skippable: #{test_suite.name}" }
+            return
+          end
+
+          if unskippable
+            Telemetry.itr_forced_run
+            test_suite.set_tag(Ext::Test::TAG_ITR_FORCED_RUN, "true")
+
+            Datadog.logger.debug { "Forced run of skippable test suite: #{test_suite.name}" }
+            return
+          end
+
+          test_suite.set_tag(Ext::Test::TAG_ITR_SKIPPED_BY_ITR, "true")
+          test_suite.skipped!(reason: Ext::Test::SkipReason::TEST_IMPACT_ANALYSIS)
+
+          Datadog.logger.debug { "Marked test suite as skippable: #{test_suite.name}" }
+        end
+
+        def on_test_suite_started(test_suite)
+          return unless enabled? && suite_skipping_mode?
+
+          mark_if_suite_skippable(test_suite)
+          return if test_suite.should_skip?
+          return unless code_coverage?
+
+          Telemetry.code_coverage_started(test_suite)
+          coverage_collector&.start
+        end
+
+        def on_test_suite_finished(test_suite, context)
+          return unless enabled? && suite_skipping_mode?
+
+          if test_suite.skipped_by_test_impact_analysis?
+            Telemetry.itr_skipped
+            context.incr_tests_skipped_by_tia_count
+            return
+          end
+
+          return unless code_coverage?
+
+          Telemetry.code_coverage_finished(test_suite)
+
+          coverage = coverage_collector&.stop
+
+          write_coverage_event(
+            test_id: nil,
+            test_suite_id: test_suite.id.to_s,
+            test_session_id: test_suite.get_tag(Ext::Test::TAG_TEST_SESSION_ID).to_s,
+            source_file: test_suite.source_file,
+            coverage: coverage
+          )
+        end
+
         def write_test_session_tags(test_session, skipped_tests_count)
           return if !enabled?
 
@@ -302,8 +364,8 @@ module Datadog
           test_session.set_tag(Ext::Test::TAG_ITR_TEST_SKIPPING_COUNT, skipped_tests_count)
         end
 
-        def skippable_tests_count
-          skippable_tests.count
+        def skippables_count
+          current_skippables.count
         end
 
         def shutdown!
@@ -314,15 +376,17 @@ module Datadog
         def serialize_state
           {
             correlation_id: @correlation_id,
-            skippable_tests: @skippable_tests
+            skippable_tests: @skippable_tests,
+            skippable_suites: @skippable_suites
           }
         end
 
         def restore_state(state)
-          @mutex.synchronize do
-            @correlation_id = state[:correlation_id]
-            @skippable_tests = state[:skippable_tests]
-          end
+          set_skippables(
+            correlation_id: state[:correlation_id],
+            tests: state[:skippable_tests] || Set.new,
+            suites: state[:skippable_suites] || Set.new
+          )
         end
 
         def storage_key
@@ -330,24 +394,20 @@ module Datadog
         end
 
         def restore_state_from_datadog_test_runner
-          Datadog.logger.debug { "Restoring skippable tests from Test Optimization cache" }
+          Datadog.logger.debug { "Restoring skippables from Test Optimization cache" }
 
-          skippable_tests_data = load_cached_skippable_tests
-          if skippable_tests_data.nil?
-            Datadog.logger.debug { "Restoring skippable tests failed, will request again" }
+          skippables_data = load_cached_skippable_tests
+          if skippables_data.nil?
+            Datadog.logger.debug { "Restoring skippables failed, will request again" }
             return false
           end
 
-          Datadog.logger.debug { "Restored skippable tests from Test Optimization: #{skippable_tests_data}" }
+          Datadog.logger.debug { "Restored skippables from Test Optimization: #{skippables_data}" }
 
-          skippable_response = Skippable::Response.from_json(skippable_tests_data)
+          skippable_response = Skippable::Response.from_json(skippables_data)
+          apply_skippable_response(skippable_response)
 
-          @mutex.synchronize do
-            @correlation_id = skippable_response.correlation_id
-            @skippable_tests = skippable_response.tests
-          end
-
-          Datadog.logger.debug { "Found [#{@skippable_tests.size}] skippable tests from context" }
+          Datadog.logger.debug { "Found [#{skippables_count}] skippable #{@test_skipping_mode}s from context" }
           Datadog.logger.debug { "ITR correlation ID from context: #{@correlation_id}" }
 
           true
@@ -406,27 +466,84 @@ module Datadog
           coverage[absolute_test_source_file_path] = true
         end
 
-        def fetch_skippable_tests(test_session)
-          return unless skipping_tests?
+        def write_coverage_event(test_id:, test_suite_id:, test_session_id:, source_file:, coverage:)
+          coverage ||= {}
+          if coverage.empty?
+            Telemetry.code_coverage_is_empty
+            return
+          end
+
+          ensure_test_source_covered(source_file, coverage) unless source_file.nil?
+
+          enrich_coverage_with_static_dependencies(coverage)
+
+          Telemetry.code_coverage_files(coverage.size)
+
+          coverage_event = Coverage::Event.new(
+            test_id: test_id,
+            test_suite_id: test_suite_id,
+            test_session_id: test_session_id,
+            coverage: coverage
+          )
+
+          Datadog.logger.debug { "Writing coverage event \n #{coverage_event.pretty_inspect}" }
+
+          write(coverage_event)
+
+          coverage_event
+        end
+
+        def fetch_skippables(test_session)
+          return unless skipping_tests? || skipping_suites?
 
           # we can only request skippable tests if git metadata is already uploaded
           git_tree_upload_worker.wait_until_done
 
           skippable_response =
-            Skippable.new(api: @api, dd_env: @dd_env, config_tags: @config_tags)
-              .fetch_skippable_tests(test_session)
+            Skippable.new(
+              api: @api,
+              dd_env: @dd_env,
+              config_tags: @config_tags,
+              test_skipping_mode: @test_skipping_mode
+            )
+              .fetch_skippables(test_session)
           @skippable_tests_fetch_error = skippable_response.error_message unless skippable_response.ok?
 
+          apply_skippable_response(skippable_response)
+
+          Datadog.logger.debug { "Fetched skippable #{@test_skipping_mode}s: \n #{current_skippables}" }
+          Datadog.logger.debug { "Found #{skippables_count} skippable #{@test_skipping_mode}s." }
+          Datadog.logger.debug { "ITR correlation ID: #{@correlation_id}" }
+
+          Utils::Telemetry.inc(skippable_response_metric, skippables_count)
+        end
+
+        def apply_skippable_response(skippable_response)
+          set_skippables(
+            correlation_id: skippable_response.correlation_id,
+            tests: skippable_response.tests,
+            suites: skippable_response.suites
+          )
+        end
+
+        def set_skippables(correlation_id:, tests:, suites:)
           @mutex.synchronize do
-            @correlation_id = skippable_response.correlation_id
-            @skippable_tests = skippable_response.tests
+            @correlation_id = correlation_id
+            @skippable_tests = tests
+            @skippable_suites = suites
           end
+        end
 
-          Datadog.logger.debug { "Fetched skippable tests: \n #{@skippable_tests}" }
-          Datadog.logger.debug { "Found #{@skippable_tests.count} skippable tests." }
-          Datadog.logger.debug { "ITR correlation ID: #{@correlation_id}" }
+        def current_skippables
+          suite_skipping_mode? ? @skippable_suites : @skippable_tests
+        end
 
-          Utils::Telemetry.inc(Ext::Telemetry::METRIC_ITR_SKIPPABLE_TESTS_RESPONSE_TESTS, @skippable_tests.count)
+        def skippable_response_metric
+          if skipping_suites?
+            Ext::Telemetry::METRIC_ITR_SKIPPABLE_TESTS_RESPONSE_SUITES
+          else
+            Ext::Telemetry::METRIC_ITR_SKIPPABLE_TESTS_RESPONSE_TESTS
+          end
         end
 
         def code_coverage_mode

data/lib/datadog/ci/test_impact_analysis/coverage/event.rb

@@ -21,7 +21,7 @@ module Datadog
           def valid?
             valid = true
 
-            %i[test_id test_suite_id test_session_id coverage].each do |key|
+            %i[test_suite_id test_session_id coverage].each do |key|
               next unless send(key).nil?
 
               Datadog.logger.warn("citestcov event is invalid: [#{key}] is nil. Event: #{self}")
@@ -34,7 +34,7 @@ module Datadog
           def to_msgpack(packer = nil)
             packer ||= MessagePack::Packer.new
 
-            packer.write_map_header(4)
+            packer.write_map_header(test_id.nil? ? 3 : 4)
 
             packer.write("test_session_id")
             packer.write(test_session_id.to_i)
@@ -42,8 +42,10 @@ module Datadog
             packer.write("test_suite_id")
             packer.write(test_suite_id.to_i)
 
-            packer.write("span_id")
-            packer.write(test_id.to_i)
+            unless test_id.nil?
+              packer.write("span_id")
+              packer.write(test_id.to_i)
+            end
 
             files = coverage.keys
             packer.write("files")

data/lib/datadog/ci/test_impact_analysis/null_component.rb

@@ -2,13 +2,16 @@
 
 require "set"
 
+require_relative "../ext/test"
+
 module Datadog
   module CI
     module TestImpactAnalysis
       # No-op implementation used when test impact analysis is disabled.
       class NullComponent
         attr_reader :enabled, :skippable_tests_fetch_error, :test_skipping_enabled,
-          :code_coverage_enabled, :skippable_tests, :correlation_id
+          :code_coverage_enabled, :skippable_tests, :skippable_suites, :correlation_id,
+          :test_skipping_mode
 
         def initialize
           @enabled = false
@@ -16,7 +19,9 @@ module Datadog
           @code_coverage_enabled = false
           @skippable_tests_fetch_error = nil
           @skippable_tests = Set.new
+          @skippable_suites = Set.new
           @correlation_id = nil
+          @test_skipping_mode = Ext::Test::TIATestSkippingMode::TEST
         end
 
         def configure(_remote_configuration = nil, _test_session = nil)
@@ -30,6 +35,10 @@ module Datadog
           false
         end
 
+        def skipping_suites?
+          false
+        end
+
         def code_coverage?
           false
         end
@@ -51,6 +60,13 @@ module Datadog
           nil
         end
 
+        def on_test_suite_started(_test_suite)
+        end
+
+        def on_test_suite_finished(_test_suite, _context)
+          nil
+        end
+
         def clear_context_coverage(_context_id)
         end
 
@@ -65,10 +81,17 @@ module Datadog
           false
         end
 
+        def skippable_suite?(_test_suite_name)
+          false
+        end
+
         def write_test_session_tags(_test_session, _skipped_tests_count)
         end
 
-        def skippable_tests_count
+        def mark_if_suite_skippable(_test_suite)
+        end
+
+        def skippables_count
           0
         end
 

data/lib/datadog/ci/test_impact_analysis/skippable.rb

@@ -36,7 +36,7 @@ module Datadog
 
             payload.fetch("data", [])
               .each do |test_data|
-                next unless test_data["type"] == Ext::Test::ITR_TEST_SKIPPING_MODE
+                next unless test_data["type"] == Ext::Test::TIATestSkippingMode::TEST
 
                 attrs = test_data["attributes"] || {}
                 res << Utils::TestRun.datadog_test_id(attrs["name"], attrs["suite"], attrs["parameters"])
@@ -45,6 +45,20 @@ module Datadog
             res
           end
 
+          def suites
+            res = Set.new
+
+            payload.fetch("data", [])
+              .each do |test_data|
+                next unless test_data["type"] == Ext::Test::TIATestSkippingMode::SUITE
+
+                suite = (test_data["attributes"] || {})["suite"]
+                res << suite unless suite.nil?
+              end
+
+            res
+          end
+
           def error_message
             return nil if ok?
 
@@ -74,18 +88,19 @@ module Datadog
           end
         end
 
-        def initialize(dd_env:, api: nil, config_tags: {})
+        def initialize(dd_env:, api: nil, config_tags: {}, test_skipping_mode: Ext::Test::TIATestSkippingMode::TEST)
           @api = api
           @dd_env = dd_env
           @config_tags = config_tags
+          @test_skipping_mode = test_skipping_mode
         end
 
-        def fetch_skippable_tests(test_session)
+        def fetch_skippables(test_session)
           api = @api
           return Response.from_http_response(nil) unless api
 
           request_payload = payload(test_session)
-          Datadog.logger.debug("Fetching skippable tests with request: #{request_payload}")
+          Datadog.logger.debug("Fetching skippable #{@test_skipping_mode}s with request: #{request_payload}")
 
           http_response = api.api_request(
             path: Ext::Transport::DD_API_SKIPPABLE_TESTS_PATH,
@@ -127,7 +142,7 @@ module Datadog
             "data" => {
               "type" => Ext::Transport::DD_API_SKIPPABLE_TESTS_TYPE,
               "attributes" => {
-                "test_level" => Ext::Test::ITR_TEST_SKIPPING_MODE,
+                "test_level" => @test_skipping_mode,
                 "service" => test_session.service,
                 "env" => @dd_env,
                 "repository_url" => test_session.git_repository_url,

data/lib/datadog/ci/test_impact_analysis/skippable_percentage/estimator.rb

@@ -48,13 +48,13 @@ module Datadog
 
             # starting and finishing a test session is required to get the skippable tests response
             Datadog::CI.start_test_session(total_tests_count: estimated_tests_count)&.finish
-            skippable_tests_count = test_impact_analysis.skippable_tests_count
+            skippables_count = test_impact_analysis.skippables_count
 
             log("Estimated tests count: #{estimated_tests_count}")
-            log("Skippable tests count: #{skippable_tests_count}")
+            log("Skippables count: #{skippables_count}")
             validate_test_impact_analysis_state!
 
-            [(skippable_tests_count.to_f / estimated_tests_count).floor(2), 0.99].min || 0.0
+            [(skippables_count.to_f / estimated_tests_count).floor(2), 0.99].min || 0.0
           end
         end
       end

data/lib/datadog/ci/test_optimization_cache/readers/legacy.rb

@@ -49,7 +49,7 @@ module Datadog
               tests_hash.each_value.flat_map do |test_configs|
                 test_configs.map do |test_config|
                   {
-                    "type" => Ext::Test::ITR_TEST_SKIPPING_MODE,
+                    "type" => Ext::Test::DEFAULT_TIA_TEST_SKIPPING_MODE,
                     "attributes" => {
                       "suite" => test_config["suite"],
                       "name" => test_config["name"],

data/lib/datadog/ci/test_suite.rb

@@ -111,6 +111,26 @@ module Datadog
         end
       end
 
+      # @internal
+      def datadog_skip_reason
+        get_tag(Ext::Test::TAG_SKIP_REASON)
+      end
+
+      # @internal
+      def should_skip?
+        skipped_by_test_impact_analysis?
+      end
+
+      # @internal
+      def skipped_by_test_impact_analysis?
+        get_tag(Ext::Test::TAG_ITR_SKIPPED_BY_ITR) == "true"
+      end
+
+      # @internal
+      def itr_unskippable?
+        get_tag(Ext::Test::TAG_ITR_UNSKIPPABLE) == "true"
+      end
+
       private
 
       def set_status_from_stats!

data/lib/datadog/ci/test_tracing/component.rb

@@ -16,6 +16,7 @@ require_relative "../ext/test"
 require_relative "../git/local_repository"
 require_relative "../utils/file_storage"
 require_relative "../utils/stateful"
+require_relative "../utils/test_name"
 
 require_relative "../worker"
 
@@ -109,6 +110,7 @@ module Datadog
         def start_test_suite(test_suite_name, service: nil, tags: {})
           return skip_tracing unless test_suite_level_visibility_enabled
 
+          test_suite_name = Utils::TestName.normalize(test_suite_name)
           context = @local_test_suites_mode ? @context : maybe_remote_context
 
           test_suite = context.start_test_suite(test_suite_name, service: service, tags: tags)
@@ -117,6 +119,9 @@ module Datadog
         end
 
         def trace_test(test_name, test_suite_name, service: nil, tags: {}, &block)
+          test_name = Utils::TestName.normalize(test_name)
+          test_suite_name = Utils::TestName.normalize(test_suite_name)
+
           test_suite = active_test_suite(test_suite_name)
           tags[Ext::Test::TAG_SUITE] ||= test_suite_name
 
@@ -165,6 +170,8 @@ module Datadog
           # we return the single active test suite because most of the time there is only one test suite running
           return single_active_test_suite if test_suite_name.nil?
 
+          test_suite_name = Utils::TestName.normalize(test_suite_name)
+
           # when fetching test_suite to use as test's context, try local context instance first
           local_test_suite = @context.active_test_suite(test_suite_name)
           return local_test_suite if local_test_suite
@@ -194,6 +201,7 @@ module Datadog
         end
 
         def deactivate_test_suite(test_suite_name)
+          test_suite_name = Utils::TestName.normalize(test_suite_name)
           test_suite = active_test_suite(test_suite_name)
           on_test_suite_finished(test_suite) if test_suite
 
@@ -272,6 +280,7 @@ module Datadog
 
         def on_test_suite_started(test_suite)
           set_codeowners(test_suite)
+          test_impact_analysis.on_test_suite_started(test_suite)
         end
 
         def on_test_started(test)
@@ -318,6 +327,7 @@ module Datadog
         end
 
         def on_test_suite_finished(test_suite)
+          test_impact_analysis.on_test_suite_finished(test_suite, maybe_remote_context)
           Telemetry.event_finished(test_suite)
         end