datadog-ci 1.8.1 → 1.10.0

data/lib/datadog/ci/contrib/rspec/example.rb

@@ -3,6 +3,7 @@
 require_relative "../../ext/test"
 require_relative "../../git/local_repository"
 require_relative "../../utils/test_run"
+require_relative "../instrumentation"
 require_relative "ext"
 
 module Datadog
@@ -20,42 +21,25 @@ module Datadog
               return super if ::RSpec.configuration.dry_run? && !datadog_configuration[:dry_run_enabled]
               return super unless datadog_configuration[:enabled]
 
-              test_name = full_description.strip
-              if metadata[:description].empty?
-                # for unnamed it blocks this appends something like "example at ./spec/some_spec.rb:10"
-                test_name << " #{description}"
-              end
-
-              test_suite_description = fetch_top_level_example_group[:description]
-              suite_name = "#{test_suite_description} at #{metadata[:example_group][:rerun_file_path]}"
-
-              # remove example group description from test name to avoid duplication
-              test_name = test_name.sub(test_suite_description, "").strip
-
-              if ci_queue?
-                suite_name = "#{suite_name} (ci-queue running example [#{test_name}])"
-                ci_queue_test_span = test_visibility_component.start_test_suite(suite_name)
-              end
+              test_suite_span = test_visibility_component.start_test_suite(datadog_test_suite_name) if ci_queue?
 
               # don't report test to RSpec::Core::Reporter until retries are done
               @skip_reporting = true
 
               test_retries_component.with_retries do
                 test_visibility_component.trace_test(
-                  test_name,
-                  suite_name,
+                  datadog_test_name,
+                  datadog_test_suite_name,
                   tags: {
                     CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK,
-                    CI::Ext::Test::TAG_FRAMEWORK_VERSION => CI::Contrib::RSpec::Integration.version.to_s,
+                    CI::Ext::Test::TAG_FRAMEWORK_VERSION => datadog_integration.version.to_s,
                     CI::Ext::Test::TAG_SOURCE_FILE => Git::LocalRepository.relative_to_root(metadata[:file_path]),
                     CI::Ext::Test::TAG_SOURCE_START => metadata[:line_number].to_s,
-                    CI::Ext::Test::TAG_PARAMETERS => Utils::TestRun.test_parameters(
-                      metadata: {"scoped_id" => metadata[:scoped_id]}
-                    )
+                    CI::Ext::Test::TAG_PARAMETERS => datadog_test_parameters
                   },
                   service: datadog_configuration[:service_name]
                 ) do |test_span|
-                  test_span&.itr_unskippable! if metadata[CI::Ext::Test::ITR_UNSKIPPABLE_OPTION]
+                  test_span&.itr_unskippable! if datadog_unskippable?
 
                   metadata[:skip] = CI::Ext::Test::ITR_TEST_SKIP_REASON if test_span&.skipped_by_itr?
 
@@ -86,8 +70,8 @@ module Datadog
                 end
               end
 
-              # this is a special case for ci-queue, we need to finish the test suite span
-              ci_queue_test_span&.finish
+              # this is a special case for ci-queue, we need to finish the test suite span created for a single test
+              test_suite_span&.finish
 
               # after retries are done, we can finally report the test to RSpec
               @skip_reporting = false
@@ -105,6 +89,18 @@ module Datadog
               super(::RSpec::Core::NullReporter)
             end
 
+            def datadog_test_id
+              @datadog_test_id ||= Utils::TestRun.datadog_test_id(
+                datadog_test_name,
+                datadog_test_suite_name,
+                datadog_test_parameters
+              )
+            end
+
+            def datadog_unskippable?
+              !!metadata[CI::Ext::Test::ITR_UNSKIPPABLE_OPTION]
+            end
+
             private
 
             def fetch_top_level_example_group
@@ -120,10 +116,53 @@ module Datadog
               res
             end
 
+            def datadog_integration
+              CI::Contrib::Instrumentation.fetch_integration(:rspec)
+            end
+
             def datadog_configuration
               Datadog.configuration.ci[:rspec]
             end
 
+            def datadog_test_suite_description
+              @datadog_test_suite_description ||= fetch_top_level_example_group[:description]
+            end
+
+            def datadog_test_name
+              return @datadog_test_name if defined?(@datadog_test_name)
+
+              test_name = full_description.strip
+              if metadata[:description].empty?
+                # for unnamed it blocks this appends something like "example at ./spec/some_spec.rb:10"
+                test_name << " #{description}"
+              end
+
+              # remove example group description from test name to avoid duplication
+              test_name = test_name.sub(datadog_test_suite_description, "").strip
+
+              @datadog_test_name = test_name
+            end
+
+            def datadog_test_suite_name
+              return @datadog_test_suite_name if defined?(@datadog_test_suite_name)
+
+              suite_name = "#{datadog_test_suite_description} at #{metadata[:example_group][:rerun_file_path]}"
+
+              if ci_queue?
+                suite_name = "#{suite_name} (ci-queue running example [#{datadog_test_name}])"
+              end
+
+              @datadog_test_suite_name = suite_name
+            end
+
+            def datadog_test_parameters
+              return @datadog_test_parameters if defined?(@datadog_test_parameters)
+
+              @datadog_test_parameters = Utils::TestRun.test_parameters(
+                metadata: {"scoped_id" => metadata[:scoped_id]}
+              )
+            end
+
             def test_visibility_component
               Datadog.send(:components).test_visibility
             end

data/lib/datadog/ci/contrib/rspec/example_group.rb

@@ -18,10 +18,17 @@ module Datadog
             def run(reporter = ::RSpec::Core::NullReporter)
               return super if ::RSpec.configuration.dry_run? && !datadog_configuration[:dry_run_enabled]
               return super unless datadog_configuration[:enabled]
+
+              # skip the context hooks if all descendant example are going to be skipped
+              # IMPORTANT: must happen before top_level? check because skipping hooks must happen
+              # even if it is a nested context
+              metadata[:skip] = true if all_examples_skipped_by_datadog?
+
+              # return early because we create Datadog::CI::TestSuite only for top-level example groups
               return super unless top_level?
 
               suite_name = "#{description} at #{file_path}"
-              test_suite = test_visibility_component.start_test_suite(
+              test_suite = test_visibility_component&.start_test_suite(
                 suite_name,
                 tags: {
                   CI::Ext::Test::TAG_SOURCE_FILE => Git::LocalRepository.relative_to_root(metadata[:file_path]),
@@ -47,6 +54,12 @@ module Datadog
 
             private
 
+            def all_examples_skipped_by_datadog?
+              descendant_filtered_examples.all? do |example|
+                !example.datadog_unskippable? && test_optimisation_component&.skippable?(example)
+              end
+            end
+
             def datadog_configuration
               Datadog.configuration.ci[:rspec]
             end
@@ -54,6 +67,10 @@ module Datadog
             def test_visibility_component
               Datadog.send(:components).test_visibility
             end
+
+            def test_optimisation_component
+              Datadog.send(:components).test_optimisation
+            end
           end
         end
       end

data/lib/datadog/ci/contrib/rspec/integration.rb

@@ -9,31 +9,28 @@ module Datadog
     module Contrib
       module RSpec
         # Description of RSpec integration
-        class Integration
-          include Datadog::CI::Contrib::Integration
-
+        class Integration < Contrib::Integration
           MINIMUM_VERSION = Gem::Version.new("3.0.0")
 
-          register_as :rspec
+          def dependants
+            %i[knapsack ciqueue]
+          end
 
-          def self.version
+          def version
             Gem.loaded_specs["rspec-core"]&.version
           end
 
-          def self.loaded?
+          def loaded?
             !defined?(::RSpec).nil? && !defined?(::RSpec::Core).nil? &&
-              !defined?(::RSpec::Core::Example).nil?
+              !defined?(::RSpec::Core::Example).nil? &&
+              !defined?(::RSpec::Core::Runner).nil? &&
+              !defined?(::RSpec::Core::ExampleGroup).nil?
           end
 
-          def self.compatible?
+          def compatible?
             super && version >= MINIMUM_VERSION
           end
 
-          # test environments should not auto instrument test libraries
-          def auto_instrument?
-            false
-          end
-
           def new_configuration
             Configuration::Settings.new
           end

data/lib/datadog/ci/contrib/rspec/patcher.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require "datadog/tracing/contrib/patcher"
+require_relative "../patcher"
 
 require_relative "example"
 require_relative "example_group"
@@ -12,46 +12,15 @@ module Datadog
       module RSpec
         # Patcher enables patching of 'rspec' module.
         module Patcher
-          include Datadog::Tracing::Contrib::Patcher
+          include Datadog::CI::Contrib::Patcher
 
           module_function
 
-          def target_version
-            Integration.version
-          end
-
           def patch
-            # ci-queue test runner instrumentation
-            # https://github.com/Shopify/ci-queue
-            if ci_queue?
-              ::RSpec::Queue::Runner.include(Runner)
-            end
-
-            if knapsack_pro?
-              # Knapsack Pro test runner instrumentation
-              # https://github.com/KnapsackPro/knapsack_pro-ruby
-              require_relative "knapsack_pro/patcher"
-              Datadog::CI::Contrib::RSpec::KnapsackPro::Patcher.patch
-            end
-
-            # default rspec test runner instrumentation
             ::RSpec::Core::Runner.include(Runner)
-
             ::RSpec::Core::Example.include(Example)
             ::RSpec::Core::ExampleGroup.include(ExampleGroup)
           end
-
-          def ci_queue?
-            !!defined?(::RSpec::Queue::Runner)
-          end
-
-          def knapsack_pro?
-            knapsack_version = Gem.loaded_specs["knapsack_pro"]&.version
-
-            # additional instrumentation is needed for KnapsackPro version 7 and later
-            !!defined?(::KnapsackPro) &&
-              !knapsack_version.nil? && knapsack_version >= Gem::Version.new("7")
-          end
         end
       end
     end

data/lib/datadog/ci/contrib/rspec/runner.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require_relative "../../ext/test"
+require_relative "../instrumentation"
 require_relative "ext"
 
 module Datadog
@@ -21,7 +22,7 @@ module Datadog
               test_session = test_visibility_component.start_test_session(
                 tags: {
                   CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK,
-                  CI::Ext::Test::TAG_FRAMEWORK_VERSION => CI::Contrib::RSpec::Integration.version.to_s
+                  CI::Ext::Test::TAG_FRAMEWORK_VERSION => datadog_integration.version.to_s
                 },
                 service: datadog_configuration[:service_name],
                 total_tests_count: ::RSpec.world.example_count
@@ -47,6 +48,10 @@ module Datadog
 
             private
 
+            def datadog_integration
+              CI::Contrib::Instrumentation.fetch_integration(:rspec)
+            end
+
             def datadog_configuration
               Datadog.configuration.ci[:rspec]
             end

data/lib/datadog/ci/contrib/selenium/capybara_driver.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require "datadog/tracing/contrib/patcher"
+require_relative "../patcher"
 
 require_relative "ext"
 require_relative "rum"

data/lib/datadog/ci/contrib/selenium/driver.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require "datadog/tracing/contrib/patcher"
+require_relative "../patcher"
 
 require_relative "ext"
 require_relative "rum"

data/lib/datadog/ci/contrib/selenium/integration.rb

@@ -9,28 +9,24 @@ module Datadog
     module Contrib
       module Selenium
         # Description of Selenium integration
-        class Integration
-          include Datadog::CI::Contrib::Integration
-
+        class Integration < Contrib::Integration
           MINIMUM_VERSION = Gem::Version.new("4.0.0")
 
-          register_as :selenium
-
-          def self.version
+          def version
             Gem.loaded_specs["selenium-webdriver"]&.version
           end
 
-          def self.loaded?
+          def loaded?
             !defined?(::Selenium).nil? && !defined?(::Selenium::WebDriver).nil? &&
               !defined?(::Selenium::WebDriver::Driver).nil?
           end
 
-          def self.compatible?
+          def compatible?
             super && version >= MINIMUM_VERSION
           end
 
-          # additional instrumentations for test helpers are auto instrumented on test session start
-          def auto_instrument?
+          # additional instrumentations for test libraries are late instrumented on test session start
+          def late_instrument?
             true
           end
 

data/lib/datadog/ci/contrib/selenium/navigation.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require "datadog/tracing/contrib/patcher"
+require_relative "../patcher"
 
 require_relative "ext"
 require_relative "../../ext/test"
@@ -43,7 +43,7 @@ module Datadog
               active_test.set_tag(CI::Ext::Test::TAG_BROWSER_DRIVER, "selenium")
               active_test.set_tag(
                 CI::Ext::Test::TAG_BROWSER_DRIVER_VERSION,
-                Integration.version
+                datadog_integration.version
               )
               active_test.set_tag(
                 CI::Ext::Test::TAG_BROWSER_NAME,
@@ -63,6 +63,10 @@ module Datadog
 
             private
 
+            def datadog_integration
+              CI::Contrib::Instrumentation.fetch_integration(:selenium)
+            end
+
             def datadog_configuration
               Datadog.configuration.ci[:selenium]
             end

data/lib/datadog/ci/contrib/selenium/patcher.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require "datadog/tracing/contrib/patcher"
+require_relative "../patcher"
 
 require_relative "capybara_driver"
 require_relative "driver"
@@ -12,14 +12,10 @@ module Datadog
       module Selenium
         # Patcher enables patching of 'Selenium::WebDriver' module.
         module Patcher
-          include Datadog::Tracing::Contrib::Patcher
+          include Datadog::CI::Contrib::Patcher
 
           module_function
 
-          def target_version
-            Integration.version
-          end
-
           def patch
             ::Selenium::WebDriver::Driver.include(Driver)
             ::Selenium::WebDriver::Navigation.include(Navigation)

data/lib/datadog/ci/contrib/selenium/rum.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "datadog/tracing/contrib/patcher"
-
 require_relative "ext"
 require_relative "../../ext/test"
 require_relative "../../utils/parsing"

data/lib/datadog/ci/contrib/simplecov/integration.rb

@@ -9,27 +9,23 @@ module Datadog
     module Contrib
       module Simplecov
         # Description of Simplecov integration
-        class Integration
-          include Datadog::CI::Contrib::Integration
-
+        class Integration < Contrib::Integration
           MINIMUM_VERSION = Gem::Version.new("0.18.0")
 
-          register_as :simplecov
-
-          def self.version
+          def version
             Gem.loaded_specs["simplecov"]&.version
           end
 
-          def self.loaded?
+          def loaded?
             !defined?(::SimpleCov).nil?
           end
 
-          def self.compatible?
+          def compatible?
             super && version >= MINIMUM_VERSION
           end
 
-          # additional instrumentations for test helpers are auto instrumented on test session start
-          def auto_instrument?
+          # additional instrumentations for test libraries are late instrumented on test session start
+          def late_instrument?
             true
           end
 

data/lib/datadog/ci/contrib/simplecov/patcher.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require "datadog/tracing/contrib/patcher"
+require_relative "../patcher"
 
 require_relative "result_extractor"
 
@@ -10,14 +10,10 @@ module Datadog
       module Simplecov
         # Patcher enables patching of 'SimpleCov' module.
         module Patcher
-          include Datadog::Tracing::Contrib::Patcher
+          include Datadog::CI::Contrib::Patcher
 
           module_function
 
-          def target_version
-            Integration.version
-          end
-
           def patch
             ::SimpleCov.include(ResultExtractor)
           end

data/lib/datadog/ci/test.rb

@@ -17,6 +17,11 @@ module Datadog
         get_tag(Ext::Test::TAG_NAME)
       end
 
+      # @return [String] the test id according to Datadog's test impact analysis.
+      def datadog_test_id
+        @datadog_test_id ||= Utils::TestRun.datadog_test_id(name, test_suite_name, parameters)
+      end
+
       # Finishes the current test.
       # @return [void]
       def finish
@@ -140,22 +145,18 @@ module Datadog
 
       # @internal
       def any_retry_passed?
-        !!test_suite&.any_test_retry_passed?(test_id)
+        !!test_suite&.any_test_retry_passed?(datadog_test_id)
       end
 
       private
 
-      def test_id
-        @test_id ||= Utils::TestRun.datadog_test_id(name, test_suite_name, parameters)
-      end
-
       def record_test_result(datadog_status)
         # if this test was already executed in this test suite, mark it as retried
-        if test_suite&.test_executed?(test_id)
+        if test_suite&.test_executed?(datadog_test_id)
           set_tag(Ext::Test::TAG_IS_RETRY, "true")
         end
 
-        test_suite&.record_test_result(test_id, datadog_status)
+        test_suite&.record_test_result(datadog_test_id, datadog_status)
       end
     end
   end

data/lib/datadog/ci/test_optimisation/component.rb

@@ -144,11 +144,16 @@ module Datadog
           event
         end
 
+        def skippable?(test)
+          return false if !enabled? || !skipping_tests?
+
+          @skippable_tests.include?(test.datadog_test_id)
+        end
+
         def mark_if_skippable(test)
           return if !enabled? || !skipping_tests?
 
-          datadog_test_id = Utils::TestRun.datadog_test_id(test.name, test.test_suite_name, test.parameters)
-          if @skippable_tests.include?(datadog_test_id)
+          if skippable?(test)
             if forked?
               Datadog.logger.warn { "Intelligent test runner is not supported for forking test runners yet" }
               return
@@ -156,9 +161,9 @@ module Datadog
 
             test.set_tag(Ext::Test::TAG_ITR_SKIPPED_BY_ITR, "true")
 
-            Datadog.logger.debug { "Marked test as skippable: #{datadog_test_id}" }
+            Datadog.logger.debug { "Marked test as skippable: #{test.datadog_test_id}" }
           else
-            Datadog.logger.debug { "Test is not skippable: #{datadog_test_id}" }
+            Datadog.logger.debug { "Test is not skippable: #{test.datadog_test_id}" }
           end
         end
 

data/lib/datadog/ci/test_retries/strategy/retry_new.rb

@@ -101,7 +101,7 @@ module Datadog
             end
             @total_limit = (tests_count * percentage_limit / 100.0).ceil
             Datadog.logger.debug do
-              "Retry new tests total limit is [#{@total_limit}] (#{percentage_limit}%) of #{tests_count}"
+              "Retry new tests total limit is [#{@total_limit}] (#{percentage_limit}% of #{tests_count})"
             end
           end
 

data/lib/datadog/ci/test_visibility/component.rb

@@ -7,7 +7,7 @@ require_relative "telemetry"
 require_relative "total_coverage"
 
 require_relative "../codeowners/parser"
-require_relative "../contrib/contrib"
+require_relative "../contrib/instrumentation"
 require_relative "../ext/test"
 require_relative "../git/local_repository"
 
@@ -149,7 +149,7 @@ module Datadog
           git_tree_upload_worker.perform(test_session.git_repository_url)
 
           # finds and instruments additional test libraries that we support (ex: selenium-webdriver)
-          Contrib.auto_instrument_on_session_start!
+          Contrib::Instrumentation.instrument_on_session_start
 
           # sends internal telemetry events
           Telemetry.test_session_started(test_session)

data/lib/datadog/ci/test_visibility/telemetry.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require_relative "../contrib/instrumentation"
 require_relative "../ext/app_types"
 require_relative "../ext/environment"
 require_relative "../ext/telemetry"
@@ -33,7 +34,7 @@ module Datadog
             Ext::Telemetry::METRIC_TEST_SESSION,
             1,
             {
-              Ext::Telemetry::TAG_AUTO_INJECTED => "false", # ruby doesn't support auto injection yet
+              Ext::Telemetry::TAG_AUTO_INJECTED => Contrib::Instrumentation.auto_instrumented?.to_s,
               Ext::Telemetry::TAG_PROVIDER => test_session.ci_provider || Ext::Telemetry::Provider::UNSUPPORTED
             }
           )

data/lib/datadog/ci/version.rb

@@ -4,8 +4,8 @@ module Datadog
   module CI
     module VERSION
       MAJOR = 1
-      MINOR = 8
-      PATCH = 1
+      MINOR = 10
+      PATCH = 0
       PRE = nil
       BUILD = nil
       # PRE and BUILD above are modified for dev gems during gem build GHA workflow

data/lib/datadog/ci.rb

@@ -406,9 +406,17 @@ module Datadog
 end
 
 # Integrations
+
+# Test frameworks (manual instrumentation)
 require_relative "ci/contrib/cucumber/integration"
-require_relative "ci/contrib/rspec/integration"
 require_relative "ci/contrib/minitest/integration"
+require_relative "ci/contrib/rspec/integration"
+
+# Test runners (instrumented automatically when corresponding frameworks are instrumented)
+require_relative "ci/contrib/knapsack/integration"
+require_relative "ci/contrib/ciqueue/integration"
+
+# Additional test libraries (auto instrumented later on test session start)
 require_relative "ci/contrib/selenium/integration"
 require_relative "ci/contrib/simplecov/integration"