datadog-ci 0.5.1 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 85bb3584dc8081e8f6e8531d2a600e3948f4f9ea23d551dd99df0eab9daf2a3c
-  data.tar.gz: b6f686d49d3dbb1695c0f6e20b434b77648a118c32e1691f4a0f30b13d38dc62
+  metadata.gz: 63e9fdf2c856764bb9567cf89d76f27a4f327244152c510abcd4d64e180a0cc6
+  data.tar.gz: 963f5b534027215799efe65a108a4aa8267c2c31aa4fc1402d19d02f996fc2e9
 SHA512:
-  metadata.gz: 8e8ae716ea7c7408c59c11efa5a3bc30e0767a02c659e231f1cec2a02d5494d451c1e2b43c7c16240cbe7c9418f004ead32faa09b08c83bcd878156fa1062ae9
-  data.tar.gz: 15bdfb45e32660356fcc4c5726c4a640c78cfc2d5007807623f532a0b7c321f0b11a32d0a75f571a64ae263159d4b86b0a750dda5d7e8a5c33f9c110bf7682a0
+  metadata.gz: e94bbd5cc929f95a386cadd633b93f2a53962eefe686354712a9801a8b9865a52fc74c6401f96635a47b9001d71287cadc568ba3bb81ed764d68bdf232b85e5b
+  data.tar.gz: 7a33d2dc09b32798f2fcae80a96bec0e919b4c60f65283f4cf158a020ecd573e450bb925c790c822c63a86b4cf214fae4427e7d4bdb6cde28ca1105dcc95a20a

data/CHANGELOG.md

@@ -1,5 +1,17 @@
 ## [Unreleased]
 
+## [0.6.0] - 2024-01-03
+
+### Added
+
+* Test suite level visibility instrumentation for RSpec ([#86][])
+* Test suite level visibility instrumentation for Cucumber ([#90][])
+* Test suite level visibility instrumentation for Minitest framework ([#92][])
+
+### Fixed
+
+* Do not instantiate TestVisibility::Recorder unless CI visibility is enabled ([#89][])
+
 ## [0.5.1] - 2023-12-11
 
 ### Fixed
@@ -153,4 +165,8 @@ Currently test suite level visibility is not used by our instrumentation: it wil
 [#82]: https://github.com/DataDog/datadog-ci-rb/issues/82
 [#84]: https://github.com/DataDog/datadog-ci-rb/issues/84
 [#85]: https://github.com/DataDog/datadog-ci-rb/issues/85
+[#86]: https://github.com/DataDog/datadog-ci-rb/issues/86
 [#87]: https://github.com/DataDog/datadog-ci-rb/issues/87
+[#89]: https://github.com/DataDog/datadog-ci-rb/issues/89
+[#90]: https://github.com/DataDog/datadog-ci-rb/issues/90
+[#92]: https://github.com/DataDog/datadog-ci-rb/issues/92

data/lib/datadog/ci/configuration/components.rb

@@ -7,6 +7,7 @@ require_relative "../ext/transport"
 require_relative "../ext/settings"
 require_relative "../test_visibility/flush"
 require_relative "../test_visibility/recorder"
+require_relative "../test_visibility/null_recorder"
 require_relative "../test_visibility/serializers/factories/test_level"
 require_relative "../test_visibility/serializers/factories/test_suite_level"
 require_relative "../test_visibility/transport"
@@ -21,14 +22,12 @@ module Datadog
 
         def initialize(settings)
           # Activate CI mode if enabled
-          activate_ci!(settings) if settings.ci.enabled
-
-          @ci_recorder = TestVisibility::Recorder.new(
-            enabled: settings.ci.enabled,
-            test_suite_level_visibility_enabled: settings.ci.experimental_test_suite_level_visibility_enabled
-          )
+          if settings.ci.enabled
+            activate_ci!(settings)
+          else
+            @ci_recorder = TestVisibility::NullRecorder.new
+          end
 
-          # Initialize normally
           super
         end
 
@@ -65,11 +64,16 @@ module Datadog
           if test_visibility_transport
             writer_options[:transport] = test_visibility_transport
             writer_options[:shutdown_timeout] = 60
+            writer_options[:buffer_size] = 10_000
 
             settings.tracing.test_mode.async = true
           end
 
           settings.tracing.test_mode.writer_options = writer_options
+
+          @ci_recorder = TestVisibility::Recorder.new(
+            test_suite_level_visibility_enabled: settings.ci.experimental_test_suite_level_visibility_enabled
+          )
         end
 
         def can_use_evp_proxy?(settings, agent_settings)

data/lib/datadog/ci/contrib/cucumber/configuration/settings.rb

@@ -21,7 +21,7 @@ module Datadog
 
             option :service_name do |o|
               o.type :string
-              o.default { Datadog.configuration.service_without_fallback || Ext::SERVICE_NAME }
+              o.default { Datadog.configuration.service_without_fallback || Ext::DEFAULT_SERVICE_NAME }
             end
 
             # @deprecated Will be removed in 1.0

data/lib/datadog/ci/contrib/cucumber/ext.rb

@@ -7,14 +7,16 @@ module Datadog
         # Cucumber integration constants
         # TODO: mark as `@public_api` when GA, to protect from resource and tag name changes.
         module Ext
-          APP = "cucumber"
           ENV_ENABLED = "DD_TRACE_CUCUMBER_ENABLED"
-          ENV_OPERATION_NAME = "DD_TRACE_CUCUMBER_OPERATION_NAME"
+          DEFAULT_SERVICE_NAME = "cucumber"
+
           FRAMEWORK = "cucumber"
-          OPERATION_NAME = "cucumber.test"
-          SERVICE_NAME = "cucumber"
+
           STEP_SPAN_TYPE = "step"
-          TEST_TYPE = "test"
+
+          # TODO: remove in 1.0
+          ENV_OPERATION_NAME = "DD_TRACE_CUCUMBER_OPERATION_NAME"
+          OPERATION_NAME = "cucumber.test"
         end
       end
     end

data/lib/datadog/ci/contrib/cucumber/formatter.rb

@@ -9,31 +9,60 @@ module Datadog
       module Cucumber
         # Defines collection of instrumented Cucumber events
         class Formatter
-          attr_reader :config, :current_feature_span, :current_step_span
+          attr_reader :config
           private :config
-          private :current_feature_span, :current_step_span
 
           def initialize(config)
             @config = config
+            @failed_tests_count = 0
+
+            @current_test_suite = nil
+            @failed_tests_in_current_test_suite = 0
 
             bind_events(config)
           end
 
           def bind_events(config)
+            config.on_event :test_run_started, &method(:on_test_run_started)
+            config.on_event :test_run_finished, &method(:on_test_run_finished)
             config.on_event :test_case_started, &method(:on_test_case_started)
             config.on_event :test_case_finished, &method(:on_test_case_finished)
             config.on_event :test_step_started, &method(:on_test_step_started)
             config.on_event :test_step_finished, &method(:on_test_step_finished)
           end
 
+          def on_test_run_started(event)
+            test_session = CI.start_test_session(
+              tags: {
+                CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK,
+                CI::Ext::Test::TAG_FRAMEWORK_VERSION => CI::Contrib::Cucumber::Integration.version.to_s,
+                CI::Ext::Test::TAG_TYPE => CI::Ext::Test::TEST_TYPE
+              },
+              service: configuration[:service_name]
+            )
+            CI.start_test_module(test_session.name)
+          end
+
+          def on_test_run_finished(event)
+            if event.respond_to?(:success)
+              finish_session(event.success)
+            else
+              finish_session(@failed_tests_count.zero?)
+            end
+          end
+
           def on_test_case_started(event)
+            test_suite_name = event.test_case.location.file
+
+            start_test_suite(test_suite_name) unless same_test_suite_as_current?(test_suite_name)
+
             CI.start_test(
               event.test_case.name,
-              event.test_case.location.file,
+              test_suite_name,
               tags: {
                 CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK,
                 CI::Ext::Test::TAG_FRAMEWORK_VERSION => CI::Contrib::Cucumber::Integration.version.to_s,
-                CI::Ext::Test::TAG_TYPE => Ext::TEST_TYPE
+                CI::Ext::Test::TAG_TYPE => CI::Ext::Test::TEST_TYPE
               },
               service: configuration[:service_name]
             )
@@ -43,15 +72,17 @@ module Datadog
             test_span = CI.active_test
             return if test_span.nil?
 
-            if event.result.skipped?
-              test_span.skipped!
-            elsif event.result.ok?
-              test_span.passed!
-            elsif event.result.failed?
-              test_span.failed!
+            # We need to track overall test failures manually if we are using cucumber < 8.0 because
+            # TestRunFinished event does not have a success attribute before 8.0.
+            #
+            # To track whether test suite failed or passed we need to
+            # track the number of failed tests in the current test suite.
+            if event.result.failed?
+              @failed_tests_count += 1
+              @failed_tests_in_current_test_suite += 1
             end
 
-            test_span.finish
+            finish_test(test_span, event.result)
           end
 
           def on_test_step_started(event)
@@ -62,18 +93,67 @@ module Datadog
             current_step_span = CI.active_span(Ext::STEP_SPAN_TYPE)
             return if current_step_span.nil?
 
-            if event.result.skipped?
-              current_step_span.skipped!
-            elsif event.result.ok?
-              current_step_span.passed!
-            elsif event.result.failed?
-              current_step_span.failed!(exception: event.result.exception)
+            finish_test(current_step_span, event.result)
+          end
+
+          private
+
+          def finish_test(span, result)
+            if result.skipped?
+              span.skipped!
+            elsif result.ok?
+              span.passed!
+            elsif result.failed?
+              span.failed!(exception: result.exception)
             end
+            span.finish
+          end
+
+          def finish_session(result)
+            finish_current_test_suite
+
+            test_session = CI.active_test_session
+            test_module = CI.active_test_module
+
+            return unless test_session && test_module
 
-            current_step_span.finish
+            if result
+              test_module.passed!
+              test_session.passed!
+            else
+              test_module.failed!
+              test_session.failed!
+            end
+
+            test_module.finish
+            test_session.finish
           end
 
-          private
+          def start_test_suite(test_suite_name)
+            finish_current_test_suite
+
+            @current_test_suite = CI.start_test_suite(test_suite_name)
+          end
+
+          def finish_current_test_suite
+            test_suite = @current_test_suite
+            return unless test_suite
+
+            if @failed_tests_in_current_test_suite.zero?
+              test_suite.passed!
+            else
+              test_suite.failed!
+            end
+            @failed_tests_in_current_test_suite = 0
+            test_suite.finish
+          end
+
+          def same_test_suite_as_current?(test_suite_name)
+            test_suite = @current_test_suite
+            return false unless test_suite
+
+            test_suite.name == test_suite_name
+          end
 
           def configuration
             Datadog.configuration.ci[:cucumber]

data/lib/datadog/ci/contrib/minitest/configuration/settings.rb

@@ -19,7 +19,7 @@ module Datadog
 
             option :service_name do |o|
               o.type :string
-              o.default { Datadog.configuration.service_without_fallback || Ext::SERVICE_NAME }
+              o.default { Datadog.configuration.service_without_fallback || Ext::DEFAULT_SERVICE_NAME }
             end
 
             # @deprecated Will be removed in 1.0

data/lib/datadog/ci/contrib/minitest/ext.rb

@@ -7,13 +7,15 @@ module Datadog
         # Minitest integration constants
         # TODO: mark as `@public_api` when GA, to protect from resource and tag name changes.
         module Ext
-          APP = "minitest"
           ENV_ENABLED = "DD_TRACE_MINITEST_ENABLED"
-          ENV_OPERATION_NAME = "DD_TRACE_MINITEST_OPERATION_NAME"
+
           FRAMEWORK = "minitest"
+
+          DEFAULT_SERVICE_NAME = "minitest"
+
+          # TODO: remove in 1.0
+          ENV_OPERATION_NAME = "DD_TRACE_MINITEST_OPERATION_NAME"
           OPERATION_NAME = "minitest.test"
-          SERVICE_NAME = "minitest"
-          TEST_TYPE = "test"
         end
       end
     end

data/lib/datadog/ci/contrib/minitest/helpers.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Datadog
+  module CI
+    module Contrib
+      module Minitest
+        module Helpers
+          def self.test_suite_name(klass, method_name)
+            source_location, = klass.instance_method(method_name).source_location
+            source_file_path = Pathname.new(source_location.to_s).relative_path_from(Pathname.pwd).to_s
+
+            "#{klass.name} at #{source_file_path}"
+          end
+
+          def self.parallel?(klass)
+            klass.ancestors.include?(::Minitest::Parallel::Test)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/contrib/minitest/hooks.rb

@@ -2,6 +2,7 @@
 
 require_relative "../../ext/test"
 require_relative "ext"
+require_relative "helpers"
 
 module Datadog
   module CI
@@ -11,22 +12,27 @@ module Datadog
         module Hooks
           def before_setup
             super
-            return unless configuration[:enabled]
+            return unless datadog_configuration[:enabled]
 
             test_name = "#{class_name}##{name}"
 
-            path, = method(name).source_location
-            test_suite = Pathname.new(path.to_s).relative_path_from(Pathname.pwd).to_s
+            test_suite_name = Helpers.test_suite_name(self.class, name)
+            if Helpers.parallel?(self.class)
+              test_suite_name = "#{test_suite_name} (#{name} concurrently)"
+
+              # for parallel execution we need to start a new test suite for each test
+              CI.start_test_suite(test_suite_name)
+            end
 
             CI.start_test(
               test_name,
-              test_suite,
+              test_suite_name,
               tags: {
                 CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK,
                 CI::Ext::Test::TAG_FRAMEWORK_VERSION => CI::Contrib::Minitest::Integration.version.to_s,
-                CI::Ext::Test::TAG_TYPE => Ext::TEST_TYPE
+                CI::Ext::Test::TAG_TYPE => CI::Ext::Test::TEST_TYPE
               },
-              service: configuration[:service_name]
+              service: datadog_configuration[:service_name]
             )
           end
 
@@ -34,24 +40,46 @@ module Datadog
             test_span = CI.active_test
             return super unless test_span
 
-            case result_code
-            when "."
-              test_span.passed!
-            when "E", "F"
-              test_span.failed!(exception: failure)
-            when "S"
-              test_span.skipped!(reason: failure.message)
+            finish_test(test_span, result_code)
+            if Helpers.parallel?(self.class)
+              finish_test_suite(test_span.test_suite, result_code)
             end
 
-            test_span.finish
-
             super
           end
 
           private
 
-          def configuration
-            ::Datadog.configuration.ci[:minitest]
+          def finish_test(test_span, result_code)
+            finish_with_result(test_span, result_code)
+
+            # mark test suite as failed if any test failed
+            if test_span.failed?
+              test_suite = test_span.test_suite
+              test_suite.failed! if test_suite
+            end
+          end
+
+          def finish_test_suite(test_suite, result_code)
+            return unless test_suite
+
+            finish_with_result(test_suite, result_code)
+          end
+
+          def finish_with_result(span, result_code)
+            case result_code
+            when "."
+              span.passed!
+            when "E", "F"
+              span.failed!(exception: failure)
+            when "S"
+              span.skipped!(reason: failure.message)
+            end
+            span.finish
+          end
+
+          def datadog_configuration
+            Datadog.configuration.ci[:minitest]
           end
         end
       end

data/lib/datadog/ci/contrib/minitest/patcher.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require_relative "hooks"
+require_relative "runnable"
 
 module Datadog
   module CI
@@ -17,7 +18,13 @@ module Datadog
           end
 
           def patch
+            require_relative "plugin"
+
             ::Minitest::Test.include(Hooks)
+            ::Minitest.include(Plugin)
+            ::Minitest::Runnable.include(Runnable)
+
+            ::Minitest.extensions << "datadog_ci"
           end
         end
       end

data/lib/datadog/ci/contrib/minitest/plugin.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+require "weakref"
+
+require_relative "../../ext/test"
+require_relative "ext"
+
+module Datadog
+  module CI
+    module Contrib
+      module Minitest
+        module Plugin
+          def self.included(base)
+            base.extend(ClassMethods)
+          end
+
+          class DatadogReporter < ::Minitest::AbstractReporter
+            def initialize(minitest_reporter)
+              # This creates circular reference as minitest_reporter also holds reference to DatadogReporter.
+              # To make sure that minitest_reporter can be garbage collected, we use WeakRef.
+              @reporter = WeakRef.new(minitest_reporter)
+            end
+
+            def report
+              active_test_session = CI.active_test_session
+              active_test_module = CI.active_test_module
+
+              return unless @reporter.weakref_alive?
+              return if active_test_session.nil? || active_test_module.nil?
+
+              if @reporter.passed?
+                active_test_module.passed!
+                active_test_session.passed!
+              else
+                active_test_module.failed!
+                active_test_session.failed!
+              end
+
+              active_test_module.finish
+              active_test_session.finish
+
+              nil
+            end
+          end
+
+          module ClassMethods
+            def plugin_datadog_ci_init(*)
+              return unless datadog_configuration[:enabled]
+
+              test_session = CI.start_test_session(
+                tags: {
+                  CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK,
+                  CI::Ext::Test::TAG_FRAMEWORK_VERSION => CI::Contrib::Minitest::Integration.version.to_s,
+                  CI::Ext::Test::TAG_TYPE => CI::Ext::Test::TEST_TYPE
+                },
+                service: datadog_configuration[:service_name]
+              )
+              CI.start_test_module(test_session.name)
+
+              reporter.reporters << DatadogReporter.new(reporter)
+            end
+
+            private
+
+            def datadog_configuration
+              Datadog.configuration.ci[:minitest]
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/contrib/minitest/runnable.rb

@@ -0,0 +1,42 @@
+require_relative "helpers"
+
+module Datadog
+  module CI
+    module Contrib
+      module Minitest
+        module Runnable
+          def self.included(base)
+            base.singleton_class.prepend(ClassMethods)
+          end
+
+          module ClassMethods
+            def run(*)
+              return super unless datadog_configuration[:enabled]
+              return super if Helpers.parallel?(self)
+
+              method = runnable_methods.first
+              return super if method.nil?
+
+              test_suite_name = Helpers.test_suite_name(self, method)
+
+              test_suite = Datadog::CI.start_test_suite(test_suite_name)
+              test_suite.passed! # will be overridden if any test fails
+
+              results = super
+
+              test_suite.finish
+
+              results
+            end
+
+            private
+
+            def datadog_configuration
+              Datadog.configuration.ci[:minitest]
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/contrib/rspec/configuration/settings.rb

@@ -19,7 +19,7 @@ module Datadog
 
             option :service_name do |o|
               o.type :string
-              o.default { Datadog.configuration.service_without_fallback || Ext::SERVICE_NAME }
+              o.default { Datadog.configuration.service_without_fallback || Ext::DEFAULT_SERVICE_NAME }
             end
 
             # @deprecated Will be removed in 1.0

data/lib/datadog/ci/contrib/rspec/example.rb

@@ -26,11 +26,11 @@ module Datadog
 
               CI.trace_test(
                 test_name,
-                metadata[:example_group][:file_path],
+                metadata[:example_group][:rerun_file_path],
                 tags: {
                   CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK,
                   CI::Ext::Test::TAG_FRAMEWORK_VERSION => CI::Contrib::RSpec::Integration.version.to_s,
-                  CI::Ext::Test::TAG_TYPE => Ext::TEST_TYPE
+                  CI::Ext::Test::TAG_TYPE => CI::Ext::Test::TEST_TYPE
                 },
                 service: configuration[:service_name]
               ) do |test_span|

data/lib/datadog/ci/contrib/rspec/example_group.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require_relative "../../ext/test"
+require_relative "ext"
+
+module Datadog
+  module CI
+    module Contrib
+      module RSpec
+        # Instrument RSpec::Core::Example
+        module ExampleGroup
+          def self.included(base)
+            base.singleton_class.prepend(ClassMethods)
+          end
+
+          # Instance methods for configuration
+          module ClassMethods
+            def run(reporter = ::RSpec::Core::NullReporter)
+              return super unless configuration[:enabled]
+              return super unless top_level?
+
+              test_suite = Datadog::CI.start_test_suite(file_path)
+
+              result = super
+
+              if result
+                test_suite.passed!
+              else
+                test_suite.failed!
+              end
+              test_suite.finish
+
+              result
+            end
+
+            private
+
+            def configuration
+              Datadog.configuration.ci[:rspec]
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/contrib/rspec/ext.rb

@@ -7,13 +7,14 @@ module Datadog
         # RSpec integration constants
         # TODO: mark as `@public_api` when GA, to protect from resource and tag name changes.
         module Ext
-          APP = "rspec"
+          FRAMEWORK = "rspec"
+          DEFAULT_SERVICE_NAME = "rspec"
+
           ENV_ENABLED = "DD_TRACE_RSPEC_ENABLED"
+
+          # TODO: remove in 1.0
           ENV_OPERATION_NAME = "DD_TRACE_RSPEC_OPERATION_NAME"
-          FRAMEWORK = "rspec"
           OPERATION_NAME = "rspec.example"
-          SERVICE_NAME = "rspec"
-          TEST_TYPE = "test"
         end
       end
     end

data/lib/datadog/ci/contrib/rspec/integration.rb

@@ -22,7 +22,7 @@ module Datadog
           end
 
           def self.loaded?
-            !defined?(::RSpec).nil? && !defined?(::RSpec::Core).nil? && \
+            !defined?(::RSpec).nil? && !defined?(::RSpec::Core).nil? &&
               !defined?(::RSpec::Core::Example).nil?
           end