datadog-ci 0.5.1 → 0.7.0

data/lib/datadog/ci/contrib/minitest/ext.rb

@@ -7,13 +7,15 @@ module Datadog
         # Minitest integration constants
         # TODO: mark as `@public_api` when GA, to protect from resource and tag name changes.
         module Ext
-          APP = "minitest"
           ENV_ENABLED = "DD_TRACE_MINITEST_ENABLED"
-          ENV_OPERATION_NAME = "DD_TRACE_MINITEST_OPERATION_NAME"
+
           FRAMEWORK = "minitest"
+
+          DEFAULT_SERVICE_NAME = "minitest"
+
+          # TODO: remove in 1.0
+          ENV_OPERATION_NAME = "DD_TRACE_MINITEST_OPERATION_NAME"
           OPERATION_NAME = "minitest.test"
-          SERVICE_NAME = "minitest"
-          TEST_TYPE = "test"
         end
       end
     end

data/lib/datadog/ci/contrib/minitest/helpers.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Datadog
+  module CI
+    module Contrib
+      module Minitest
+        module Helpers
+          def self.test_suite_name(klass, method_name)
+            source_location, = klass.instance_method(method_name).source_location
+            source_file_path = Pathname.new(source_location.to_s).relative_path_from(Pathname.pwd).to_s
+
+            "#{klass.name} at #{source_file_path}"
+          end
+
+          def self.parallel?(klass)
+            klass.ancestors.include?(::Minitest::Parallel::Test) ||
+              (defined?(::Minitest::Queue) && ::Minitest.singleton_class.ancestors.include?(::Minitest::Queue))
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/contrib/minitest/hooks.rb

@@ -2,6 +2,7 @@
 
 require_relative "../../ext/test"
 require_relative "ext"
+require_relative "helpers"
 
 module Datadog
   module CI
@@ -11,22 +12,28 @@ module Datadog
         module Hooks
           def before_setup
             super
-            return unless configuration[:enabled]
+            return unless datadog_configuration[:enabled]
 
-            test_name = "#{class_name}##{name}"
+            test_suite_name = Helpers.test_suite_name(self.class, name)
+            if Helpers.parallel?(self.class)
+              test_suite_name = "#{test_suite_name} (#{name} concurrently)"
 
-            path, = method(name).source_location
-            test_suite = Pathname.new(path.to_s).relative_path_from(Pathname.pwd).to_s
+              # for parallel execution we need to start a new test suite for each test
+              CI.start_test_suite(test_suite_name)
+            end
+
+            source_file, line_number = method(name).source_location
 
             CI.start_test(
-              test_name,
-              test_suite,
+              name,
+              test_suite_name,
               tags: {
                 CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK,
                 CI::Ext::Test::TAG_FRAMEWORK_VERSION => CI::Contrib::Minitest::Integration.version.to_s,
-                CI::Ext::Test::TAG_TYPE => Ext::TEST_TYPE
+                CI::Ext::Test::TAG_SOURCE_FILE => Utils::Git.relative_to_root(source_file),
+                CI::Ext::Test::TAG_SOURCE_START => line_number.to_s
               },
-              service: configuration[:service_name]
+              service: datadog_configuration[:service_name]
             )
           end
 
@@ -34,24 +41,32 @@ module Datadog
             test_span = CI.active_test
             return super unless test_span
 
-            case result_code
-            when "."
-              test_span.passed!
-            when "E", "F"
-              test_span.failed!(exception: failure)
-            when "S"
-              test_span.skipped!(reason: failure.message)
+            finish_with_result(test_span, result_code)
+            if Helpers.parallel?(self.class)
+              finish_with_result(test_span.test_suite, result_code)
             end
 
-            test_span.finish
-
             super
           end
 
           private
 
-          def configuration
-            ::Datadog.configuration.ci[:minitest]
+          def finish_with_result(span, result_code)
+            return unless span
+
+            case result_code
+            when "."
+              span.passed!
+            when "E", "F"
+              span.failed!(exception: failure)
+            when "S"
+              span.skipped!(reason: failure.message)
+            end
+            span.finish
+          end
+
+          def datadog_configuration
+            Datadog.configuration.ci[:minitest]
           end
         end
       end

data/lib/datadog/ci/contrib/minitest/patcher.rb

@@ -1,6 +1,9 @@
 # frozen_string_literal: true
 
+require_relative "runner"
+require_relative "reporter"
 require_relative "hooks"
+require_relative "runnable"
 
 module Datadog
   module CI
@@ -17,7 +20,14 @@ module Datadog
           end
 
           def patch
+            # test session start
+            ::Minitest.include(Runner)
+            # test suites (when not executed concurrently)
+            ::Minitest::Runnable.include(Runnable)
+            # tests; test suites (when executed concurrently)
             ::Minitest::Test.include(Hooks)
+            # test session finish
+            ::Minitest::CompositeReporter.include(Reporter)
           end
         end
       end

data/lib/datadog/ci/contrib/minitest/reporter.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+require_relative "../../ext/test"
+require_relative "ext"
+
+module Datadog
+  module CI
+    module Contrib
+      module Minitest
+        module Reporter
+          def self.included(base)
+            base.prepend(InstanceMethods)
+          end
+
+          module InstanceMethods
+            def report(*)
+              return super unless datadog_configuration[:enabled]
+
+              res = super
+
+              active_test_session = CI.active_test_session
+              active_test_module = CI.active_test_module
+
+              return res if active_test_session.nil? || active_test_module.nil?
+
+              if passed?
+                active_test_module.passed!
+                active_test_session.passed!
+              else
+                active_test_module.failed!
+                active_test_session.failed!
+              end
+
+              active_test_module.finish
+              active_test_session.finish
+
+              res
+            end
+
+            private
+
+            def datadog_configuration
+              Datadog.configuration.ci[:minitest]
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/contrib/minitest/runnable.rb

@@ -0,0 +1,42 @@
+require_relative "helpers"
+
+module Datadog
+  module CI
+    module Contrib
+      module Minitest
+        module Runnable
+          def self.included(base)
+            base.singleton_class.prepend(ClassMethods)
+          end
+
+          module ClassMethods
+            def run(*)
+              return super unless datadog_configuration[:enabled]
+              return super if Helpers.parallel?(self)
+
+              method = runnable_methods.first
+              return super if method.nil?
+
+              test_suite_name = Helpers.test_suite_name(self, method)
+
+              test_suite = Datadog::CI.start_test_suite(test_suite_name)
+
+              results = super
+              return results unless test_suite
+
+              test_suite.finish
+
+              results
+            end
+
+            private
+
+            def datadog_configuration
+              Datadog.configuration.ci[:minitest]
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/contrib/minitest/runner.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+require_relative "../../ext/test"
+require_relative "ext"
+
+module Datadog
+  module CI
+    module Contrib
+      module Minitest
+        module Runner
+          def self.included(base)
+            base.singleton_class.prepend(ClassMethods)
+          end
+
+          module ClassMethods
+            def init_plugins(*)
+              super
+
+              return unless datadog_configuration[:enabled]
+
+              test_session = CI.start_test_session(
+                tags: {
+                  CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK,
+                  CI::Ext::Test::TAG_FRAMEWORK_VERSION => CI::Contrib::Minitest::Integration.version.to_s
+                },
+                service: datadog_configuration[:service_name]
+              )
+              CI.start_test_module(test_session.name) if test_session
+            end
+
+            private
+
+            def datadog_configuration
+              Datadog.configuration.ci[:minitest]
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/contrib/rspec/configuration/settings.rb

@@ -2,6 +2,7 @@
 
 require_relative "../ext"
 require_relative "../../settings"
+require_relative "../../../utils/configuration"
 
 module Datadog
   module CI
@@ -19,7 +20,9 @@ module Datadog
 
             option :service_name do |o|
               o.type :string
-              o.default { Datadog.configuration.service_without_fallback || Ext::SERVICE_NAME }
+              o.default do
+                Utils::Configuration.fetch_service_name(Ext::DEFAULT_SERVICE_NAME)
+              end
             end
 
             # @deprecated Will be removed in 1.0

data/lib/datadog/ci/contrib/rspec/example.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require_relative "../../ext/test"
+require_relative "../../utils/git"
 require_relative "ext"
 
 module Datadog
@@ -13,10 +14,9 @@ module Datadog
             base.prepend(InstanceMethods)
           end
 
-          # Instance methods for configuration
           module InstanceMethods
-            def run(example_group_instance, reporter)
-              return super unless configuration[:enabled]
+            def run(*)
+              return super unless datadog_configuration[:enabled]
 
               test_name = full_description.strip
               if metadata[:description].empty?
@@ -24,36 +24,77 @@ module Datadog
                 test_name += " #{description}"
               end
 
+              test_suite_description = fetch_top_level_example_group[:description]
+              suite_name = "#{test_suite_description} at #{metadata[:example_group][:rerun_file_path]}"
+
+              # remove example group description from test name to avoid duplication
+              test_name = test_name.sub(test_suite_description, "").strip
+
+              if ci_queue?
+                suite_name += " (ci-queue running example [#{test_name}])"
+                test_suite_span = CI.start_test_suite(suite_name)
+              end
+
               CI.trace_test(
                 test_name,
-                metadata[:example_group][:file_path],
+                suite_name,
                 tags: {
                   CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK,
                   CI::Ext::Test::TAG_FRAMEWORK_VERSION => CI::Contrib::RSpec::Integration.version.to_s,
-                  CI::Ext::Test::TAG_TYPE => Ext::TEST_TYPE
+                  CI::Ext::Test::TAG_SOURCE_FILE => Utils::Git.relative_to_root(metadata[:file_path]),
+                  CI::Ext::Test::TAG_SOURCE_START => metadata[:line_number].to_s
                 },
-                service: configuration[:service_name]
+                service: datadog_configuration[:service_name]
               ) do |test_span|
                 result = super
 
-                case execution_result.status
-                when :passed
-                  test_span.passed!
-                when :failed
-                  test_span.failed!(exception: execution_result.exception)
-                else
-                  test_span.skipped!(exception: execution_result.exception) if execution_result.example_skipped?
+                if test_span
+                  test_span.set_parameters({}, {"scoped_id" => metadata[:scoped_id]})
+
+                  case execution_result.status
+                  when :passed
+                    test_span.passed!
+                    test_suite_span.passed! if test_suite_span
+                  when :failed
+                    test_span.failed!(exception: execution_result.exception)
+                    test_suite_span.failed! if test_suite_span
+                  else
+                    # :pending or nil
+                    test_span.skipped!(
+                      reason: execution_result.pending_message,
+                      exception: execution_result.pending_exception
+                    )
+
+                    test_suite_span.skipped! if test_suite_span
+                  end
                 end
 
+                test_suite_span.finish if test_suite_span
+
                 result
               end
             end
 
             private
 
-            def configuration
+            def fetch_top_level_example_group
+              return metadata[:example_group] unless metadata[:example_group][:parent_example_group]
+
+              res = metadata[:example_group][:parent_example_group]
+              while (parent = res[:parent_example_group])
+                res = parent
+              end
+              res
+            end
+
+            def datadog_configuration
               Datadog.configuration.ci[:rspec]
             end
+
+            def ci_queue?
+              defined?(::RSpec::Queue::ExampleExtension) &&
+                self.class.ancestors.include?(::RSpec::Queue::ExampleExtension)
+            end
           end
         end
       end

data/lib/datadog/ci/contrib/rspec/example_group.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+require_relative "../../ext/test"
+require_relative "ext"
+
+module Datadog
+  module CI
+    module Contrib
+      module RSpec
+        # Instrument RSpec::Core::ExampleGroup
+        module ExampleGroup
+          def self.included(base)
+            base.singleton_class.prepend(ClassMethods)
+          end
+
+          # Instance methods for configuration
+          module ClassMethods
+            def run(reporter = ::RSpec::Core::NullReporter)
+              return super unless datadog_configuration[:enabled]
+              return super unless top_level?
+
+              suite_name = "#{description} at #{file_path}"
+              test_suite = Datadog::CI.start_test_suite(suite_name)
+
+              success = super
+              return success unless test_suite
+
+              if success && test_suite.passed_tests_count > 0
+                test_suite.passed!
+              elsif success
+                test_suite.skipped!
+              else
+                test_suite.failed!
+              end
+
+              test_suite.finish
+
+              success
+            end
+
+            private
+
+            def datadog_configuration
+              Datadog.configuration.ci[:rspec]
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/contrib/rspec/ext.rb

@@ -7,13 +7,14 @@ module Datadog
         # RSpec integration constants
         # TODO: mark as `@public_api` when GA, to protect from resource and tag name changes.
         module Ext
-          APP = "rspec"
+          FRAMEWORK = "rspec"
+          DEFAULT_SERVICE_NAME = "rspec"
+
           ENV_ENABLED = "DD_TRACE_RSPEC_ENABLED"
+
+          # TODO: remove in 1.0
           ENV_OPERATION_NAME = "DD_TRACE_RSPEC_OPERATION_NAME"
-          FRAMEWORK = "rspec"
           OPERATION_NAME = "rspec.example"
-          SERVICE_NAME = "rspec"
-          TEST_TYPE = "test"
         end
       end
     end

data/lib/datadog/ci/contrib/rspec/integration.rb

@@ -22,7 +22,7 @@ module Datadog
           end
 
           def self.loaded?
-            !defined?(::RSpec).nil? && !defined?(::RSpec::Core).nil? && \
+            !defined?(::RSpec).nil? && !defined?(::RSpec::Core).nil? &&
               !defined?(::RSpec::Core::Example).nil?
           end
 

data/lib/datadog/ci/contrib/rspec/patcher.rb

@@ -1,7 +1,10 @@
 # frozen_string_literal: true
 
 require "datadog/tracing/contrib/patcher"
+
 require_relative "example"
+require_relative "example_group"
+require_relative "runner"
 
 module Datadog
   module CI
@@ -18,7 +21,18 @@ module Datadog
           end
 
           def patch
+            if ci_queue?
+              ::RSpec::Queue::Runner.include(Runner)
+            end
+
+            ::RSpec::Core::Runner.include(Runner)
             ::RSpec::Core::Example.include(Example)
+            ::RSpec::Core::ExampleGroup.include(ExampleGroup)
+          end
+
+          def ci_queue?
+            # ::RSpec::Queue::Runner is a ci-queue runner
+            defined?(::RSpec::Queue::Runner)
           end
         end
       end

data/lib/datadog/ci/contrib/rspec/runner.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+require_relative "../../ext/test"
+require_relative "ext"
+
+module Datadog
+  module CI
+    module Contrib
+      module RSpec
+        # Instrument RSpec::Core::Runner
+        module Runner
+          def self.included(base)
+            base.prepend(InstanceMethods)
+          end
+
+          module InstanceMethods
+            def run_specs(*)
+              return super unless datadog_configuration[:enabled]
+
+              test_session = CI.start_test_session(
+                tags: {
+                  CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK,
+                  CI::Ext::Test::TAG_FRAMEWORK_VERSION => CI::Contrib::RSpec::Integration.version.to_s
+                },
+                service: datadog_configuration[:service_name]
+              )
+
+              test_module = CI.start_test_module(test_session.name) if test_session
+
+              result = super
+              return result unless test_module && test_session
+
+              if result != 0
+                # TODO: repeating this twice feels clunky, we need to remove test_module API before GA
+                test_module.failed!
+                test_session.failed!
+              else
+                test_module.passed!
+                test_session.passed!
+              end
+              test_module.finish
+              test_session.finish
+
+              result
+            end
+
+            private
+
+            def datadog_configuration
+              Datadog.configuration.ci[:rspec]
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/contrib/settings.rb

@@ -27,7 +27,7 @@ module Datadog
         end
 
         def []=(name, value)
-          respond_to?("#{name}=") ? send("#{name}=", value) : set_option(name, value)
+          respond_to?(:"#{name}=") ? send(:"#{name}=", value) : set_option(name, value)
         end
       end
     end

data/lib/datadog/ci/ext/app_types.rb

@@ -3,6 +3,8 @@
 module Datadog
   module CI
     module Ext
+      # Defines span types for CI visibility
+      # @public_api
       module AppTypes
         TYPE_TEST = "test"
         TYPE_TEST_SESSION = "test_session_end"