datadog-ci 0.6.0 → 0.8.0

data/lib/datadog/ci/contrib/cucumber/formatter.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require_relative "../../ext/test"
+require_relative "../../utils/git"
 require_relative "ext"
 
 module Datadog
@@ -13,11 +14,12 @@ module Datadog
           private :config
 
           def initialize(config)
+            @ast_lookup = ::Cucumber::Formatter::AstLookup.new(config) if defined?(::Cucumber::Formatter::AstLookup)
             @config = config
-            @failed_tests_count = 0
 
             @current_test_suite = nil
-            @failed_tests_in_current_test_suite = 0
+
+            @failed_tests_count = 0
 
             bind_events(config)
           end
@@ -32,15 +34,14 @@ module Datadog
           end
 
           def on_test_run_started(event)
-            test_session = CI.start_test_session(
+            CI.start_test_session(
               tags: {
                 CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK,
-                CI::Ext::Test::TAG_FRAMEWORK_VERSION => CI::Contrib::Cucumber::Integration.version.to_s,
-                CI::Ext::Test::TAG_TYPE => CI::Ext::Test::TEST_TYPE
+                CI::Ext::Test::TAG_FRAMEWORK_VERSION => CI::Contrib::Cucumber::Integration.version.to_s
               },
               service: configuration[:service_name]
             )
-            CI.start_test_module(test_session.name)
+            CI.start_test_module(Ext::FRAMEWORK)
           end
 
           def on_test_run_finished(event)
@@ -52,58 +53,71 @@ module Datadog
           end
 
           def on_test_case_started(event)
-            test_suite_name = event.test_case.location.file
+            test_suite_name = test_suite_name(event.test_case)
+
+            tags = {
+              CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK,
+              CI::Ext::Test::TAG_FRAMEWORK_VERSION => CI::Contrib::Cucumber::Integration.version.to_s,
+              CI::Ext::Test::TAG_SOURCE_FILE => Utils::Git.relative_to_root(event.test_case.location.file),
+              CI::Ext::Test::TAG_SOURCE_START => event.test_case.location.line.to_s
+            }
 
             start_test_suite(test_suite_name) unless same_test_suite_as_current?(test_suite_name)
 
-            CI.start_test(
+            test_span = CI.start_test(
               event.test_case.name,
               test_suite_name,
-              tags: {
-                CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK,
-                CI::Ext::Test::TAG_FRAMEWORK_VERSION => CI::Contrib::Cucumber::Integration.version.to_s,
-                CI::Ext::Test::TAG_TYPE => CI::Ext::Test::TEST_TYPE
-              },
+              tags: tags,
               service: configuration[:service_name]
             )
+
+            if test_span && (parameters = extract_parameters_hash(event.test_case))
+              test_span.set_parameters(parameters)
+            end
           end
 
           def on_test_case_finished(event)
             test_span = CI.active_test
             return if test_span.nil?
 
-            # We need to track overall test failures manually if we are using cucumber < 8.0 because
-            # TestRunFinished event does not have a success attribute before 8.0.
-            #
-            # To track whether test suite failed or passed we need to
-            # track the number of failed tests in the current test suite.
-            if event.result.failed?
-              @failed_tests_count += 1
-              @failed_tests_in_current_test_suite += 1
-            end
-
-            finish_test(test_span, event.result)
+            finish_span(test_span, event.result)
+            @failed_tests_count += 1 if test_span.failed?
           end
 
           def on_test_step_started(event)
-            CI.trace(Ext::STEP_SPAN_TYPE, event.test_step.to_s)
+            CI.trace(event.test_step.to_s, type: Ext::STEP_SPAN_TYPE)
           end
 
           def on_test_step_finished(event)
-            current_step_span = CI.active_span(Ext::STEP_SPAN_TYPE)
+            current_step_span = CI.active_span
             return if current_step_span.nil?
 
-            finish_test(current_step_span, event.result)
+            finish_span(current_step_span, event.result)
           end
 
           private
 
-          def finish_test(span, result)
-            if result.skipped?
-              span.skipped!
-            elsif result.ok?
+          def test_suite_name(test_case)
+            feature = if test_case.respond_to?(:feature)
+              test_case.feature
+            elsif @ast_lookup
+              gherkin_doc = @ast_lookup.gherkin_document(test_case.location.file)
+              gherkin_doc.feature if gherkin_doc
+            end
+
+            if feature
+              "#{feature.name} at #{test_case.location.file}"
+            else
+              test_case.location.file
+            end
+          end
+
+          def finish_span(span, result)
+            if !result.passed? && result.ok?(@config.strict)
+              span.skipped!(reason: result.message)
+            elsif result.passed?
               span.passed!
-            elsif result.failed?
+            else
               span.failed!(exception: result.exception)
             end
             span.finish
@@ -139,13 +153,9 @@ module Datadog
             test_suite = @current_test_suite
             return unless test_suite
 
-            if @failed_tests_in_current_test_suite.zero?
-              test_suite.passed!
-            else
-              test_suite.failed!
-            end
-            @failed_tests_in_current_test_suite = 0
             test_suite.finish
+
+            @current_test_suite = nil
           end
 
           def same_test_suite_as_current?(test_suite_name)
@@ -155,6 +165,27 @@ module Datadog
             test_suite.name == test_suite_name
           end
 
+          def extract_parameters_hash(test_case)
+            # not supported in cucumber < 4.0
+            return nil unless @ast_lookup
+
+            scenario_source = @ast_lookup.scenario_source(test_case)
+
+            # cucumber examples are only supported for scenario outlines
+            return nil unless scenario_source.type == :ScenarioOutline
+
+            scenario_source.examples.table_header.cells.map(&:value).zip(
+              scenario_source.row.cells.map(&:value)
+            ).to_h
+          rescue => e
+            Datadog.logger.warn do
+              "Unable to extract parameters from test case #{test_case.name}: " \
+                "#{e.class.name} #{e.message} at #{Array(e.backtrace).first}"
+            end
+
+            nil
+          end
+
           def configuration
             Datadog.configuration.ci[:cucumber]
           end

data/lib/datadog/ci/contrib/cucumber/instrumentation.rb

@@ -17,8 +17,9 @@ module Datadog
             attr_reader :datadog_formatter
 
             def formatters
+              existing_formatters = super
               @datadog_formatter ||= CI::Contrib::Cucumber::Formatter.new(@configuration)
-              [@datadog_formatter] + super
+              [@datadog_formatter] + existing_formatters
             end
           end
         end

data/lib/datadog/ci/contrib/minitest/configuration/settings.rb

@@ -2,6 +2,7 @@
 
 require_relative "../ext"
 require_relative "../../settings"
+require_relative "../../../utils/configuration"
 
 module Datadog
   module CI
@@ -19,7 +20,9 @@ module Datadog
 
             option :service_name do |o|
               o.type :string
-              o.default { Datadog.configuration.service_without_fallback || Ext::DEFAULT_SERVICE_NAME }
+              o.default do
+                Utils::Configuration.fetch_service_name(Ext::DEFAULT_SERVICE_NAME)
+              end
             end
 
             # @deprecated Will be removed in 1.0

data/lib/datadog/ci/contrib/minitest/helpers.rb

@@ -13,7 +13,8 @@ module Datadog
           end
 
           def self.parallel?(klass)
-            klass.ancestors.include?(::Minitest::Parallel::Test)
+            klass.ancestors.include?(::Minitest::Parallel::Test) ||
+              (defined?(::Minitest::Queue) && ::Minitest.singleton_class.ancestors.include?(::Minitest::Queue))
           end
         end
       end

data/lib/datadog/ci/contrib/minitest/hooks.rb

@@ -14,8 +14,6 @@ module Datadog
             super
             return unless datadog_configuration[:enabled]
 
-            test_name = "#{class_name}##{name}"
-
             test_suite_name = Helpers.test_suite_name(self.class, name)
             if Helpers.parallel?(self.class)
               test_suite_name = "#{test_suite_name} (#{name} concurrently)"
@@ -24,13 +22,16 @@ module Datadog
               CI.start_test_suite(test_suite_name)
             end
 
+            source_file, line_number = method(name).source_location
+
             CI.start_test(
-              test_name,
+              name,
               test_suite_name,
               tags: {
                 CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK,
                 CI::Ext::Test::TAG_FRAMEWORK_VERSION => CI::Contrib::Minitest::Integration.version.to_s,
-                CI::Ext::Test::TAG_TYPE => CI::Ext::Test::TEST_TYPE
+                CI::Ext::Test::TAG_SOURCE_FILE => Utils::Git.relative_to_root(source_file),
+                CI::Ext::Test::TAG_SOURCE_START => line_number.to_s
               },
               service: datadog_configuration[:service_name]
             )
@@ -40,9 +41,9 @@ module Datadog
             test_span = CI.active_test
             return super unless test_span
 
-            finish_test(test_span, result_code)
+            finish_with_result(test_span, result_code)
             if Helpers.parallel?(self.class)
-              finish_test_suite(test_span.test_suite, result_code)
+              finish_with_result(test_span.test_suite, result_code)
             end
 
             super
@@ -50,23 +51,9 @@ module Datadog
 
           private
 
-          def finish_test(test_span, result_code)
-            finish_with_result(test_span, result_code)
-
-            # mark test suite as failed if any test failed
-            if test_span.failed?
-              test_suite = test_span.test_suite
-              test_suite.failed! if test_suite
-            end
-          end
-
-          def finish_test_suite(test_suite, result_code)
-            return unless test_suite
-
-            finish_with_result(test_suite, result_code)
-          end
-
           def finish_with_result(span, result_code)
+            return unless span
+
             case result_code
             when "."
               span.passed!

data/lib/datadog/ci/contrib/minitest/patcher.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require_relative "runner"
+require_relative "reporter"
 require_relative "hooks"
 require_relative "runnable"
 
@@ -18,13 +20,14 @@ module Datadog
           end
 
           def patch
-            require_relative "plugin"
-
-            ::Minitest::Test.include(Hooks)
-            ::Minitest.include(Plugin)
+            # test session start
+            ::Minitest.include(Runner)
+            # test suites (when not executed concurrently)
             ::Minitest::Runnable.include(Runnable)
-
-            ::Minitest.extensions << "datadog_ci"
+            # tests; test suites (when executed concurrently)
+            ::Minitest::Test.include(Hooks)
+            # test session finish
+            ::Minitest::CompositeReporter.include(Reporter)
           end
         end
       end

data/lib/datadog/ci/contrib/minitest/reporter.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+require_relative "../../ext/test"
+require_relative "ext"
+
+module Datadog
+  module CI
+    module Contrib
+      module Minitest
+        module Reporter
+          def self.included(base)
+            base.prepend(InstanceMethods)
+          end
+
+          module InstanceMethods
+            def report(*)
+              return super unless datadog_configuration[:enabled]
+
+              res = super
+
+              active_test_session = CI.active_test_session
+              active_test_module = CI.active_test_module
+
+              return res if active_test_session.nil? || active_test_module.nil?
+
+              if passed?
+                active_test_module.passed!
+                active_test_session.passed!
+              else
+                active_test_module.failed!
+                active_test_session.failed!
+              end
+
+              active_test_module.finish
+              active_test_session.finish
+
+              res
+            end
+
+            private
+
+            def datadog_configuration
+              Datadog.configuration.ci[:minitest]
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/contrib/minitest/runnable.rb

@@ -20,9 +20,9 @@ module Datadog
               test_suite_name = Helpers.test_suite_name(self, method)
 
               test_suite = Datadog::CI.start_test_suite(test_suite_name)
-              test_suite.passed! # will be overridden if any test fails
 
               results = super
+              return results unless test_suite
 
               test_suite.finish
 

data/lib/datadog/ci/contrib/minitest/runner.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+require_relative "../../ext/test"
+require_relative "ext"
+
+module Datadog
+  module CI
+    module Contrib
+      module Minitest
+        module Runner
+          def self.included(base)
+            base.singleton_class.prepend(ClassMethods)
+          end
+
+          module ClassMethods
+            def init_plugins(*)
+              super
+
+              return unless datadog_configuration[:enabled]
+
+              CI.start_test_session(
+                tags: {
+                  CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK,
+                  CI::Ext::Test::TAG_FRAMEWORK_VERSION => CI::Contrib::Minitest::Integration.version.to_s
+                },
+                service: datadog_configuration[:service_name]
+              )
+              CI.start_test_module(Ext::FRAMEWORK)
+            end
+
+            private
+
+            def datadog_configuration
+              Datadog.configuration.ci[:minitest]
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/contrib/rspec/configuration/settings.rb

@@ -2,6 +2,7 @@
 
 require_relative "../ext"
 require_relative "../../settings"
+require_relative "../../../utils/configuration"
 
 module Datadog
   module CI
@@ -19,7 +20,9 @@ module Datadog
 
             option :service_name do |o|
               o.type :string
-              o.default { Datadog.configuration.service_without_fallback || Ext::DEFAULT_SERVICE_NAME }
+              o.default do
+                Utils::Configuration.fetch_service_name(Ext::DEFAULT_SERVICE_NAME)
+              end
             end
 
             # @deprecated Will be removed in 1.0

data/lib/datadog/ci/contrib/rspec/example.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require_relative "../../ext/test"
+require_relative "../../utils/git"
 require_relative "ext"
 
 module Datadog
@@ -13,10 +14,9 @@ module Datadog
             base.prepend(InstanceMethods)
           end
 
-          # Instance methods for configuration
           module InstanceMethods
-            def run(example_group_instance, reporter)
-              return super unless configuration[:enabled]
+            def run(*)
+              return super unless datadog_configuration[:enabled]
 
               test_name = full_description.strip
               if metadata[:description].empty?
@@ -24,36 +24,77 @@ module Datadog
                 test_name += " #{description}"
               end
 
+              test_suite_description = fetch_top_level_example_group[:description]
+              suite_name = "#{test_suite_description} at #{metadata[:example_group][:rerun_file_path]}"
+
+              # remove example group description from test name to avoid duplication
+              test_name = test_name.sub(test_suite_description, "").strip
+
+              if ci_queue?
+                suite_name += " (ci-queue running example [#{test_name}])"
+                test_suite_span = CI.start_test_suite(suite_name)
+              end
+
               CI.trace_test(
                 test_name,
-                metadata[:example_group][:rerun_file_path],
+                suite_name,
                 tags: {
                   CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK,
                   CI::Ext::Test::TAG_FRAMEWORK_VERSION => CI::Contrib::RSpec::Integration.version.to_s,
-                  CI::Ext::Test::TAG_TYPE => CI::Ext::Test::TEST_TYPE
+                  CI::Ext::Test::TAG_SOURCE_FILE => Utils::Git.relative_to_root(metadata[:file_path]),
+                  CI::Ext::Test::TAG_SOURCE_START => metadata[:line_number].to_s
                 },
-                service: configuration[:service_name]
+                service: datadog_configuration[:service_name]
               ) do |test_span|
                 result = super
 
-                case execution_result.status
-                when :passed
-                  test_span.passed!
-                when :failed
-                  test_span.failed!(exception: execution_result.exception)
-                else
-                  test_span.skipped!(exception: execution_result.exception) if execution_result.example_skipped?
+                if test_span
+                  test_span.set_parameters({}, {"scoped_id" => metadata[:scoped_id]})
+
+                  case execution_result.status
+                  when :passed
+                    test_span.passed!
+                    test_suite_span.passed! if test_suite_span
+                  when :failed
+                    test_span.failed!(exception: execution_result.exception)
+                    test_suite_span.failed! if test_suite_span
+                  else
+                    # :pending or nil
+                    test_span.skipped!(
+                      reason: execution_result.pending_message,
+                      exception: execution_result.pending_exception
+                    )
+
+                    test_suite_span.skipped! if test_suite_span
+                  end
                 end
 
+                test_suite_span.finish if test_suite_span
+
                 result
               end
             end
 
             private
 
-            def configuration
+            def fetch_top_level_example_group
+              return metadata[:example_group] unless metadata[:example_group][:parent_example_group]
+
+              res = metadata[:example_group][:parent_example_group]
+              while (parent = res[:parent_example_group])
+                res = parent
+              end
+              res
+            end
+
+            def datadog_configuration
               Datadog.configuration.ci[:rspec]
             end
+
+            def ci_queue?
+              defined?(::RSpec::Queue::ExampleExtension) &&
+                self.class.ancestors.include?(::RSpec::Queue::ExampleExtension)
+            end
           end
         end
       end

data/lib/datadog/ci/contrib/rspec/example_group.rb

@@ -7,7 +7,7 @@ module Datadog
   module CI
     module Contrib
       module RSpec
-        # Instrument RSpec::Core::Example
+        # Instrument RSpec::Core::ExampleGroup
         module ExampleGroup
           def self.included(base)
             base.singleton_class.prepend(ClassMethods)
@@ -16,26 +16,31 @@ module Datadog
           # Instance methods for configuration
           module ClassMethods
             def run(reporter = ::RSpec::Core::NullReporter)
-              return super unless configuration[:enabled]
+              return super unless datadog_configuration[:enabled]
               return super unless top_level?
 
-              test_suite = Datadog::CI.start_test_suite(file_path)
+              suite_name = "#{description} at #{file_path}"
+              test_suite = Datadog::CI.start_test_suite(suite_name)
 
-              result = super
+              success = super
+              return success unless test_suite
 
-              if result
+              if success && test_suite.passed_tests_count > 0
                 test_suite.passed!
+              elsif success
+                test_suite.skipped!
               else
                 test_suite.failed!
               end
+
               test_suite.finish
 
-              result
+              success
             end
 
             private
 
-            def configuration
+            def datadog_configuration
               Datadog.configuration.ci[:rspec]
             end
           end

data/lib/datadog/ci/contrib/rspec/patcher.rb

@@ -21,10 +21,19 @@ module Datadog
           end
 
           def patch
-            ::RSpec::Core::Example.include(Example)
+            if ci_queue?
+              ::RSpec::Queue::Runner.include(Runner)
+            end
+
             ::RSpec::Core::Runner.include(Runner)
+            ::RSpec::Core::Example.include(Example)
             ::RSpec::Core::ExampleGroup.include(ExampleGroup)
           end
+
+          def ci_queue?
+            # ::RSpec::Queue::Runner is a ci-queue runner
+            defined?(::RSpec::Queue::Runner)
+          end
         end
       end
     end

data/lib/datadog/ci/contrib/rspec/runner.rb

@@ -14,24 +14,23 @@ module Datadog
           end
 
           module InstanceMethods
-            def run_specs(example_groups)
-              return super unless configuration[:enabled]
+            def run_specs(*)
+              return super unless datadog_configuration[:enabled]
 
               test_session = CI.start_test_session(
                 tags: {
                   CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK,
-                  CI::Ext::Test::TAG_FRAMEWORK_VERSION => CI::Contrib::RSpec::Integration.version.to_s,
-                  CI::Ext::Test::TAG_TYPE => CI::Ext::Test::TEST_TYPE
+                  CI::Ext::Test::TAG_FRAMEWORK_VERSION => CI::Contrib::RSpec::Integration.version.to_s
                 },
-                service: configuration[:service_name]
+                service: datadog_configuration[:service_name]
               )
 
-              test_module = CI.start_test_module(test_session.name)
+              test_module = CI.start_test_module(Ext::FRAMEWORK)
 
               result = super
+              return result unless test_module && test_session
 
               if result != 0
-                # TODO: repeating this twice feels clunky, we need to remove test_module API before GA
                 test_module.failed!
                 test_session.failed!
               else
@@ -46,7 +45,7 @@ module Datadog
 
             private
 
-            def configuration
+            def datadog_configuration
               Datadog.configuration.ci[:rspec]
             end
           end

data/lib/datadog/ci/ext/app_types.rb

@@ -3,6 +3,8 @@
 module Datadog
   module CI
     module Ext
+      # Defines span types for CI visibility
+      # @public_api
       module AppTypes
         TYPE_TEST = "test"
         TYPE_TEST_SESSION = "test_session_end"