datadog-ci 0.6.0 → 0.7.0

data/lib/datadog/ci/contrib/minitest/reporter.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+require_relative "../../ext/test"
+require_relative "ext"
+
+module Datadog
+  module CI
+    module Contrib
+      module Minitest
+        module Reporter
+          def self.included(base)
+            base.prepend(InstanceMethods)
+          end
+
+          module InstanceMethods
+            def report(*)
+              return super unless datadog_configuration[:enabled]
+
+              res = super
+
+              active_test_session = CI.active_test_session
+              active_test_module = CI.active_test_module
+
+              return res if active_test_session.nil? || active_test_module.nil?
+
+              if passed?
+                active_test_module.passed!
+                active_test_session.passed!
+              else
+                active_test_module.failed!
+                active_test_session.failed!
+              end
+
+              active_test_module.finish
+              active_test_session.finish
+
+              res
+            end
+
+            private
+
+            def datadog_configuration
+              Datadog.configuration.ci[:minitest]
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/contrib/minitest/runnable.rb

@@ -20,9 +20,9 @@ module Datadog
               test_suite_name = Helpers.test_suite_name(self, method)
 
               test_suite = Datadog::CI.start_test_suite(test_suite_name)
-              test_suite.passed! # will be overridden if any test fails
 
               results = super
+              return results unless test_suite
 
               test_suite.finish
 

data/lib/datadog/ci/contrib/minitest/runner.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+require_relative "../../ext/test"
+require_relative "ext"
+
+module Datadog
+  module CI
+    module Contrib
+      module Minitest
+        module Runner
+          def self.included(base)
+            base.singleton_class.prepend(ClassMethods)
+          end
+
+          module ClassMethods
+            def init_plugins(*)
+              super
+
+              return unless datadog_configuration[:enabled]
+
+              test_session = CI.start_test_session(
+                tags: {
+                  CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK,
+                  CI::Ext::Test::TAG_FRAMEWORK_VERSION => CI::Contrib::Minitest::Integration.version.to_s
+                },
+                service: datadog_configuration[:service_name]
+              )
+              CI.start_test_module(test_session.name) if test_session
+            end
+
+            private
+
+            def datadog_configuration
+              Datadog.configuration.ci[:minitest]
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/contrib/rspec/configuration/settings.rb

@@ -2,6 +2,7 @@
 
 require_relative "../ext"
 require_relative "../../settings"
+require_relative "../../../utils/configuration"
 
 module Datadog
   module CI
@@ -19,7 +20,9 @@ module Datadog
 
             option :service_name do |o|
               o.type :string
-              o.default { Datadog.configuration.service_without_fallback || Ext::DEFAULT_SERVICE_NAME }
+              o.default do
+                Utils::Configuration.fetch_service_name(Ext::DEFAULT_SERVICE_NAME)
+              end
             end
 
             # @deprecated Will be removed in 1.0

data/lib/datadog/ci/contrib/rspec/example.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require_relative "../../ext/test"
+require_relative "../../utils/git"
 require_relative "ext"
 
 module Datadog
@@ -13,10 +14,9 @@ module Datadog
             base.prepend(InstanceMethods)
           end
 
-          # Instance methods for configuration
           module InstanceMethods
-            def run(example_group_instance, reporter)
-              return super unless configuration[:enabled]
+            def run(*)
+              return super unless datadog_configuration[:enabled]
 
               test_name = full_description.strip
               if metadata[:description].empty?
@@ -24,36 +24,77 @@ module Datadog
                 test_name += " #{description}"
               end
 
+              test_suite_description = fetch_top_level_example_group[:description]
+              suite_name = "#{test_suite_description} at #{metadata[:example_group][:rerun_file_path]}"
+
+              # remove example group description from test name to avoid duplication
+              test_name = test_name.sub(test_suite_description, "").strip
+
+              if ci_queue?
+                suite_name += " (ci-queue running example [#{test_name}])"
+                test_suite_span = CI.start_test_suite(suite_name)
+              end
+
               CI.trace_test(
                 test_name,
-                metadata[:example_group][:rerun_file_path],
+                suite_name,
                 tags: {
                   CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK,
                   CI::Ext::Test::TAG_FRAMEWORK_VERSION => CI::Contrib::RSpec::Integration.version.to_s,
-                  CI::Ext::Test::TAG_TYPE => CI::Ext::Test::TEST_TYPE
+                  CI::Ext::Test::TAG_SOURCE_FILE => Utils::Git.relative_to_root(metadata[:file_path]),
+                  CI::Ext::Test::TAG_SOURCE_START => metadata[:line_number].to_s
                 },
-                service: configuration[:service_name]
+                service: datadog_configuration[:service_name]
               ) do |test_span|
                 result = super
 
-                case execution_result.status
-                when :passed
-                  test_span.passed!
-                when :failed
-                  test_span.failed!(exception: execution_result.exception)
-                else
-                  test_span.skipped!(exception: execution_result.exception) if execution_result.example_skipped?
+                if test_span
+                  test_span.set_parameters({}, {"scoped_id" => metadata[:scoped_id]})
+
+                  case execution_result.status
+                  when :passed
+                    test_span.passed!
+                    test_suite_span.passed! if test_suite_span
+                  when :failed
+                    test_span.failed!(exception: execution_result.exception)
+                    test_suite_span.failed! if test_suite_span
+                  else
+                    # :pending or nil
+                    test_span.skipped!(
+                      reason: execution_result.pending_message,
+                      exception: execution_result.pending_exception
+                    )
+
+                    test_suite_span.skipped! if test_suite_span
+                  end
                 end
 
+                test_suite_span.finish if test_suite_span
+
                 result
               end
             end
 
             private
 
-            def configuration
+            def fetch_top_level_example_group
+              return metadata[:example_group] unless metadata[:example_group][:parent_example_group]
+
+              res = metadata[:example_group][:parent_example_group]
+              while (parent = res[:parent_example_group])
+                res = parent
+              end
+              res
+            end
+
+            def datadog_configuration
               Datadog.configuration.ci[:rspec]
             end
+
+            def ci_queue?
+              defined?(::RSpec::Queue::ExampleExtension) &&
+                self.class.ancestors.include?(::RSpec::Queue::ExampleExtension)
+            end
           end
         end
       end

data/lib/datadog/ci/contrib/rspec/example_group.rb

@@ -7,7 +7,7 @@ module Datadog
   module CI
     module Contrib
       module RSpec
-        # Instrument RSpec::Core::Example
+        # Instrument RSpec::Core::ExampleGroup
         module ExampleGroup
           def self.included(base)
             base.singleton_class.prepend(ClassMethods)
@@ -16,26 +16,31 @@ module Datadog
           # Instance methods for configuration
           module ClassMethods
             def run(reporter = ::RSpec::Core::NullReporter)
-              return super unless configuration[:enabled]
+              return super unless datadog_configuration[:enabled]
               return super unless top_level?
 
-              test_suite = Datadog::CI.start_test_suite(file_path)
+              suite_name = "#{description} at #{file_path}"
+              test_suite = Datadog::CI.start_test_suite(suite_name)
 
-              result = super
+              success = super
+              return success unless test_suite
 
-              if result
+              if success && test_suite.passed_tests_count > 0
                 test_suite.passed!
+              elsif success
+                test_suite.skipped!
               else
                 test_suite.failed!
               end
+
               test_suite.finish
 
-              result
+              success
             end
 
             private
 
-            def configuration
+            def datadog_configuration
               Datadog.configuration.ci[:rspec]
             end
           end

data/lib/datadog/ci/contrib/rspec/patcher.rb

@@ -21,10 +21,19 @@ module Datadog
           end
 
           def patch
-            ::RSpec::Core::Example.include(Example)
+            if ci_queue?
+              ::RSpec::Queue::Runner.include(Runner)
+            end
+
             ::RSpec::Core::Runner.include(Runner)
+            ::RSpec::Core::Example.include(Example)
             ::RSpec::Core::ExampleGroup.include(ExampleGroup)
           end
+
+          def ci_queue?
+            # ::RSpec::Queue::Runner is a ci-queue runner
+            defined?(::RSpec::Queue::Runner)
+          end
         end
       end
     end

data/lib/datadog/ci/contrib/rspec/runner.rb

@@ -14,21 +14,21 @@ module Datadog
           end
 
           module InstanceMethods
-            def run_specs(example_groups)
-              return super unless configuration[:enabled]
+            def run_specs(*)
+              return super unless datadog_configuration[:enabled]
 
               test_session = CI.start_test_session(
                 tags: {
                   CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK,
-                  CI::Ext::Test::TAG_FRAMEWORK_VERSION => CI::Contrib::RSpec::Integration.version.to_s,
-                  CI::Ext::Test::TAG_TYPE => CI::Ext::Test::TEST_TYPE
+                  CI::Ext::Test::TAG_FRAMEWORK_VERSION => CI::Contrib::RSpec::Integration.version.to_s
                 },
-                service: configuration[:service_name]
+                service: datadog_configuration[:service_name]
               )
 
-              test_module = CI.start_test_module(test_session.name)
+              test_module = CI.start_test_module(test_session.name) if test_session
 
               result = super
+              return result unless test_module && test_session
 
               if result != 0
                 # TODO: repeating this twice feels clunky, we need to remove test_module API before GA
@@ -46,7 +46,7 @@ module Datadog
 
             private
 
-            def configuration
+            def datadog_configuration
               Datadog.configuration.ci[:rspec]
             end
           end

data/lib/datadog/ci/ext/app_types.rb

@@ -3,6 +3,8 @@
 module Datadog
   module CI
     module Ext
+      # Defines span types for CI visibility
+      # @public_api
       module AppTypes
         TYPE_TEST = "test"
         TYPE_TEST_SESSION = "test_session_end"

data/lib/datadog/ci/ext/environment/providers/local_git.rb

@@ -1,9 +1,7 @@
 # frozen_string_literal: true
 
-require "open3"
-
 require_relative "base"
-require_relative "../../git"
+require_relative "../../../utils/git"
 
 module Datadog
   module CI
@@ -13,7 +11,7 @@ module Datadog
           # As a fallback we try to fetch git information from the local git repository
           class LocalGit < Base
             def git_repository_url
-              exec_git_command("git ls-remote --get-url")
+              Utils::Git.exec_git_command("git ls-remote --get-url")
             rescue => e
               Datadog.logger.debug(
                 "Unable to read git repository url: #{e.class.name} #{e.message} at #{Array(e.backtrace).first}"
@@ -22,7 +20,7 @@ module Datadog
             end
 
             def git_commit_sha
-              exec_git_command("git rev-parse HEAD")
+              Utils::Git.exec_git_command("git rev-parse HEAD")
             rescue => e
               Datadog.logger.debug(
                 "Unable to read git commit SHA: #{e.class.name} #{e.message} at #{Array(e.backtrace).first}"
@@ -31,7 +29,7 @@ module Datadog
             end
 
             def git_branch
-              exec_git_command("git rev-parse --abbrev-ref HEAD")
+              Utils::Git.exec_git_command("git rev-parse --abbrev-ref HEAD")
             rescue => e
               Datadog.logger.debug(
                 "Unable to read git branch: #{e.class.name} #{e.message} at #{Array(e.backtrace).first}"
@@ -40,7 +38,7 @@ module Datadog
             end
 
             def git_tag
-              exec_git_command("git tag --points-at HEAD")
+              Utils::Git.exec_git_command("git tag --points-at HEAD")
             rescue => e
               Datadog.logger.debug(
                 "Unable to read git tag: #{e.class.name} #{e.message} at #{Array(e.backtrace).first}"
@@ -49,7 +47,7 @@ module Datadog
             end
 
             def git_commit_message
-              exec_git_command("git show -s --format=%s")
+              Utils::Git.exec_git_command("git show -s --format=%s")
             rescue => e
               Datadog.logger.debug(
                 "Unable to read git commit message: #{e.class.name} #{e.message} at #{Array(e.backtrace).first}"
@@ -82,7 +80,7 @@ module Datadog
             end
 
             def workspace_path
-              exec_git_command("git rev-parse --show-toplevel")
+              Utils::Git.exec_git_command("git rev-parse --show-toplevel")
             rescue => e
               Datadog.logger.debug(
                 "Unable to read git base directory: #{e.class.name} #{e.message} at #{Array(e.backtrace).first}"
@@ -92,25 +90,6 @@ module Datadog
 
             private
 
-            def exec_git_command(cmd)
-              out, status = Open3.capture2e(cmd)
-
-              raise "Failed to run git command #{cmd}: #{out}" unless status.success?
-
-              # Sometimes Encoding.default_external is somehow set to US-ASCII which breaks
-              # commit messages with UTF-8 characters like emojis
-              # We force output's encoding to be UTF-8 in this case
-              # This is safe to do as UTF-8 is compatible with US-ASCII
-              if Encoding.default_external == Encoding::US_ASCII
-                out = out.force_encoding(Encoding::UTF_8)
-              end
-              out.strip! # There's always a "\n" at the end of the command output
-
-              return nil if out.empty?
-
-              out
-            end
-
             def author
               return @author if defined?(@author)
 
@@ -127,7 +106,7 @@ module Datadog
 
             def set_git_commit_users
               # Get committer and author information in one command.
-              output = exec_git_command("git show -s --format='%an\t%ae\t%at\t%cn\t%ce\t%ct'")
+              output = Utils::Git.exec_git_command("git show -s --format='%an\t%ae\t%at\t%cn\t%ce\t%ct'")
               unless output
                 Datadog.logger.debug(
                   "Unable to read git commit users: git command output is nil"

data/lib/datadog/ci/ext/settings.rb

@@ -9,6 +9,7 @@ module Datadog
         ENV_AGENTLESS_MODE_ENABLED = "DD_CIVISIBILITY_AGENTLESS_ENABLED"
         ENV_AGENTLESS_URL = "DD_CIVISIBILITY_AGENTLESS_URL"
         ENV_EXPERIMENTAL_TEST_SUITE_LEVEL_VISIBILITY_ENABLED = "DD_CIVISIBILITY_EXPERIMENTAL_TEST_SUITE_LEVEL_VISIBILITY_ENABLED"
+        ENV_FORCE_TEST_LEVEL_VISIBILITY = "DD_CIVISIBILITY_FORCE_TEST_LEVEL_VISIBILITY"
 
         # Source: https://docs.datadoghq.com/getting_started/site/
         DD_SITE_ALLOWLIST = [

data/lib/datadog/ci/ext/test.rb

@@ -12,24 +12,26 @@ module Datadog
         TAG_FRAMEWORK = "test.framework"
         TAG_FRAMEWORK_VERSION = "test.framework_version"
         TAG_NAME = "test.name"
-        TAG_SKIP_REASON = "test.skip_reason" # DEV: Not populated yet
+        TAG_SKIP_REASON = "test.skip_reason"
         TAG_STATUS = "test.status"
         TAG_SUITE = "test.suite"
         TAG_MODULE = "test.module"
-        TAG_TRAITS = "test.traits"
         TAG_TYPE = "test.type"
         TAG_COMMAND = "test.command"
+        TAG_SOURCE_FILE = "test.source.file"
+        TAG_SOURCE_START = "test.source.start"
+        TAG_CODEOWNERS = "test.codeowners"
+        TAG_PARAMETERS = "test.parameters"
 
-        TEST_TYPE = "test"
-
-        # those tags are special and they are used to correlate tests with the test sessions, suites, and modules
+        # those tags are special and used to correlate tests with the test sessions, suites, and modules
+        # they are transient and not sent to the backend
         TAG_TEST_SESSION_ID = "_test.session_id"
         TAG_TEST_MODULE_ID = "_test.module_id"
         TAG_TEST_SUITE_ID = "_test.suite_id"
-        SPECIAL_TAGS = [TAG_TEST_SESSION_ID, TAG_TEST_MODULE_ID, TAG_TEST_SUITE_ID].freeze
+        TRANSIENT_TAGS = [TAG_TEST_SESSION_ID, TAG_TEST_MODULE_ID, TAG_TEST_SUITE_ID].freeze
 
-        # tags that can be inherited from the test session
-        INHERITABLE_TAGS = [TAG_FRAMEWORK, TAG_FRAMEWORK_VERSION, TAG_TYPE].freeze
+        # tags that are common for the whole session and can be inherited from the test session
+        INHERITABLE_TAGS = [TAG_FRAMEWORK, TAG_FRAMEWORK_VERSION].freeze
 
         # Environment runtime tags
         TAG_OS_ARCHITECTURE = "os.architecture"
@@ -37,13 +39,22 @@ module Datadog
         TAG_RUNTIME_NAME = "runtime.name"
         TAG_RUNTIME_VERSION = "runtime.version"
 
+        # internal APM tag to mark a span as a test span
         TAG_SPAN_KIND = "span.kind"
+        SPAN_KIND_TEST = "test"
 
+        # test status as recognized by Datadog
         module Status
           PASS = "pass"
           FAIL = "fail"
           SKIP = "skip"
         end
+
+        # test types (e.g. test, benchmark, browser)
+        module Type
+          TEST = "test"
+          BENCHMARK = "benchmark" # DEV: not used yet, will be used when benchmarks are supported
+        end
       end
     end
   end

data/lib/datadog/ci/span.rb

@@ -32,7 +32,7 @@ module Datadog
       end
 
       # @return [String] the type of the span (for example "test" or type that was provided to [Datadog::CI.trace]).
-      def span_type
+      def type
         tracer_span.type
       end
 
@@ -130,12 +130,19 @@ module Datadog
       end
 
       def set_default_tags
-        tracer_span.set_tag(Ext::Test::TAG_SPAN_KIND, Ext::AppTypes::TYPE_TEST)
+        tracer_span.set_tag(Ext::Test::TAG_SPAN_KIND, Ext::Test::SPAN_KIND_TEST)
       end
 
       def to_s
         "#{self.class}(name:#{name},tracer_span:#{@tracer_span})"
       end
+
+      private
+
+      # provides access to global CI recorder for CI models to deactivate themselves
+      def recorder
+        Datadog.send(:components).ci_recorder
+      end
     end
   end
 end

data/lib/datadog/ci/test.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "json"
+
 require_relative "span"
 
 module Datadog
@@ -18,7 +20,7 @@ module Datadog
       def finish
         super
 
-        CI.deactivate_test(self)
+        recorder.deactivate_test
       end
 
       # Running test suite that this test is part of (if any).
@@ -35,6 +37,8 @@ module Datadog
         get_tag(Ext::Test::TAG_TEST_SUITE_ID)
       end
 
+      # Name of the running test suite this test belongs to.
+      # @return [String] the name of the test suite.
       def test_suite_name
         get_tag(Ext::Test::TAG_SUITE)
       end
@@ -45,11 +49,72 @@ module Datadog
         get_tag(Ext::Test::TAG_TEST_MODULE_ID)
       end
 
-      # Span id of the running test module this test belongs to.
+      # Span id of the running test session this test belongs to.
       # @return [String] the span id of the test session.
       def test_session_id
         get_tag(Ext::Test::TAG_TEST_SESSION_ID)
       end
+
+      # Source file path of the test relative to git repository root.
+      # @return [String] the source file path of the test
+      # @return [nil] if the source file path is not found
+      def source_file
+        get_tag(Ext::Test::TAG_SOURCE_FILE)
+      end
+
+      # Sets the status of the span to "pass".
+      # @return [void]
+      def passed!
+        super
+
+        record_test_result(Ext::Test::Status::PASS)
+      end
+
+      # Sets the status of the span to "fail".
+      # @param [Exception] exception the exception that caused the test to fail.
+      # @return [void]
+      def failed!(exception: nil)
+        super
+
+        record_test_result(Ext::Test::Status::FAIL)
+      end
+
+      # Sets the status of the span to "skip".
+      # @param [Exception] exception the exception that caused the test to fail.
+      # @param [String] reason the reason why the test was skipped.
+      # @return [void]
+      def skipped!(exception: nil, reason: nil)
+        super
+
+        record_test_result(Ext::Test::Status::SKIP)
+      end
+
+      # Sets the parameters for this test (e.g. Cucumber example or RSpec shared specs).
+      # Parameters are needed to compute test fingerprint to distinguish between different tests having same names.
+      #
+      # @param [Hash] arguments the arguments that test accepts as key-value hash
+      # @param [Hash] metadata optional metadata
+      # @return [void]
+      def set_parameters(arguments, metadata = {})
+        return if arguments.nil?
+
+        set_tag(
+          Ext::Test::TAG_PARAMETERS,
+          JSON.generate(
+            {
+              arguments: arguments,
+              metadata: metadata
+            }
+          )
+        )
+      end
+
+      private
+
+      def record_test_result(datadog_status)
+        suite = test_suite
+        suite.record_test_result(datadog_status) if suite
+      end
     end
   end
 end

data/lib/datadog/ci/test_module.rb

@@ -16,7 +16,7 @@ module Datadog
       def finish
         super
 
-        CI.deactivate_test_module
+        recorder.deactivate_test_module
       end
     end
   end

data/lib/datadog/ci/test_session.rb

@@ -17,7 +17,7 @@ module Datadog
       def finish
         super
 
-        CI.deactivate_test_session
+        recorder.deactivate_test_session
       end
 
       # Return the test session's name which is equal to test command used
@@ -32,7 +32,7 @@ module Datadog
         return @inheritable_tags if defined?(@inheritable_tags)
 
         # this method is not synchronized because it does not iterate over the tags collection, but rather
-        # uses synchronized method to get each tag value
+        # uses synchronized method #get_tag to get each tag value
         res = {}
         Ext::Test::INHERITABLE_TAGS.each do |tag|
           res[tag] = get_tag(tag)