datadog-ci 0.5.1 → 0.7.0

data/lib/datadog/ci/ext/environment/providers/local_git.rb

@@ -1,9 +1,7 @@
 # frozen_string_literal: true
 
-require "open3"
-
 require_relative "base"
-require_relative "../../git"
+require_relative "../../../utils/git"
 
 module Datadog
   module CI
@@ -13,7 +11,7 @@ module Datadog
           # As a fallback we try to fetch git information from the local git repository
           class LocalGit < Base
             def git_repository_url
-              exec_git_command("git ls-remote --get-url")
+              Utils::Git.exec_git_command("git ls-remote --get-url")
             rescue => e
               Datadog.logger.debug(
                 "Unable to read git repository url: #{e.class.name} #{e.message} at #{Array(e.backtrace).first}"
@@ -22,7 +20,7 @@ module Datadog
             end
 
             def git_commit_sha
-              exec_git_command("git rev-parse HEAD")
+              Utils::Git.exec_git_command("git rev-parse HEAD")
             rescue => e
               Datadog.logger.debug(
                 "Unable to read git commit SHA: #{e.class.name} #{e.message} at #{Array(e.backtrace).first}"
@@ -31,7 +29,7 @@ module Datadog
             end
 
             def git_branch
-              exec_git_command("git rev-parse --abbrev-ref HEAD")
+              Utils::Git.exec_git_command("git rev-parse --abbrev-ref HEAD")
             rescue => e
               Datadog.logger.debug(
                 "Unable to read git branch: #{e.class.name} #{e.message} at #{Array(e.backtrace).first}"
@@ -40,7 +38,7 @@ module Datadog
             end
 
             def git_tag
-              exec_git_command("git tag --points-at HEAD")
+              Utils::Git.exec_git_command("git tag --points-at HEAD")
             rescue => e
               Datadog.logger.debug(
                 "Unable to read git tag: #{e.class.name} #{e.message} at #{Array(e.backtrace).first}"
@@ -49,7 +47,7 @@ module Datadog
             end
 
             def git_commit_message
-              exec_git_command("git show -s --format=%s")
+              Utils::Git.exec_git_command("git show -s --format=%s")
             rescue => e
               Datadog.logger.debug(
                 "Unable to read git commit message: #{e.class.name} #{e.message} at #{Array(e.backtrace).first}"
@@ -82,7 +80,7 @@ module Datadog
             end
 
             def workspace_path
-              exec_git_command("git rev-parse --show-toplevel")
+              Utils::Git.exec_git_command("git rev-parse --show-toplevel")
             rescue => e
               Datadog.logger.debug(
                 "Unable to read git base directory: #{e.class.name} #{e.message} at #{Array(e.backtrace).first}"
@@ -92,25 +90,6 @@ module Datadog
 
             private
 
-            def exec_git_command(cmd)
-              out, status = Open3.capture2e(cmd)
-
-              raise "Failed to run git command #{cmd}: #{out}" unless status.success?
-
-              # Sometimes Encoding.default_external is somehow set to US-ASCII which breaks
-              # commit messages with UTF-8 characters like emojis
-              # We force output's encoding to be UTF-8 in this case
-              # This is safe to do as UTF-8 is compatible with US-ASCII
-              if Encoding.default_external == Encoding::US_ASCII
-                out = out.force_encoding(Encoding::UTF_8)
-              end
-              out.strip! # There's always a "\n" at the end of the command output
-
-              return nil if out.empty?
-
-              out
-            end
-
             def author
               return @author if defined?(@author)
 
@@ -127,7 +106,7 @@ module Datadog
 
             def set_git_commit_users
               # Get committer and author information in one command.
-              output = exec_git_command("git show -s --format='%an\t%ae\t%at\t%cn\t%ce\t%ct'")
+              output = Utils::Git.exec_git_command("git show -s --format='%an\t%ae\t%at\t%cn\t%ce\t%ct'")
               unless output
                 Datadog.logger.debug(
                   "Unable to read git commit users: git command output is nil"

data/lib/datadog/ci/ext/settings.rb

@@ -9,6 +9,7 @@ module Datadog
         ENV_AGENTLESS_MODE_ENABLED = "DD_CIVISIBILITY_AGENTLESS_ENABLED"
         ENV_AGENTLESS_URL = "DD_CIVISIBILITY_AGENTLESS_URL"
         ENV_EXPERIMENTAL_TEST_SUITE_LEVEL_VISIBILITY_ENABLED = "DD_CIVISIBILITY_EXPERIMENTAL_TEST_SUITE_LEVEL_VISIBILITY_ENABLED"
+        ENV_FORCE_TEST_LEVEL_VISIBILITY = "DD_CIVISIBILITY_FORCE_TEST_LEVEL_VISIBILITY"
 
         # Source: https://docs.datadoghq.com/getting_started/site/
         DD_SITE_ALLOWLIST = [

data/lib/datadog/ci/ext/test.rb

@@ -12,22 +12,26 @@ module Datadog
         TAG_FRAMEWORK = "test.framework"
         TAG_FRAMEWORK_VERSION = "test.framework_version"
         TAG_NAME = "test.name"
-        TAG_SKIP_REASON = "test.skip_reason" # DEV: Not populated yet
+        TAG_SKIP_REASON = "test.skip_reason"
         TAG_STATUS = "test.status"
         TAG_SUITE = "test.suite"
         TAG_MODULE = "test.module"
-        TAG_TRAITS = "test.traits"
         TAG_TYPE = "test.type"
         TAG_COMMAND = "test.command"
+        TAG_SOURCE_FILE = "test.source.file"
+        TAG_SOURCE_START = "test.source.start"
+        TAG_CODEOWNERS = "test.codeowners"
+        TAG_PARAMETERS = "test.parameters"
 
-        # those tags are special and they are used to correlate tests with the test sessions, suites, and modules
+        # those tags are special and used to correlate tests with the test sessions, suites, and modules
+        # they are transient and not sent to the backend
         TAG_TEST_SESSION_ID = "_test.session_id"
         TAG_TEST_MODULE_ID = "_test.module_id"
         TAG_TEST_SUITE_ID = "_test.suite_id"
-        SPECIAL_TAGS = [TAG_TEST_SESSION_ID, TAG_TEST_MODULE_ID, TAG_TEST_SUITE_ID].freeze
+        TRANSIENT_TAGS = [TAG_TEST_SESSION_ID, TAG_TEST_MODULE_ID, TAG_TEST_SUITE_ID].freeze
 
-        # tags that can be inherited from the test session
-        INHERITABLE_TAGS = [TAG_FRAMEWORK, TAG_FRAMEWORK_VERSION, TAG_TYPE].freeze
+        # tags that are common for the whole session and can be inherited from the test session
+        INHERITABLE_TAGS = [TAG_FRAMEWORK, TAG_FRAMEWORK_VERSION].freeze
 
         # Environment runtime tags
         TAG_OS_ARCHITECTURE = "os.architecture"
@@ -35,13 +39,22 @@ module Datadog
         TAG_RUNTIME_NAME = "runtime.name"
         TAG_RUNTIME_VERSION = "runtime.version"
 
+        # internal APM tag to mark a span as a test span
         TAG_SPAN_KIND = "span.kind"
+        SPAN_KIND_TEST = "test"
 
+        # test status as recognized by Datadog
         module Status
           PASS = "pass"
           FAIL = "fail"
           SKIP = "skip"
         end
+
+        # test types (e.g. test, benchmark, browser)
+        module Type
+          TEST = "test"
+          BENCHMARK = "benchmark" # DEV: not used yet, will be used when benchmarks are supported
+        end
       end
     end
   end

data/lib/datadog/ci/span.rb

@@ -32,10 +32,34 @@ module Datadog
       end
 
       # @return [String] the type of the span (for example "test" or type that was provided to [Datadog::CI.trace]).
-      def span_type
+      def type
         tracer_span.type
       end
 
+      # Checks whether span status is not set yet.
+      # @return [bool] true if span does not have status
+      def undefined?
+        tracer_span.get_tag(Ext::Test::TAG_STATUS).nil?
+      end
+
+      # Checks whether span status is PASS.
+      # @return [bool] true if span status is PASS
+      def passed?
+        tracer_span.get_tag(Ext::Test::TAG_STATUS) == Ext::Test::Status::PASS
+      end
+
+      # Checks whether span status is FAIL.
+      # @return [bool] true if span status is FAIL
+      def failed?
+        tracer_span.get_tag(Ext::Test::TAG_STATUS) == Ext::Test::Status::FAIL
+      end
+
+      # Checks whether span status is SKIP.
+      # @return [bool] true if span status is SKIP
+      def skipped?
+        tracer_span.get_tag(Ext::Test::TAG_STATUS) == Ext::Test::Status::SKIP
+      end
+
       # Sets the status of the span to "pass".
       # @return [void]
       def passed!
@@ -106,12 +130,19 @@ module Datadog
       end
 
       def set_default_tags
-        tracer_span.set_tag(Ext::Test::TAG_SPAN_KIND, Ext::AppTypes::TYPE_TEST)
+        tracer_span.set_tag(Ext::Test::TAG_SPAN_KIND, Ext::Test::SPAN_KIND_TEST)
       end
 
       def to_s
         "#{self.class}(name:#{name},tracer_span:#{@tracer_span})"
       end
+
+      private
+
+      # provides access to global CI recorder for CI models to deactivate themselves
+      def recorder
+        Datadog.send(:components).ci_recorder
+      end
     end
   end
 end

data/lib/datadog/ci/test.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "json"
+
 require_relative "span"
 
 module Datadog
@@ -18,7 +20,100 @@ module Datadog
       def finish
         super
 
-        CI.deactivate_test(self)
+        recorder.deactivate_test
+      end
+
+      # Running test suite that this test is part of (if any).
+      # @return [Datadog::CI::TestSuite] the test suite this test belongs to
+      # @return [nil] if the test suite is not found
+      def test_suite
+        suite_name = test_suite_name
+        CI.active_test_suite(suite_name) if suite_name
+      end
+
+      # Span id of the running test suite this test belongs to.
+      # @return [String] the span id of the test suite.
+      def test_suite_id
+        get_tag(Ext::Test::TAG_TEST_SUITE_ID)
+      end
+
+      # Name of the running test suite this test belongs to.
+      # @return [String] the name of the test suite.
+      def test_suite_name
+        get_tag(Ext::Test::TAG_SUITE)
+      end
+
+      # Span id of the running test module this test belongs to.
+      # @return [String] the span id of the test module.
+      def test_module_id
+        get_tag(Ext::Test::TAG_TEST_MODULE_ID)
+      end
+
+      # Span id of the running test session this test belongs to.
+      # @return [String] the span id of the test session.
+      def test_session_id
+        get_tag(Ext::Test::TAG_TEST_SESSION_ID)
+      end
+
+      # Source file path of the test relative to git repository root.
+      # @return [String] the source file path of the test
+      # @return [nil] if the source file path is not found
+      def source_file
+        get_tag(Ext::Test::TAG_SOURCE_FILE)
+      end
+
+      # Sets the status of the span to "pass".
+      # @return [void]
+      def passed!
+        super
+
+        record_test_result(Ext::Test::Status::PASS)
+      end
+
+      # Sets the status of the span to "fail".
+      # @param [Exception] exception the exception that caused the test to fail.
+      # @return [void]
+      def failed!(exception: nil)
+        super
+
+        record_test_result(Ext::Test::Status::FAIL)
+      end
+
+      # Sets the status of the span to "skip".
+      # @param [Exception] exception the exception that caused the test to fail.
+      # @param [String] reason the reason why the test was skipped.
+      # @return [void]
+      def skipped!(exception: nil, reason: nil)
+        super
+
+        record_test_result(Ext::Test::Status::SKIP)
+      end
+
+      # Sets the parameters for this test (e.g. Cucumber example or RSpec shared specs).
+      # Parameters are needed to compute test fingerprint to distinguish between different tests having same names.
+      #
+      # @param [Hash] arguments the arguments that test accepts as key-value hash
+      # @param [Hash] metadata optional metadata
+      # @return [void]
+      def set_parameters(arguments, metadata = {})
+        return if arguments.nil?
+
+        set_tag(
+          Ext::Test::TAG_PARAMETERS,
+          JSON.generate(
+            {
+              arguments: arguments,
+              metadata: metadata
+            }
+          )
+        )
+      end
+
+      private
+
+      def record_test_result(datadog_status)
+        suite = test_suite
+        suite.record_test_result(datadog_status) if suite
       end
     end
   end

data/lib/datadog/ci/test_module.rb

@@ -16,7 +16,7 @@ module Datadog
       def finish
         super
 
-        CI.deactivate_test_module
+        recorder.deactivate_test_module
       end
     end
   end

data/lib/datadog/ci/test_session.rb

@@ -17,14 +17,22 @@ module Datadog
       def finish
         super
 
-        CI.deactivate_test_session
+        recorder.deactivate_test_session
       end
 
+      # Return the test session's name which is equal to test command used
+      # @return [String] the command for this test session.
+      def name
+        get_tag(Ext::Test::TAG_COMMAND)
+      end
+
+      # Return the test session tags that could be inherited by sub-spans
+      # @return [Hash] the tags to be inherited by sub-spans.
       def inheritable_tags
         return @inheritable_tags if defined?(@inheritable_tags)
 
         # this method is not synchronized because it does not iterate over the tags collection, but rather
-        # uses synchronized method to get each tag value
+        # uses synchronized method #get_tag to get each tag value
         res = {}
         Ext::Test::INHERITABLE_TAGS.each do |tag|
           res[tag] = get_tag(tag)

data/lib/datadog/ci/test_suite.rb

@@ -13,12 +13,63 @@ module Datadog
     #
     # @public_api
     class TestSuite < ConcurrentSpan
+      def initialize(tracer_span)
+        super
+
+        @test_suite_stats = Hash.new(0)
+      end
+
       # Finishes this test suite.
       # @return [void]
       def finish
-        super
+        synchronize do
+          # we try to derive test suite status from execution stats if no status was set explicitly
+          set_status_from_stats! if undefined?
+
+          super
+
+          recorder.deactivate_test_suite(name)
+        end
+      end
+
+      # @internal
+      def record_test_result(datadog_test_status)
+        synchronize do
+          @test_suite_stats[datadog_test_status] += 1
+        end
+      end
+
+      # @internal
+      def passed_tests_count
+        synchronize do
+          @test_suite_stats[Ext::Test::Status::PASS]
+        end
+      end
+
+      # @internal
+      def skipped_tests_count
+        synchronize do
+          @test_suite_stats[Ext::Test::Status::SKIP]
+        end
+      end
+
+      # @internal
+      def failed_tests_count
+        synchronize do
+          @test_suite_stats[Ext::Test::Status::FAIL]
+        end
+      end
+
+      private
 
-        CI.deactivate_test_suite(name)
+      def set_status_from_stats!
+        if failed_tests_count > 0
+          failed!
+        elsif passed_tests_count == 0
+          skipped!
+        else
+          passed!
+        end
       end
     end
   end

data/lib/datadog/ci/test_visibility/context/local.rb

@@ -11,7 +11,7 @@ module Datadog
             self.active_test = nil
           end
 
-          def activate_test!(test)
+          def activate_test(test)
             raise "Nested tests are not supported. Currently active test: #{active_test}" unless active_test.nil?
 
             if block_given?
@@ -26,14 +26,8 @@ module Datadog
             end
           end
 
-          def deactivate_test!(test)
-            return if active_test.nil?
-
-            if active_test == test
-              self.active_test = nil
-            else
-              raise "Trying to deactivate test #{test}, but currently active test is #{active_test}"
-            end
+          def deactivate_test
+            self.active_test = nil
           end
 
           def active_test

data/lib/datadog/ci/test_visibility/null_recorder.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+require_relative "recorder"
+
+module Datadog
+  module CI
+    module TestVisibility
+      # Special recorder that does not record anything
+      class NullRecorder
+        def start_test_session(service: nil, tags: {})
+          skip_tracing
+        end
+
+        def start_test_module(test_module_name, service: nil, tags: {})
+          skip_tracing
+        end
+
+        def start_test_suite(test_suite_name, service: nil, tags: {})
+          skip_tracing
+        end
+
+        def trace_test(test_name, test_suite_name, service: nil, tags: {}, &block)
+          skip_tracing(block)
+        end
+
+        def trace(type, span_name, tags: {}, &block)
+          skip_tracing(block)
+        end
+
+        def active_span
+        end
+
+        def active_test
+        end
+
+        def active_test_session
+        end
+
+        def active_test_module
+        end
+
+        def active_test_suite(test_suite_name)
+        end
+
+        private
+
+        def skip_tracing(block = nil)
+          block.call(nil) if block
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/test_visibility/recorder.rb

@@ -8,12 +8,13 @@ require "rbconfig"
 require_relative "context/global"
 require_relative "context/local"
 
+require_relative "../codeowners/parser"
 require_relative "../ext/app_types"
 require_relative "../ext/test"
 require_relative "../ext/environment"
+require_relative "../utils/git"
 
 require_relative "../span"
-require_relative "../null_span"
 require_relative "../test"
 require_relative "../test_session"
 require_relative "../test_module"
@@ -25,15 +26,18 @@ module Datadog
       # Common behavior for CI tests
       # Note: this class has too many responsibilities and should be split into multiple classes
       class Recorder
-        attr_reader :environment_tags, :test_suite_level_visibility_enabled, :enabled
+        attr_reader :environment_tags, :test_suite_level_visibility_enabled
 
-        def initialize(enabled: true, test_suite_level_visibility_enabled: false)
-          @enabled = enabled
-          @test_suite_level_visibility_enabled = enabled && test_suite_level_visibility_enabled
+        def initialize(
+          test_suite_level_visibility_enabled: false,
+          codeowners: Codeowners::Parser.new(Utils::Git.root).parse
+        )
+          @test_suite_level_visibility_enabled = test_suite_level_visibility_enabled
 
-          @environment_tags = @enabled ? Ext::Environment.tags(ENV).freeze : {}
+          @environment_tags = Ext::Environment.tags(ENV).freeze
           @local_context = Context::Local.new
           @global_context = Context::Global.new
+          @codeowners = codeowners
         end
 
         def start_test_session(service: nil, tags: {})
@@ -83,14 +87,13 @@ module Datadog
         end
 
         def trace_test(test_name, test_suite_name, service: nil, tags: {}, &block)
-          return skip_tracing(block) unless enabled
-
           set_inherited_globals(tags)
           set_session_context(tags)
           set_module_context(tags)
           set_suite_context(tags, name: test_suite_name)
 
           tags[Ext::Test::TAG_NAME] = test_name
+          tags[Ext::Test::TAG_TYPE] ||= Ext::Test::Type::TEST
 
           span_options = build_span_options(
             service,
@@ -104,7 +107,7 @@ module Datadog
             start_datadog_tracer_span(test_name, span_options) do |tracer_span|
               test = build_test(tracer_span, tags)
 
-              @local_context.activate_test!(test) do
+              @local_context.activate_test(test) do
                 block.call(test)
               end
             end
@@ -112,17 +115,15 @@ module Datadog
             tracer_span = start_datadog_tracer_span(test_name, span_options)
 
             test = build_test(tracer_span, tags)
-            @local_context.activate_test!(test)
+            @local_context.activate_test(test)
             test
           end
         end
 
-        def trace(span_type, span_name, tags: {}, &block)
-          return skip_tracing(block) unless enabled
-
+        def trace(span_name, type: "span", tags: {}, &block)
           span_options = build_span_options(
             nil, # service name is completely optional for custom spans
-            span_type,
+            type,
             {resource: span_name}
           )
 
@@ -158,8 +159,8 @@ module Datadog
           @global_context.active_test_suite(test_suite_name)
         end
 
-        def deactivate_test(test)
-          @local_context.deactivate_test!(test)
+        def deactivate_test
+          @local_context.deactivate_test
         end
 
         def deactivate_test_session
@@ -177,11 +178,7 @@ module Datadog
         private
 
         def skip_tracing(block = nil)
-          if block
-            block.call(null_span)
-          else
-            null_span
-          end
+          block.call(nil) if block
         end
 
         # Sets trace's origin to ciapp-test
@@ -210,6 +207,9 @@ module Datadog
         def build_test(tracer_span, tags)
           test = Test.new(tracer_span)
           set_initial_tags(test, tags)
+          validate_test_suite_level_visibility_correctness(test)
+          set_codeowners(test)
+
           test
         end
 
@@ -219,9 +219,9 @@ module Datadog
           span
         end
 
-        def build_span_options(service, span_type, other_options = {})
+        def build_span_options(service, type, other_options = {})
           other_options[:service] = service || @global_context.service
-          other_options[:span_type] = span_type
+          other_options[:type] = type
 
           other_options
         end
@@ -255,6 +255,12 @@ module Datadog
           end
         end
 
+        def set_codeowners(test)
+          source = test.source_file
+          owners = @codeowners.list_owners(source) if source
+          test.set_tag(Ext::Test::TAG_CODEOWNERS, owners) unless owners.nil?
+        end
+
         def set_suite_context(tags, span: nil, name: nil)
           return if span.nil? && name.nil?
 
@@ -284,8 +290,29 @@ module Datadog
           end
         end
 
-        def null_span
-          @null_span ||= NullSpan.new
+        def validate_test_suite_level_visibility_correctness(test)
+          return unless test_suite_level_visibility_enabled
+
+          if test.test_suite_id.nil?
+            Datadog.logger.debug do
+              "Test [#{test.name}] does not have a test suite associated with it. " \
+              "Expected test suite [#{test.test_suite_name}] to be running."
+            end
+          end
+
+          if test.test_module_id.nil?
+            Datadog.logger.debug do
+              "Test [#{test.name}] does not have a test module associated with it. " \
+              "Make sure that there is a test module running within a session."
+            end
+          end
+
+          if test.test_session_id.nil?
+            Datadog.logger.debug do
+              "Test [#{test.name}] does not have a test session associated with it. " \
+              "Make sure that there is a test session running."
+            end
+          end
         end
       end
     end