datadog-ci 0.5.0 → 0.6.0

data/lib/datadog/ci/test_visibility/recorder.rb

@@ -0,0 +1,314 @@
+# frozen_string_literal: true
+
+require "datadog/tracing"
+require "datadog/tracing/trace_digest"
+
+require "rbconfig"
+
+require_relative "context/global"
+require_relative "context/local"
+
+require_relative "../ext/app_types"
+require_relative "../ext/test"
+require_relative "../ext/environment"
+
+require_relative "../span"
+require_relative "../null_span"
+require_relative "../test"
+require_relative "../test_session"
+require_relative "../test_module"
+require_relative "../test_suite"
+
+module Datadog
+  module CI
+    module TestVisibility
+      # Common behavior for CI tests
+      # Note: this class has too many responsibilities and should be split into multiple classes
+      class Recorder
+        attr_reader :environment_tags, :test_suite_level_visibility_enabled
+
+        def initialize(test_suite_level_visibility_enabled: false)
+          @test_suite_level_visibility_enabled = test_suite_level_visibility_enabled
+
+          @environment_tags = Ext::Environment.tags(ENV).freeze
+          @local_context = Context::Local.new
+          @global_context = Context::Global.new
+        end
+
+        def start_test_session(service: nil, tags: {})
+          return skip_tracing unless test_suite_level_visibility_enabled
+
+          @global_context.fetch_or_activate_test_session do
+            tracer_span = start_datadog_tracer_span(
+              "test.session", build_span_options(service, Ext::AppTypes::TYPE_TEST_SESSION)
+            )
+            set_session_context(tags, tracer_span)
+
+            build_test_session(tracer_span, tags)
+          end
+        end
+
+        def start_test_module(test_module_name, service: nil, tags: {})
+          return skip_tracing unless test_suite_level_visibility_enabled
+
+          @global_context.fetch_or_activate_test_module do
+            set_inherited_globals(tags)
+            set_session_context(tags)
+
+            tracer_span = start_datadog_tracer_span(
+              test_module_name, build_span_options(service, Ext::AppTypes::TYPE_TEST_MODULE)
+            )
+            set_module_context(tags, tracer_span)
+
+            build_test_module(tracer_span, tags)
+          end
+        end
+
+        def start_test_suite(test_suite_name, service: nil, tags: {})
+          return skip_tracing unless test_suite_level_visibility_enabled
+
+          @global_context.fetch_or_activate_test_suite(test_suite_name) do
+            set_inherited_globals(tags)
+            set_session_context(tags)
+            set_module_context(tags)
+
+            tracer_span = start_datadog_tracer_span(
+              test_suite_name, build_span_options(service, Ext::AppTypes::TYPE_TEST_SUITE)
+            )
+            set_suite_context(tags, span: tracer_span)
+
+            build_test_suite(tracer_span, tags)
+          end
+        end
+
+        def trace_test(test_name, test_suite_name, service: nil, tags: {}, &block)
+          set_inherited_globals(tags)
+          set_session_context(tags)
+          set_module_context(tags)
+          set_suite_context(tags, name: test_suite_name)
+
+          tags[Ext::Test::TAG_NAME] = test_name
+
+          span_options = build_span_options(
+            service,
+            Ext::AppTypes::TYPE_TEST,
+            # :resource is needed for the agent APM protocol to work correctly (for older agent versions)
+            # :continue_from is required to start a new trace for each test
+            {resource: test_name, continue_from: Datadog::Tracing::TraceDigest.new}
+          )
+
+          if block
+            start_datadog_tracer_span(test_name, span_options) do |tracer_span|
+              test = build_test(tracer_span, tags)
+
+              @local_context.activate_test!(test) do
+                block.call(test)
+              end
+            end
+          else
+            tracer_span = start_datadog_tracer_span(test_name, span_options)
+
+            test = build_test(tracer_span, tags)
+            @local_context.activate_test!(test)
+            test
+          end
+        end
+
+        def trace(span_type, span_name, tags: {}, &block)
+          span_options = build_span_options(
+            nil, # service name is completely optional for custom spans
+            span_type,
+            {resource: span_name}
+          )
+
+          if block
+            start_datadog_tracer_span(span_name, span_options) do |tracer_span|
+              block.call(build_span(tracer_span, tags))
+            end
+          else
+            tracer_span = start_datadog_tracer_span(span_name, span_options)
+
+            build_span(tracer_span, tags)
+          end
+        end
+
+        def active_span
+          tracer_span = Datadog::Tracing.active_span
+          Span.new(tracer_span) if tracer_span
+        end
+
+        def active_test
+          @local_context.active_test
+        end
+
+        def active_test_session
+          @global_context.active_test_session
+        end
+
+        def active_test_module
+          @global_context.active_test_module
+        end
+
+        def active_test_suite(test_suite_name)
+          @global_context.active_test_suite(test_suite_name)
+        end
+
+        def deactivate_test(test)
+          @local_context.deactivate_test!(test)
+        end
+
+        def deactivate_test_session
+          @global_context.deactivate_test_session!
+        end
+
+        def deactivate_test_module
+          @global_context.deactivate_test_module!
+        end
+
+        def deactivate_test_suite(test_suite_name)
+          @global_context.deactivate_test_suite!(test_suite_name)
+        end
+
+        private
+
+        def skip_tracing(block = nil)
+          if block
+            block.call(null_span)
+          else
+            null_span
+          end
+        end
+
+        # Sets trace's origin to ciapp-test
+        def set_trace_origin(trace)
+          trace.origin = Ext::Test::CONTEXT_ORIGIN if trace
+        end
+
+        def build_test_session(tracer_span, tags)
+          test_session = TestSession.new(tracer_span)
+          set_initial_tags(test_session, tags)
+          test_session
+        end
+
+        def build_test_module(tracer_span, tags)
+          test_module = TestModule.new(tracer_span)
+          set_initial_tags(test_module, tags)
+          test_module
+        end
+
+        def build_test_suite(tracer_span, tags)
+          test_suite = TestSuite.new(tracer_span)
+          set_initial_tags(test_suite, tags)
+          test_suite
+        end
+
+        def build_test(tracer_span, tags)
+          test = Test.new(tracer_span)
+          set_initial_tags(test, tags)
+          validate_test_suite_level_visibility_correctness(test)
+          test
+        end
+
+        def build_span(tracer_span, tags)
+          span = Span.new(tracer_span)
+          set_initial_tags(span, tags)
+          span
+        end
+
+        def build_span_options(service, span_type, other_options = {})
+          other_options[:service] = service || @global_context.service
+          other_options[:span_type] = span_type
+
+          other_options
+        end
+
+        def set_inherited_globals(tags)
+          # this code achieves the same as @global_context.inheritable_session_tags.merge(tags)
+          # but without allocating a new hash
+          @global_context.inheritable_session_tags.each do |key, value|
+            tags[key] = value unless tags.key?(key)
+          end
+        end
+
+        def set_initial_tags(ci_span, tags)
+          ci_span.set_default_tags
+          ci_span.set_environment_runtime_tags
+
+          ci_span.set_tags(tags)
+          ci_span.set_tags(environment_tags)
+        end
+
+        def set_session_context(tags, test_session = nil)
+          test_session ||= active_test_session
+          tags[Ext::Test::TAG_TEST_SESSION_ID] = test_session.id.to_s if test_session
+        end
+
+        def set_module_context(tags, test_module = nil)
+          test_module ||= active_test_module
+          if test_module
+            tags[Ext::Test::TAG_TEST_MODULE_ID] = test_module.id.to_s
+            tags[Ext::Test::TAG_MODULE] = test_module.name
+          end
+        end
+
+        def set_suite_context(tags, span: nil, name: nil)
+          return if span.nil? && name.nil?
+
+          test_suite = span || active_test_suite(name)
+
+          if test_suite
+            tags[Ext::Test::TAG_TEST_SUITE_ID] = test_suite.id.to_s
+            tags[Ext::Test::TAG_SUITE] = test_suite.name
+          else
+            tags[Ext::Test::TAG_SUITE] = name
+          end
+        end
+
+        def start_datadog_tracer_span(span_name, span_options, &block)
+          if block
+            Datadog::Tracing.trace(span_name, **span_options) do |tracer_span, trace|
+              set_trace_origin(trace)
+
+              yield tracer_span
+            end
+          else
+            tracer_span = Datadog::Tracing.trace(span_name, **span_options)
+            trace = Datadog::Tracing.active_trace
+            set_trace_origin(trace)
+
+            tracer_span
+          end
+        end
+
+        def null_span
+          @null_span ||= NullSpan.new
+        end
+
+        def validate_test_suite_level_visibility_correctness(test)
+          return unless test_suite_level_visibility_enabled
+
+          if test.test_suite_id.nil?
+            Datadog.logger.debug do
+              "Test [#{test.name}] does not have a test suite associated with it. " \
+              "Expected test suite [#{test.test_suite_name}] to be running."
+            end
+          end
+
+          if test.test_module_id.nil?
+            Datadog.logger.debug do
+              "Test [#{test.name}] does not have a test module associated with it. " \
+              "Make sure that there is a test module running within a session."
+            end
+          end
+
+          if test.test_session_id.nil?
+            Datadog.logger.debug do
+              "Test [#{test.name}] does not have a test session associated with it. " \
+              "Make sure that there is a test session running."
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/version.rb

@@ -4,7 +4,7 @@ module Datadog
   module CI
     module VERSION
       MAJOR = "0"
-      MINOR = "5"
+      MINOR = "6"
       PATCH = "0"
       PRE = nil
       BUILD = nil

data/lib/datadog/ci.rb

@@ -389,6 +389,6 @@ require_relative "ci/contrib/cucumber/integration"
 require_relative "ci/contrib/rspec/integration"
 require_relative "ci/contrib/minitest/integration"
 
-# Extensions
-require_relative "ci/extensions"
-Datadog::CI::Extensions.activate!
+# Configuration extensions
+require_relative "ci/configuration/extensions"
+Datadog::CI::Configuration::Extensions.activate!

data/sig/datadog/ci/configuration/components.rbs

@@ -2,9 +2,9 @@ module Datadog
   module CI
     module Configuration
       module Components : Datadog::Core::Configuration::Components
-        @ci_recorder: Datadog::CI::Recorder
+        @ci_recorder: Datadog::CI::TestVisibility::Recorder
 
-        attr_reader ci_recorder: Datadog::CI::Recorder
+        attr_reader ci_recorder: Datadog::CI::TestVisibility::Recorder
 
         def initialize: (untyped settings) -> void
 

data/sig/datadog/ci/configuration/extensions.rbs

@@ -0,0 +1,9 @@
+module Datadog
+  module CI
+    module Configuration
+      module Extensions
+        def self.activate!: () -> untyped
+      end
+    end
+  end
+end

data/sig/datadog/ci/contrib/cucumber/ext.rbs

@@ -3,8 +3,6 @@ module Datadog
     module Contrib
       module Cucumber
         module Ext
-          APP: String
-
           ENV_ENABLED: String
 
           ENV_OPERATION_NAME: String
@@ -13,11 +11,9 @@ module Datadog
 
           OPERATION_NAME: String
 
-          SERVICE_NAME: String
+          DEFAULT_SERVICE_NAME: String
 
           STEP_SPAN_TYPE: String
-
-          TEST_TYPE: String
         end
       end
     end

data/sig/datadog/ci/contrib/cucumber/formatter.rbs

@@ -4,19 +4,22 @@ module Datadog
       module Cucumber
         class Formatter
           private
+          @failed_tests_count: Integer
+          @current_test_suite: Datadog::CI::Span?
+          @failed_tests_in_current_test_suite: Integer
 
           attr_reader config: untyped
 
-          attr_reader current_feature_span: untyped
-
-          attr_reader current_step_span: untyped
-
           public
 
           def initialize: (untyped config) -> void
 
           def bind_events: (untyped config) -> untyped
 
+          def on_test_run_started: (untyped event) -> untyped
+
+          def on_test_run_finished: (untyped event) -> untyped
+
           def on_test_case_started: (untyped event) -> untyped
 
           def on_test_case_finished: (untyped event) -> (nil | untyped)
@@ -27,6 +30,16 @@ module Datadog
 
           private
 
+          def start_test_suite: (String test_suite_name) -> void
+
+          def finish_current_test_suite: () -> void
+
+          def same_test_suite_as_current?: (String test_suite_name) -> bool
+
+          def finish_session: (bool result) -> void
+
+          def finish_test: (Datadog::CI::Span test, Cucumber::Core::Test::Result result) -> void
+
           def configuration: () -> untyped
         end
       end

data/sig/datadog/ci/contrib/minitest/ext.rbs

@@ -3,8 +3,6 @@ module Datadog
     module Contrib
       module Minitest
         module Ext
-          APP: String
-
           ENV_ENABLED: String
 
           ENV_OPERATION_NAME: String
@@ -13,9 +11,7 @@ module Datadog
 
           OPERATION_NAME: String
 
-          SERVICE_NAME: String
-
-          TEST_TYPE: String
+          DEFAULT_SERVICE_NAME: String
         end
       end
     end

data/sig/datadog/ci/contrib/minitest/helpers.rbs

@@ -0,0 +1,13 @@
+module Datadog
+  module CI
+    module Contrib
+      module Minitest
+        module Helpers
+          def self.test_suite_name: (untyped klass, String? method_name) -> ::String
+
+          def self.parallel?: (untyped klass) -> bool
+        end
+      end
+    end
+  end
+end

data/sig/datadog/ci/contrib/minitest/hooks.rbs

@@ -11,7 +11,15 @@ module Datadog
 
           private
 
-          def configuration: () -> untyped
+          def datadog_configuration: () -> untyped
+
+          def finish_test: (Datadog::CI::Test test_span, String result_code) -> void
+
+          def finish_test_suite: (Datadog::CI::TestSuite? test_suite, String result_code) -> void
+
+          def finish_with_result: (Datadog::CI::Span span, String result_code) -> void
+
+          def self.test_order: () -> (nil | :parallel | :sorted | :random | :alpha)
         end
       end
     end

data/sig/datadog/ci/contrib/minitest/plugin.rbs

@@ -0,0 +1,31 @@
+module Datadog
+  module CI
+    module Contrib
+      module Minitest
+        module Plugin
+          def self.included: (untyped base) -> untyped
+
+          class DatadogReporter < ::Minitest::AbstractReporter
+            @reporter: WeakRef
+
+            def initialize: (Minitest::AbstractReporter reporter) -> void
+          end
+
+          module ClassMethods
+            def plugin_datadog_ci_init: (*untyped) -> (nil | untyped)
+
+            def initialize: (untyped reporter) -> void
+
+            def report: () -> (nil | untyped)
+
+            def reporter: () -> Minitest::CompositeReporter
+
+            private
+
+            def datadog_configuration: () -> untyped
+          end
+        end
+      end
+    end
+  end
+end

data/sig/datadog/ci/contrib/minitest/runnable.rbs

@@ -0,0 +1,24 @@
+module Datadog
+  module CI
+    module Contrib
+      module Minitest
+        module Runnable
+          def self.included: (untyped base) -> untyped
+
+          module ClassMethods : ::Minitest::Runnable
+
+            def run: (*untyped) -> untyped
+
+            private
+
+            def datadog_configuration: () -> untyped
+
+            def test_order: () -> (nil | :parallel | :random | :sorted | :alpha)
+
+            def runnable_methods: () -> Array[String]
+          end
+        end
+      end
+    end
+  end
+end

data/sig/datadog/ci/contrib/rspec/example_group.rbs

@@ -0,0 +1,21 @@
+module Datadog
+  module CI
+    module Contrib
+      module RSpec
+        module ExampleGroup
+          def self.included: (untyped base) -> untyped
+
+          module ClassMethods
+            include ::RSpec::Core::ExampleGroup::ClassMethods
+
+            def run: (?untyped reporter) -> untyped
+
+            private
+
+            def configuration: () -> untyped
+          end
+        end
+      end
+    end
+  end
+end

data/sig/datadog/ci/contrib/rspec/ext.rbs

@@ -3,19 +3,13 @@ module Datadog
     module Contrib
       module RSpec
         module Ext
-          APP: String
-
           ENV_ENABLED: String
 
-          ENV_OPERATION_NAME: String
-
           FRAMEWORK: String
+          DEFAULT_SERVICE_NAME: String
 
           OPERATION_NAME: String
-
-          SERVICE_NAME: String
-
-          TEST_TYPE: String
+          ENV_OPERATION_NAME: String
         end
       end
     end

data/sig/datadog/ci/contrib/rspec/runner.rbs

@@ -0,0 +1,21 @@
+module Datadog
+  module CI
+    module Contrib
+      module RSpec
+        module Runner
+          def self.included: (untyped base) -> untyped
+
+          module InstanceMethods
+            include ::RSpec::Core::Runner
+
+            def run_specs: (untyped example_groups) -> untyped
+
+            private
+
+            def configuration: () -> untyped
+          end
+        end
+      end
+    end
+  end
+end

data/sig/datadog/ci/ext/test.rbs

@@ -32,6 +32,8 @@ module Datadog
 
         TAG_TEST_SUITE_ID: String
 
+        TEST_TYPE: String
+
         SPECIAL_TAGS: Array[String]
 
         INHERITABLE_TAGS: Array[String]

data/sig/datadog/ci/span.rbs

@@ -13,6 +13,14 @@ module Datadog
 
       def service: () -> String
 
+      def undefined?: () -> bool
+
+      def passed?: () -> bool
+
+      def failed?: () -> bool
+
+      def skipped?: () -> bool
+
       def passed!: () -> void
 
       def failed!: (?exception: untyped?) -> void

data/sig/datadog/ci/test.rbs

@@ -2,6 +2,11 @@ module Datadog
   module CI
     class Test < Span
       def finish: () -> void
+      def test_suite: () -> Datadog::CI::TestSuite?
+      def test_suite_id: () -> String?
+      def test_suite_name: () -> String?
+      def test_module_id: () -> String?
+      def test_session_id: () -> String?
     end
   end
 end

data/sig/datadog/ci/test_visibility/context/global.rbs

@@ -0,0 +1,39 @@
+module Datadog
+  module CI
+    module TestVisibility
+      module Context
+        class Global
+          @mutex: Thread::Mutex
+
+          @test_session: Datadog::CI::TestSession?
+          @test_module: Datadog::CI::TestModule?
+          @test_suites: Hash[String, Datadog::CI::TestSuite]
+
+          def initialize: () -> void
+
+          def fetch_or_activate_test_suite: (String test_suite_name) {() -> Datadog::CI::TestSuite} -> Datadog::CI::TestSuite
+
+          def fetch_or_activate_test_module: () {() -> Datadog::CI::TestModule} -> Datadog::CI::TestModule
+
+          def fetch_or_activate_test_session: () {() -> Datadog::CI::TestSession} -> Datadog::CI::TestSession
+
+          def active_test_session: () -> Datadog::CI::TestSession?
+
+          def active_test_suite: (String test_suite_name) -> Datadog::CI::TestSuite?
+
+          def service: () -> String?
+
+          def inheritable_session_tags: () -> Hash[untyped, untyped]
+
+          def active_test_module: () -> Datadog::CI::TestModule?
+
+          def deactivate_test_session!: () -> void
+
+          def deactivate_test_module!: () -> void
+
+          def deactivate_test_suite!: (String test_suite_name) -> void
+        end
+      end
+    end
+  end
+end