datadog-ci 0.5.1 → 0.6.0

data/lib/datadog/ci/contrib/rspec/patcher.rb

@@ -1,7 +1,10 @@
 # frozen_string_literal: true
 
 require "datadog/tracing/contrib/patcher"
+
 require_relative "example"
+require_relative "example_group"
+require_relative "runner"
 
 module Datadog
   module CI
@@ -19,6 +22,8 @@ module Datadog
 
           def patch
             ::RSpec::Core::Example.include(Example)
+            ::RSpec::Core::Runner.include(Runner)
+            ::RSpec::Core::ExampleGroup.include(ExampleGroup)
           end
         end
       end

data/lib/datadog/ci/contrib/rspec/runner.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+require_relative "../../ext/test"
+require_relative "ext"
+
+module Datadog
+  module CI
+    module Contrib
+      module RSpec
+        # Instrument RSpec::Core::Runner
+        module Runner
+          def self.included(base)
+            base.prepend(InstanceMethods)
+          end
+
+          module InstanceMethods
+            def run_specs(example_groups)
+              return super unless configuration[:enabled]
+
+              test_session = CI.start_test_session(
+                tags: {
+                  CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK,
+                  CI::Ext::Test::TAG_FRAMEWORK_VERSION => CI::Contrib::RSpec::Integration.version.to_s,
+                  CI::Ext::Test::TAG_TYPE => CI::Ext::Test::TEST_TYPE
+                },
+                service: configuration[:service_name]
+              )
+
+              test_module = CI.start_test_module(test_session.name)
+
+              result = super
+
+              if result != 0
+                # TODO: repeating this twice feels clunky, we need to remove test_module API before GA
+                test_module.failed!
+                test_session.failed!
+              else
+                test_module.passed!
+                test_session.passed!
+              end
+              test_module.finish
+              test_session.finish
+
+              result
+            end
+
+            private
+
+            def configuration
+              Datadog.configuration.ci[:rspec]
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/contrib/settings.rb

@@ -27,7 +27,7 @@ module Datadog
         end
 
         def []=(name, value)
-          respond_to?("#{name}=") ? send("#{name}=", value) : set_option(name, value)
+          respond_to?(:"#{name}=") ? send(:"#{name}=", value) : set_option(name, value)
         end
       end
     end

data/lib/datadog/ci/ext/test.rb

@@ -20,6 +20,8 @@ module Datadog
         TAG_TYPE = "test.type"
         TAG_COMMAND = "test.command"
 
+        TEST_TYPE = "test"
+
         # those tags are special and they are used to correlate tests with the test sessions, suites, and modules
         TAG_TEST_SESSION_ID = "_test.session_id"
         TAG_TEST_MODULE_ID = "_test.module_id"

data/lib/datadog/ci/span.rb

@@ -36,6 +36,30 @@ module Datadog
         tracer_span.type
       end
 
+      # Checks whether span status is not set yet.
+      # @return [bool] true if span does not have status
+      def undefined?
+        tracer_span.get_tag(Ext::Test::TAG_STATUS).nil?
+      end
+
+      # Checks whether span status is PASS.
+      # @return [bool] true if span status is PASS
+      def passed?
+        tracer_span.get_tag(Ext::Test::TAG_STATUS) == Ext::Test::Status::PASS
+      end
+
+      # Checks whether span status is FAIL.
+      # @return [bool] true if span status is FAIL
+      def failed?
+        tracer_span.get_tag(Ext::Test::TAG_STATUS) == Ext::Test::Status::FAIL
+      end
+
+      # Checks whether span status is SKIP.
+      # @return [bool] true if span status is SKIP
+      def skipped?
+        tracer_span.get_tag(Ext::Test::TAG_STATUS) == Ext::Test::Status::SKIP
+      end
+
       # Sets the status of the span to "pass".
       # @return [void]
       def passed!

data/lib/datadog/ci/test.rb

@@ -20,6 +20,36 @@ module Datadog
 
         CI.deactivate_test(self)
       end
+
+      # Running test suite that this test is part of (if any).
+      # @return [Datadog::CI::TestSuite] the test suite this test belongs to
+      # @return [nil] if the test suite is not found
+      def test_suite
+        suite_name = test_suite_name
+        CI.active_test_suite(suite_name) if suite_name
+      end
+
+      # Span id of the running test suite this test belongs to.
+      # @return [String] the span id of the test suite.
+      def test_suite_id
+        get_tag(Ext::Test::TAG_TEST_SUITE_ID)
+      end
+
+      def test_suite_name
+        get_tag(Ext::Test::TAG_SUITE)
+      end
+
+      # Span id of the running test module this test belongs to.
+      # @return [String] the span id of the test module.
+      def test_module_id
+        get_tag(Ext::Test::TAG_TEST_MODULE_ID)
+      end
+
+      # Span id of the running test module this test belongs to.
+      # @return [String] the span id of the test session.
+      def test_session_id
+        get_tag(Ext::Test::TAG_TEST_SESSION_ID)
+      end
     end
   end
 end

data/lib/datadog/ci/test_session.rb

@@ -20,6 +20,14 @@ module Datadog
         CI.deactivate_test_session
       end
 
+      # Return the test session's name which is equal to test command used
+      # @return [String] the command for this test session.
+      def name
+        get_tag(Ext::Test::TAG_COMMAND)
+      end
+
+      # Return the test session tags that could be inherited by sub-spans
+      # @return [Hash] the tags to be inherited by sub-spans.
       def inheritable_tags
         return @inheritable_tags if defined?(@inheritable_tags)
 

data/lib/datadog/ci/test_visibility/null_recorder.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+require_relative "recorder"
+
+module Datadog
+  module CI
+    module TestVisibility
+      # Special recorder that does not record anything
+      class NullRecorder
+        def start_test_session(service: nil, tags: {})
+          skip_tracing
+        end
+
+        def start_test_module(test_module_name, service: nil, tags: {})
+          skip_tracing
+        end
+
+        def start_test_suite(test_suite_name, service: nil, tags: {})
+          skip_tracing
+        end
+
+        def trace_test(test_name, test_suite_name, service: nil, tags: {}, &block)
+          skip_tracing(block)
+        end
+
+        def trace(span_type, span_name, tags: {}, &block)
+          skip_tracing(block)
+        end
+
+        def active_span
+        end
+
+        def active_test
+        end
+
+        def active_test_session
+        end
+
+        def active_test_module
+        end
+
+        def active_test_suite(test_suite_name)
+        end
+
+        def deactivate_test(test)
+        end
+
+        def deactivate_test_session
+        end
+
+        def deactivate_test_module
+        end
+
+        def deactivate_test_suite(test_suite_name)
+        end
+
+        private
+
+        def skip_tracing(block = nil)
+          if block
+            block.call(null_span)
+          else
+            null_span
+          end
+        end
+
+        def null_span
+          @null_span ||= NullSpan.new
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/test_visibility/recorder.rb

@@ -25,13 +25,12 @@ module Datadog
       # Common behavior for CI tests
       # Note: this class has too many responsibilities and should be split into multiple classes
       class Recorder
-        attr_reader :environment_tags, :test_suite_level_visibility_enabled, :enabled
+        attr_reader :environment_tags, :test_suite_level_visibility_enabled
 
-        def initialize(enabled: true, test_suite_level_visibility_enabled: false)
-          @enabled = enabled
-          @test_suite_level_visibility_enabled = enabled && test_suite_level_visibility_enabled
+        def initialize(test_suite_level_visibility_enabled: false)
+          @test_suite_level_visibility_enabled = test_suite_level_visibility_enabled
 
-          @environment_tags = @enabled ? Ext::Environment.tags(ENV).freeze : {}
+          @environment_tags = Ext::Environment.tags(ENV).freeze
           @local_context = Context::Local.new
           @global_context = Context::Global.new
         end
@@ -83,8 +82,6 @@ module Datadog
         end
 
         def trace_test(test_name, test_suite_name, service: nil, tags: {}, &block)
-          return skip_tracing(block) unless enabled
-
           set_inherited_globals(tags)
           set_session_context(tags)
           set_module_context(tags)
@@ -118,8 +115,6 @@ module Datadog
         end
 
         def trace(span_type, span_name, tags: {}, &block)
-          return skip_tracing(block) unless enabled
-
           span_options = build_span_options(
             nil, # service name is completely optional for custom spans
             span_type,
@@ -210,6 +205,7 @@ module Datadog
         def build_test(tracer_span, tags)
           test = Test.new(tracer_span)
           set_initial_tags(test, tags)
+          validate_test_suite_level_visibility_correctness(test)
           test
         end
 
@@ -287,6 +283,31 @@ module Datadog
         def null_span
           @null_span ||= NullSpan.new
         end
+
+        def validate_test_suite_level_visibility_correctness(test)
+          return unless test_suite_level_visibility_enabled
+
+          if test.test_suite_id.nil?
+            Datadog.logger.debug do
+              "Test [#{test.name}] does not have a test suite associated with it. " \
+              "Expected test suite [#{test.test_suite_name}] to be running."
+            end
+          end
+
+          if test.test_module_id.nil?
+            Datadog.logger.debug do
+              "Test [#{test.name}] does not have a test module associated with it. " \
+              "Make sure that there is a test module running within a session."
+            end
+          end
+
+          if test.test_session_id.nil?
+            Datadog.logger.debug do
+              "Test [#{test.name}] does not have a test session associated with it. " \
+              "Make sure that there is a test session running."
+            end
+          end
+        end
       end
     end
   end

data/lib/datadog/ci/version.rb

@@ -4,8 +4,8 @@ module Datadog
   module CI
     module VERSION
       MAJOR = "0"
-      MINOR = "5"
-      PATCH = "1"
+      MINOR = "6"
+      PATCH = "0"
       PRE = nil
       BUILD = nil
       # PRE and BUILD above are modified for dev gems during gem build GHA workflow

data/sig/datadog/ci/contrib/cucumber/ext.rbs

@@ -3,8 +3,6 @@ module Datadog
     module Contrib
       module Cucumber
         module Ext
-          APP: String
-
           ENV_ENABLED: String
 
           ENV_OPERATION_NAME: String
@@ -13,11 +11,9 @@ module Datadog
 
           OPERATION_NAME: String
 
-          SERVICE_NAME: String
+          DEFAULT_SERVICE_NAME: String
 
           STEP_SPAN_TYPE: String
-
-          TEST_TYPE: String
         end
       end
     end

data/sig/datadog/ci/contrib/cucumber/formatter.rbs

@@ -4,19 +4,22 @@ module Datadog
       module Cucumber
         class Formatter
           private
+          @failed_tests_count: Integer
+          @current_test_suite: Datadog::CI::Span?
+          @failed_tests_in_current_test_suite: Integer
 
           attr_reader config: untyped
 
-          attr_reader current_feature_span: untyped
-
-          attr_reader current_step_span: untyped
-
           public
 
           def initialize: (untyped config) -> void
 
           def bind_events: (untyped config) -> untyped
 
+          def on_test_run_started: (untyped event) -> untyped
+
+          def on_test_run_finished: (untyped event) -> untyped
+
           def on_test_case_started: (untyped event) -> untyped
 
           def on_test_case_finished: (untyped event) -> (nil | untyped)
@@ -27,6 +30,16 @@ module Datadog
 
           private
 
+          def start_test_suite: (String test_suite_name) -> void
+
+          def finish_current_test_suite: () -> void
+
+          def same_test_suite_as_current?: (String test_suite_name) -> bool
+
+          def finish_session: (bool result) -> void
+
+          def finish_test: (Datadog::CI::Span test, Cucumber::Core::Test::Result result) -> void
+
           def configuration: () -> untyped
         end
       end

data/sig/datadog/ci/contrib/minitest/ext.rbs

@@ -3,8 +3,6 @@ module Datadog
     module Contrib
       module Minitest
         module Ext
-          APP: String
-
           ENV_ENABLED: String
 
           ENV_OPERATION_NAME: String
@@ -13,9 +11,7 @@ module Datadog
 
           OPERATION_NAME: String
 
-          SERVICE_NAME: String
-
-          TEST_TYPE: String
+          DEFAULT_SERVICE_NAME: String
         end
       end
     end

data/sig/datadog/ci/contrib/minitest/helpers.rbs

@@ -0,0 +1,13 @@
+module Datadog
+  module CI
+    module Contrib
+      module Minitest
+        module Helpers
+          def self.test_suite_name: (untyped klass, String? method_name) -> ::String
+
+          def self.parallel?: (untyped klass) -> bool
+        end
+      end
+    end
+  end
+end

data/sig/datadog/ci/contrib/minitest/hooks.rbs

@@ -11,7 +11,15 @@ module Datadog
 
           private
 
-          def configuration: () -> untyped
+          def datadog_configuration: () -> untyped
+
+          def finish_test: (Datadog::CI::Test test_span, String result_code) -> void
+
+          def finish_test_suite: (Datadog::CI::TestSuite? test_suite, String result_code) -> void
+
+          def finish_with_result: (Datadog::CI::Span span, String result_code) -> void
+
+          def self.test_order: () -> (nil | :parallel | :sorted | :random | :alpha)
         end
       end
     end

data/sig/datadog/ci/contrib/minitest/plugin.rbs

@@ -0,0 +1,31 @@
+module Datadog
+  module CI
+    module Contrib
+      module Minitest
+        module Plugin
+          def self.included: (untyped base) -> untyped
+
+          class DatadogReporter < ::Minitest::AbstractReporter
+            @reporter: WeakRef
+
+            def initialize: (Minitest::AbstractReporter reporter) -> void
+          end
+
+          module ClassMethods
+            def plugin_datadog_ci_init: (*untyped) -> (nil | untyped)
+
+            def initialize: (untyped reporter) -> void
+
+            def report: () -> (nil | untyped)
+
+            def reporter: () -> Minitest::CompositeReporter
+
+            private
+
+            def datadog_configuration: () -> untyped
+          end
+        end
+      end
+    end
+  end
+end

data/sig/datadog/ci/contrib/minitest/runnable.rbs

@@ -0,0 +1,24 @@
+module Datadog
+  module CI
+    module Contrib
+      module Minitest
+        module Runnable
+          def self.included: (untyped base) -> untyped
+
+          module ClassMethods : ::Minitest::Runnable
+
+            def run: (*untyped) -> untyped
+
+            private
+
+            def datadog_configuration: () -> untyped
+
+            def test_order: () -> (nil | :parallel | :random | :sorted | :alpha)
+
+            def runnable_methods: () -> Array[String]
+          end
+        end
+      end
+    end
+  end
+end

data/sig/datadog/ci/contrib/rspec/example_group.rbs

@@ -0,0 +1,21 @@
+module Datadog
+  module CI
+    module Contrib
+      module RSpec
+        module ExampleGroup
+          def self.included: (untyped base) -> untyped
+
+          module ClassMethods
+            include ::RSpec::Core::ExampleGroup::ClassMethods
+
+            def run: (?untyped reporter) -> untyped
+
+            private
+
+            def configuration: () -> untyped
+          end
+        end
+      end
+    end
+  end
+end

data/sig/datadog/ci/contrib/rspec/ext.rbs

@@ -3,19 +3,13 @@ module Datadog
     module Contrib
       module RSpec
         module Ext
-          APP: String
-
           ENV_ENABLED: String
 
-          ENV_OPERATION_NAME: String
-
           FRAMEWORK: String
+          DEFAULT_SERVICE_NAME: String
 
           OPERATION_NAME: String
-
-          SERVICE_NAME: String
-
-          TEST_TYPE: String
+          ENV_OPERATION_NAME: String
         end
       end
     end

data/sig/datadog/ci/contrib/rspec/runner.rbs

@@ -0,0 +1,21 @@
+module Datadog
+  module CI
+    module Contrib
+      module RSpec
+        module Runner
+          def self.included: (untyped base) -> untyped
+
+          module InstanceMethods
+            include ::RSpec::Core::Runner
+
+            def run_specs: (untyped example_groups) -> untyped
+
+            private
+
+            def configuration: () -> untyped
+          end
+        end
+      end
+    end
+  end
+end

data/sig/datadog/ci/ext/test.rbs

@@ -32,6 +32,8 @@ module Datadog
 
         TAG_TEST_SUITE_ID: String
 
+        TEST_TYPE: String
+
         SPECIAL_TAGS: Array[String]
 
         INHERITABLE_TAGS: Array[String]

data/sig/datadog/ci/span.rbs

@@ -13,6 +13,14 @@ module Datadog
 
       def service: () -> String
 
+      def undefined?: () -> bool
+
+      def passed?: () -> bool
+
+      def failed?: () -> bool
+
+      def skipped?: () -> bool
+
       def passed!: () -> void
 
       def failed!: (?exception: untyped?) -> void

data/sig/datadog/ci/test.rbs

@@ -2,6 +2,11 @@ module Datadog
   module CI
     class Test < Span
       def finish: () -> void
+      def test_suite: () -> Datadog::CI::TestSuite?
+      def test_suite_id: () -> String?
+      def test_suite_name: () -> String?
+      def test_module_id: () -> String?
+      def test_session_id: () -> String?
     end
   end
 end

data/sig/datadog/ci/test_visibility/null_recorder.rbs

@@ -0,0 +1,45 @@
+module Datadog
+  module CI
+    module TestVisibility
+      class NullRecorder
+        @null_span: Datadog::CI::NullSpan
+
+        def initialize: (?untyped args) -> void
+
+        def trace_test: (String span_name, String test_suite_name, ?service: String?, ?tags: Hash[untyped, untyped]) ?{ (Datadog::CI::Span span) -> untyped } -> untyped
+
+        def trace: (String span_type, String span_name, ?tags: Hash[untyped, untyped]) ?{ (Datadog::CI::Span span) -> untyped } -> untyped
+
+        def start_test_session: (?service: String?, ?tags: Hash[untyped, untyped]) -> Datadog::CI::Span
+
+        def start_test_module: (String test_module_name, ?service: String?, ?tags: Hash[untyped, untyped]) -> Datadog::CI::Span
+
+        def start_test_suite: (String test_suite_name, ?service: String?, ?tags: Hash[untyped, untyped]) -> Datadog::CI::Span
+
+        def active_test_session: () -> Datadog::CI::TestSession?
+
+        def active_test_module: () -> Datadog::CI::TestModule?
+
+        def active_test_suite: (String test_suite_name) -> Datadog::CI::TestSuite?
+
+        def active_test: () -> Datadog::CI::Test?
+
+        def active_span: () -> Datadog::CI::Span?
+
+        def deactivate_test: (Datadog::CI::Test test) -> void
+
+        def deactivate_test_session: () -> void
+
+        def deactivate_test_module: () -> void
+
+        def deactivate_test_suite: (String test_suite_name) -> void
+
+        private
+
+        def null_span: () -> Datadog::CI::Span
+
+        def skip_tracing: (?untyped block) -> untyped
+      end
+    end
+  end
+end