datadog-ci 1.1.0 → 1.3.0

data/lib/datadog/ci/test_visibility/{recorder.rb → context.rb}

@@ -4,87 +4,49 @@ require "datadog/tracing"
 require "datadog/tracing/contrib/component"
 require "datadog/tracing/trace_digest"
 
-require "rbconfig"
+require_relative "store/global"
+require_relative "store/local"
 
-require_relative "context/global"
-require_relative "context/local"
-
-require_relative "../codeowners/parser"
-require_relative "../contrib/contrib"
 require_relative "../ext/app_types"
-require_relative "../ext/test"
 require_relative "../ext/environment"
-require_relative "../git/local_repository"
+require_relative "../ext/test"
 
 require_relative "../span"
 require_relative "../test"
 require_relative "../test_session"
 require_relative "../test_module"
 require_relative "../test_suite"
-require_relative "../worker"
 
 module Datadog
   module CI
     module TestVisibility
-      # Common behavior for CI tests
-      # Note: this class has too many responsibilities and should be split into multiple classes
-      class Recorder
-        attr_reader :environment_tags, :test_suite_level_visibility_enabled
-
-        def initialize(
-          itr:,
-          remote_settings_api:,
-          git_tree_upload_worker: DummyWorker.new,
-          test_suite_level_visibility_enabled: false,
-          codeowners: Codeowners::Parser.new(Git::LocalRepository.root).parse
-        )
-          @test_suite_level_visibility_enabled = test_suite_level_visibility_enabled
-
-          @environment_tags = Ext::Environment.tags(ENV).freeze
-          @local_context = Context::Local.new
-          @global_context = Context::Global.new
-
-          @codeowners = codeowners
-
-          @itr = itr
-          @remote_settings_api = remote_settings_api
-          @git_tree_upload_worker = git_tree_upload_worker
-        end
-
-        def shutdown!
-          @git_tree_upload_worker.stop
+      # Manages current in-memory context for test visibility (such as active test session, suite, test, etc.).
+      # Its responsibility includes building domain models for test visibility as well.
+      # Internally it uses Datadog::Tracing module to create spans.
+      class Context
+        def initialize
+          @local_context = Store::Local.new
+          @global_context = Store::Global.new
         end
 
         def start_test_session(service: nil, tags: {})
-          return skip_tracing unless test_suite_level_visibility_enabled
-
-          # finds and instruments additional test libraries that we support (ex: selenium-webdriver)
-          Contrib.auto_instrument_on_session_start!
-
           @global_context.fetch_or_activate_test_session do
             tracer_span = start_datadog_tracer_span(
-              "test.session", build_span_options(service, Ext::AppTypes::TYPE_TEST_SESSION)
+              "test.session", build_tracing_span_options(service, Ext::AppTypes::TYPE_TEST_SESSION)
             )
             set_session_context(tags, tracer_span)
 
-            test_session = build_test_session(tracer_span, tags)
-
-            @git_tree_upload_worker.perform(test_session.git_repository_url)
-            configure_library(test_session)
-
-            test_session
+            build_test_session(tracer_span, tags)
           end
         end
 
         def start_test_module(test_module_name, service: nil, tags: {})
-          return skip_tracing unless test_suite_level_visibility_enabled
-
           @global_context.fetch_or_activate_test_module do
             set_inherited_globals(tags)
             set_session_context(tags)
 
             tracer_span = start_datadog_tracer_span(
-              test_module_name, build_span_options(service, Ext::AppTypes::TYPE_TEST_MODULE)
+              test_module_name, build_tracing_span_options(service, Ext::AppTypes::TYPE_TEST_MODULE)
             )
             set_module_context(tags, tracer_span)
 
@@ -93,15 +55,13 @@ module Datadog
         end
 
         def start_test_suite(test_suite_name, service: nil, tags: {})
-          return skip_tracing unless test_suite_level_visibility_enabled
-
           @global_context.fetch_or_activate_test_suite(test_suite_name) do
             set_inherited_globals(tags)
             set_session_context(tags)
             set_module_context(tags)
 
             tracer_span = start_datadog_tracer_span(
-              test_suite_name, build_span_options(service, Ext::AppTypes::TYPE_TEST_SUITE)
+              test_suite_name, build_tracing_span_options(service, Ext::AppTypes::TYPE_TEST_SUITE)
             )
             set_suite_context(tags, span: tracer_span)
 
@@ -118,7 +78,7 @@ module Datadog
           tags[Ext::Test::TAG_NAME] = test_name
           tags[Ext::Test::TAG_TYPE] ||= Ext::Test::Type::TEST
 
-          span_options = build_span_options(
+          span_options = build_tracing_span_options(
             service,
             Ext::AppTypes::TYPE_TEST,
             # :resource is needed for the agent APM protocol to work correctly (for older agent versions)
@@ -131,25 +91,19 @@ module Datadog
               test = build_test(tracer_span, tags)
 
               @local_context.activate_test(test) do
-                on_test_started(test)
-                res = block.call(test)
-                on_test_finished(test)
-                res
+                block.call(test)
               end
             end
           else
             tracer_span = start_datadog_tracer_span(test_name, span_options)
             test = build_test(tracer_span, tags)
-
             @local_context.activate_test(test)
-            on_test_started(test)
-
             test
           end
         end
 
         def trace(span_name, type: "span", tags: {}, &block)
-          span_options = build_span_options(
+          span_options = build_tracing_span_options(
             nil, # service name is completely optional for custom spans
             type,
             {resource: span_name}
@@ -187,17 +141,15 @@ module Datadog
           @global_context.active_test_suite(test_suite_name)
         end
 
-        def deactivate_test
-          test = active_test
-          on_test_finished(test) if test
+        def single_active_test_suite
+          @global_context.fetch_single_test_suite
+        end
 
+        def deactivate_test
           @local_context.deactivate_test
         end
 
         def deactivate_test_session
-          test_session = active_test_session
-          on_test_session_finished(test_session) if test_session
-
           @global_context.deactivate_test_session!
         end
 
@@ -209,47 +161,9 @@ module Datadog
           @global_context.deactivate_test_suite!(test_suite_name)
         end
 
-        def itr_enabled?
-          @itr.enabled?
-        end
-
         private
 
-        def configure_library(test_session)
-          # this will change when EFD is implemented
-          return unless itr_enabled?
-
-          remote_configuration = @remote_settings_api.fetch_library_settings(test_session)
-          # sometimes we can skip code coverage for default branch if there are no changes in the repository
-          # backend needs git metadata uploaded for this test session to check if we can skip code coverage
-          if remote_configuration.require_git?
-            Datadog.logger.debug { "Library configuration endpoint requires git upload to be finished, waiting..." }
-            @git_tree_upload_worker.wait_until_done
-
-            Datadog.logger.debug { "Requesting library configuration again..." }
-            remote_configuration = @remote_settings_api.fetch_library_settings(test_session)
-
-            if remote_configuration.require_git?
-              Datadog.logger.debug { "git metadata upload did not complete in time when configuring library" }
-            end
-          end
-
-          @itr.configure(
-            remote_configuration.payload,
-            test_session: test_session,
-            git_tree_upload_worker: @git_tree_upload_worker
-          )
-        end
-
-        def skip_tracing(block = nil)
-          block&.call(nil)
-        end
-
-        # Sets trace's origin to ciapp-test
-        def set_trace_origin(trace)
-          trace&.origin = Ext::Test::CONTEXT_ORIGIN
-        end
-
+        # BUILDING DOMAIN MODELS
         def build_test_session(tracer_span, tags)
           test_session = TestSession.new(tracer_span)
           set_initial_tags(test_session, tags)
@@ -271,14 +185,6 @@ module Datadog
         def build_test(tracer_span, tags)
           test = Test.new(tracer_span)
           set_initial_tags(test, tags)
-
-          # sometimes test suite is not being assigned correctly
-          # fix it by fetching the one single running test suite from the global context
-          fix_test_suite!(test) if test.test_suite_id.nil?
-
-          validate_test_suite_level_visibility_correctness(test)
-          set_codeowners(test)
-
           test
         end
 
@@ -288,13 +194,18 @@ module Datadog
           span
         end
 
-        def build_span_options(service, type, other_options = {})
-          other_options[:service] = service || @global_context.service
-          other_options[:type] = type
+        # TAGGING
+        def set_initial_tags(ci_span, tags)
+          @environment_tags ||= Ext::Environment.tags(ENV).freeze
 
-          other_options
+          ci_span.set_default_tags
+          ci_span.set_environment_runtime_tags
+
+          ci_span.set_tags(tags)
+          ci_span.set_tags(@environment_tags)
         end
 
+        # PROPAGATING CONTEXT FROM TOP-LEVEL TO THE LOWER LEVELS
         def set_inherited_globals(tags)
           # this code achieves the same as @global_context.inheritable_session_tags.merge(tags)
           # but without allocating a new hash
@@ -303,14 +214,6 @@ module Datadog
           end
         end
 
-        def set_initial_tags(ci_span, tags)
-          ci_span.set_default_tags
-          ci_span.set_environment_runtime_tags
-
-          ci_span.set_tags(tags)
-          ci_span.set_tags(environment_tags)
-        end
-
         def set_session_context(tags, test_session = nil)
           test_session ||= active_test_session
           tags[Ext::Test::TAG_TEST_SESSION_ID] = test_session.id.to_s if test_session
@@ -324,12 +227,6 @@ module Datadog
           end
         end
 
-        def set_codeowners(test)
-          source = test.source_file
-          owners = @codeowners.list_owners(source) if source
-          test.set_tag(Ext::Test::TAG_CODEOWNERS, owners) unless owners.nil?
-        end
-
         def set_suite_context(tags, span: nil, name: nil)
           return if span.nil? && name.nil?
 
@@ -343,24 +240,7 @@ module Datadog
           end
         end
 
-        def fix_test_suite!(test)
-          test_suite = @global_context.fetch_single_test_suite
-          unless test_suite
-            Datadog.logger.debug do
-              "Trying to fix test suite for test [#{test.name}] but no single test suite is running."
-            end
-            return
-          end
-
-          Datadog.logger.debug do
-            "For test [#{test.name}]: expected test suite [#{test.test_suite_name}] to be running, " \
-            "but it was not found. Fixing it by assigning test suite [#{test_suite.name}] to the test."
-          end
-
-          test.set_tag(Ext::Test::TAG_TEST_SUITE_ID, test_suite.id.to_s)
-          test.set_tag(Ext::Test::TAG_SUITE, test_suite.name)
-        end
-
+        # INTERACTIONS WITH TRACING
         def start_datadog_tracer_span(span_name, span_options, &block)
           if block
             Datadog::Tracing.trace(span_name, **span_options) do |tracer_span, trace|
@@ -377,44 +257,16 @@ module Datadog
           end
         end
 
-        def validate_test_suite_level_visibility_correctness(test)
-          return unless test_suite_level_visibility_enabled
-
-          if test.test_suite_id.nil?
-            Datadog.logger.debug do
-              "Test [#{test.name}] does not have a test suite associated with it. " \
-              "Expected test suite [#{test.test_suite_name}] to be running."
-            end
-          end
-
-          if test.test_module_id.nil?
-            Datadog.logger.debug do
-              "Test [#{test.name}] does not have a test module associated with it. " \
-              "Make sure that there is a test module running within a session."
-            end
-          end
-
-          if test.test_session_id.nil?
-            Datadog.logger.debug do
-              "Test [#{test.name}] does not have a test session associated with it. " \
-              "Make sure that there is a test session running."
-            end
-          end
-        end
-
-        # TODO: use kind of event system to notify about test finished?
-        def on_test_finished(test)
-          @itr.stop_coverage(test)
-          @itr.count_skipped_test(test)
+        # Sets trace's origin to ciapp-test because tracing requires so
+        def set_trace_origin(trace)
+          trace&.origin = Ext::Test::CONTEXT_ORIGIN
         end
 
-        def on_test_started(test)
-          @itr.mark_if_skippable(test)
-          @itr.start_coverage(test)
-        end
+        def build_tracing_span_options(service, type, other_options = {})
+          other_options[:service] = service || @global_context.service
+          other_options[:type] = type
 
-        def on_test_session_finished(test_session)
-          @itr.write_test_session_tags(test_session)
+          other_options
         end
       end
     end

data/lib/datadog/ci/test_visibility/{null_recorder.rb → null_component.rb}

@@ -1,12 +1,10 @@
 # frozen_string_literal: true
 
-require_relative "recorder"
-
 module Datadog
   module CI
     module TestVisibility
-      # Special recorder that does not record anything
-      class NullRecorder
+      # Special test visibility component that does not record anything
+      class NullComponent
         def start_test_session(service: nil, tags: {})
           skip_tracing
         end
@@ -45,6 +43,10 @@ module Datadog
         def shutdown!
         end
 
+        def itr_enabled?
+          false
+        end
+
         private
 
         def skip_tracing(block = nil)

data/lib/datadog/ci/test_visibility/{context → store}/global.rb

@@ -3,7 +3,7 @@
 module Datadog
   module CI
     module TestVisibility
-      module Context
+      module Store
         # This context is shared between threads and represents the current test session and test module.
         class Global
           def initialize

data/lib/datadog/ci/test_visibility/{context → store}/local.rb

@@ -3,7 +3,7 @@
 module Datadog
   module CI
     module TestVisibility
-      module Context
+      module Store
         class Local
           def initialize
             @key = :datadog_ci_active_test

data/lib/datadog/ci/test_visibility/telemetry.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+require_relative "../ext/app_types"
+require_relative "../ext/environment"
+require_relative "../ext/telemetry"
+require_relative "../ext/test"
+require_relative "../utils/telemetry"
+
+module Datadog
+  module CI
+    module TestVisibility
+      # Telemetry for test visibility
+      module Telemetry
+        SPAN_TYPE_TO_TELEMETRY_EVENT_TYPE = {
+          Ext::AppTypes::TYPE_TEST => Ext::Telemetry::EventType::TEST,
+          Ext::AppTypes::TYPE_TEST_SUITE => Ext::Telemetry::EventType::SUITE,
+          Ext::AppTypes::TYPE_TEST_MODULE => Ext::Telemetry::EventType::MODULE,
+          Ext::AppTypes::TYPE_TEST_SESSION => Ext::Telemetry::EventType::SESSION
+        }.freeze
+
+        def self.event_created(span)
+          Utils::Telemetry.inc(Ext::Telemetry::METRIC_EVENT_CREATED, 1, event_tags_from_span(span))
+        end
+
+        def self.event_finished(span)
+          tags = event_tags_from_span(span)
+          add_browser_tags!(span, tags)
+          Utils::Telemetry.inc(Ext::Telemetry::METRIC_EVENT_FINISHED, 1, tags)
+        end
+
+        def self.test_session_started(test_session)
+          Utils::Telemetry.inc(
+            Ext::Telemetry::METRIC_TEST_SESSION,
+            1,
+            {
+              Ext::Telemetry::TAG_AUTO_INJECTED => "false", # ruby doesn't support auto injection yet
+              Ext::Telemetry::TAG_PROVIDER =>
+                test_session.get_tag(Ext::Environment::TAG_PROVIDER_NAME) ||
+                Ext::Telemetry::Provider::UNSUPPORTED
+            }
+          )
+        end
+
+        def self.event_tags_from_span(span)
+          # base tags for span
+          # @type var tags: Hash[String, String]
+          tags = {
+            Ext::Telemetry::TAG_EVENT_TYPE => SPAN_TYPE_TO_TELEMETRY_EVENT_TYPE.fetch(span.type, "unknown"),
+            Ext::Telemetry::TAG_TEST_FRAMEWORK => span.get_tag(Ext::Test::TAG_FRAMEWORK)
+          }
+
+          # ci provider tag
+          tags[Ext::Telemetry::TAG_IS_UNSUPPORTED_CI] = "true" if span.get_tag(Ext::Environment::TAG_PROVIDER_NAME).nil?
+
+          # codeowner tag
+          tags[Ext::Telemetry::TAG_HAS_CODEOWNER] = "true" if span.get_tag(Ext::Test::TAG_CODEOWNERS)
+
+          tags
+        end
+
+        def self.add_browser_tags!(span, tags)
+          tags[Ext::Telemetry::TAG_IS_RUM] = "true" if span.get_tag(Ext::Test::TAG_IS_RUM_ACTIVE)
+          browser_driver = span.get_tag(Ext::Test::TAG_BROWSER_DRIVER)
+          tags[Ext::Telemetry::TAG_BROWSER_DRIVER] = browser_driver if browser_driver
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/test_visibility/transport.rb

@@ -3,8 +3,10 @@
 require "datadog/core/environment/identity"
 
 require_relative "serializers/factories/test_level"
+require_relative "../ext/telemetry"
 require_relative "../ext/transport"
 require_relative "../transport/event_platform_transport"
+require_relative "../transport/telemetry"
 
 module Datadog
   module CI
@@ -31,6 +33,10 @@ module Datadog
 
         private
 
+        def telemetry_endpoint_tag
+          Ext::Telemetry::Endpoint::TEST_CYCLE
+        end
+
         def send_payload(encoded_payload)
           api.citestcycle_request(
             path: Datadog::CI::Ext::Transport::TEST_VISIBILITY_INTAKE_PATH,
@@ -46,21 +52,14 @@ module Datadog
 
         def encode_span(trace, span)
           serializer = serializers_factory.serializer(trace, span, options: {itr_correlation_id: itr&.correlation_id})
-
           if serializer.valid?
             encoded = encoder.encode(serializer)
-
-            if encoded.size > max_payload_size
-              # This single event is too large, we can't flush it
-              Datadog.logger.warn("Dropping test event. Payload too large: '#{span.inspect}'")
-              Datadog.logger.warn(encoded)
-
-              return nil
-            end
+            return nil if event_too_large?(span, encoded)
 
             encoded
           else
             Datadog.logger.warn("Invalid event skipped: #{serializer} Errors: #{serializer.validation_errors}")
+            CI::Transport::Telemetry.endpoint_payload_dropped(1, endpoint: telemetry_endpoint_tag)
             nil
           end
         end
@@ -100,7 +99,7 @@ module Datadog
         end
 
         def itr
-          @itr ||= Datadog::CI.send(:itr_runner)
+          @test_optimisation ||= Datadog::CI.send(:test_optimisation)
         end
       end
     end

data/lib/datadog/ci/transport/adapters/net.rb

@@ -3,7 +3,9 @@
 require "datadog/core/transport/response"
 require "datadog/core/transport/ext"
 
+require_relative "net_http_client"
 require_relative "../gzip"
+require_relative "../../ext/telemetry"
 require_relative "../../ext/transport"
 
 module Datadog
@@ -26,7 +28,7 @@ module Datadog
           end
 
           def open(&block)
-            req = net_http_client.new(hostname, port)
+            req = NetHttpClient.original_net_http.new(hostname, port)
 
             req.use_ssl = ssl
             req.open_timeout = req.read_timeout = timeout
@@ -63,6 +65,8 @@ module Datadog
             include Datadog::Core::Transport::Response
 
             attr_reader :http_response
+            # Stats for telemetry
+            attr_accessor :duration_ms, :request_compressed, :request_size
 
             def initialize(http_response)
               @http_response = http_response
@@ -77,6 +81,10 @@ module Datadog
               @decompressed_payload = Gzip.decompress(http_response.body)
             end
 
+            def response_size
+              http_response.body.bytesize
+            end
+
             def header(name)
               http_response[name]
             end
@@ -86,7 +94,10 @@ module Datadog
             end
 
             def ok?
-              code.between?(200, 299)
+              http_code = code
+              return false if http_code.nil?
+
+              http_code.between?(200, 299)
             end
 
             def unsupported?
@@ -98,11 +109,17 @@ module Datadog
             end
 
             def client_error?
-              code.between?(400, 499)
+              http_code = code
+              return false if http_code.nil?
+
+              http_code.between?(400, 499)
             end
 
             def server_error?
-              code.between?(500, 599)
+              http_code = code
+              return false if http_code.nil?
+
+              http_code.between?(500, 599)
             end
 
             def gzipped_content?
@@ -119,17 +136,31 @@ module Datadog
               first_bytes.b == Ext::Transport::GZIP_MAGIC_NUMBER
             end
 
-            def inspect
-              "#{super}, http_response:#{http_response}"
+            def error
+              nil
             end
-          end
 
-          private
+            def telemetry_error_type
+              return nil if ok?
+
+              case error
+              when nil
+                Ext::Telemetry::ErrorType::STATUS_CODE
+              when Timeout::Error
+                Ext::Telemetry::ErrorType::TIMEOUT
+              else
+                Ext::Telemetry::ErrorType::NETWORK
+              end
+            end
 
-          def net_http_client
-            return ::Net::HTTP unless defined?(WebMock::HttpLibAdapters::NetHttpAdapter::OriginalNetHTTP)
+            # compatibility with Datadog::Tracing transport layer
+            def trace_count
+              0
+            end
 
-            WebMock::HttpLibAdapters::NetHttpAdapter::OriginalNetHTTP
+            def inspect
+              "#{super}, http_response:#{http_response}"
+            end
           end
         end
       end

data/lib/datadog/ci/transport/adapters/net_http_client.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Datadog
+  module CI
+    module Transport
+      module Adapters
+        module NetHttpClient
+          def self.original_net_http
+            return ::Net::HTTP unless defined?(WebMock::HttpLibAdapters::NetHttpAdapter::OriginalNetHTTP)
+
+            WebMock::HttpLibAdapters::NetHttpAdapter::OriginalNetHTTP
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/transport/adapters/telemetry_webmock_safe_adapter.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require_relative "net_http_client"
+
+module Datadog
+  module CI
+    module Transport
+      module Adapters
+        module TelemetryWebmockSafeAdapter
+          def self.included(base)
+            base.prepend(InstanceMethods)
+          end
+
+          module InstanceMethods
+            def open(&block)
+              req = NetHttpClient.original_net_http.new(@hostname, @port)
+
+              req.use_ssl = @ssl
+              req.open_timeout = req.read_timeout = @timeout
+
+              req.start(&block)
+            end
+          end
+        end
+      end
+    end
+  end
+end