datadog-ci 1.14.0 → 1.16.0

data/lib/datadog/ci/test_visibility/context.rb

@@ -4,8 +4,9 @@ require "datadog/tracing"
 require "datadog/tracing/contrib/component"
 require "datadog/tracing/trace_digest"
 
-require_relative "store/global"
-require_relative "store/local"
+require_relative "store/process"
+require_relative "store/fiber_local"
+require_relative "telemetry"
 
 require_relative "../ext/app_types"
 require_relative "../ext/environment"
@@ -29,9 +30,11 @@ module Datadog
       class Context
         attr_reader :total_tests_count, :tests_skipped_by_tia_count
 
-        def initialize
-          @local_context = Store::Local.new
-          @global_context = Store::Global.new
+        def initialize(test_visibility_component:)
+          @test_visibility_component = test_visibility_component
+
+          @fiber_local_context = Store::FiberLocal.new
+          @process_context = Store::Process.new
 
           @mutex = Mutex.new
 
@@ -40,18 +43,23 @@ module Datadog
         end
 
         def start_test_session(service: nil, tags: {})
-          @global_context.fetch_or_activate_test_session do
+          @process_context.fetch_or_activate_test_session do
             tracer_span = start_datadog_tracer_span(
               "test.session", build_tracing_span_options(service, Ext::AppTypes::TYPE_TEST_SESSION)
             )
             set_session_context(tags, tracer_span)
 
-            build_test_session(tracer_span, tags)
+            test_session = build_test_session(tracer_span, tags)
+
+            Telemetry.test_session_started(test_session)
+            Telemetry.event_created(test_session)
+
+            test_session
           end
         end
 
         def start_test_module(test_module_name, service: nil, tags: {})
-          @global_context.fetch_or_activate_test_module do
+          @process_context.fetch_or_activate_test_module do
             set_inherited_globals(tags)
             set_session_context(tags)
 
@@ -60,12 +68,16 @@ module Datadog
             )
             set_module_context(tags, tracer_span)
 
-            build_test_module(tracer_span, tags)
+            test_module = build_test_module(tracer_span, tags)
+
+            Telemetry.event_created(test_module)
+
+            test_module
           end
         end
 
         def start_test_suite(test_suite_name, service: nil, tags: {})
-          @global_context.fetch_or_activate_test_suite(test_suite_name) do
+          @process_context.fetch_or_activate_test_suite(test_suite_name) do
             set_inherited_globals(tags)
             set_session_context(tags)
             set_module_context(tags)
@@ -75,7 +87,11 @@ module Datadog
             )
             set_suite_context(tags, test_suite: tracer_span)
 
-            build_test_suite(tracer_span, tags)
+            test_suite = build_test_suite(tracer_span, tags)
+
+            Telemetry.event_created(test_suite)
+
+            test_suite
           end
         end
 
@@ -100,14 +116,14 @@ module Datadog
             start_datadog_tracer_span(test_name, span_options) do |tracer_span|
               test = build_test(tracer_span, tags)
 
-              @local_context.activate_test(test) do
+              @fiber_local_context.activate_test(test) do
                 block.call(test)
               end
             end
           else
             tracer_span = start_datadog_tracer_span(test_name, span_options)
             test = build_test(tracer_span, tags)
-            @local_context.activate_test(test)
+            @fiber_local_context.activate_test(test)
             test
           end
         end
@@ -136,43 +152,43 @@ module Datadog
         end
 
         def active_test
-          @local_context.active_test
+          @fiber_local_context.active_test
         end
 
         def active_test_session
-          @global_context.active_test_session
+          @process_context.active_test_session
         end
 
         def active_test_module
-          @global_context.active_test_module
+          @process_context.active_test_module
         end
 
         def active_test_suite(test_suite_name)
-          @global_context.active_test_suite(test_suite_name)
+          @process_context.active_test_suite(test_suite_name)
         end
 
         def single_active_test_suite
-          @global_context.fetch_single_test_suite
+          @process_context.fetch_single_test_suite
         end
 
         def stop_all_test_suites
-          @global_context.stop_all_test_suites
+          @process_context.stop_all_test_suites
         end
 
         def deactivate_test
-          @local_context.deactivate_test
+          @fiber_local_context.deactivate_test
         end
 
         def deactivate_test_session
-          @global_context.deactivate_test_session!
+          @process_context.deactivate_test_session!
         end
 
         def deactivate_test_module
-          @global_context.deactivate_test_module!
+          @process_context.deactivate_test_module!
         end
 
         def deactivate_test_suite(test_suite_name)
-          @global_context.deactivate_test_suite!(test_suite_name)
+          @process_context.deactivate_test_suite!(test_suite_name)
         end
 
         def incr_total_tests_count
@@ -231,20 +247,21 @@ module Datadog
 
         # PROPAGATING CONTEXT FROM TOP-LEVEL TO THE LOWER LEVELS
         def set_inherited_globals(tags)
-          # this code achieves the same as @global_context.inheritable_session_tags.merge(tags)
-          # but without allocating a new hash
-          @global_context.inheritable_session_tags.each do |key, value|
+          # Copy inheritable tags from the test session context to the provided tags
+          test_session_context&.inheritable_tags&.each do |key, value|
             tags[key] = value unless tags.key?(key)
           end
         end
 
         def set_session_context(tags, test_session = nil)
-          test_session ||= active_test_session
+          # we need to call TestVisibility::Component here because active test session might be remote
+          test_session ||= test_session_context
           tags[Ext::Test::TAG_TEST_SESSION_ID] = test_session.id.to_s if test_session
         end
 
         def set_module_context(tags, test_module = nil)
-          test_module ||= active_test_module
+          # we need to call TestVisibility::Component here because active test module might be remote
+          test_module ||= test_module_context
           if test_module
             tags[Ext::Test::TAG_TEST_MODULE_ID] = test_module.id.to_s
             tags[Ext::Test::TAG_MODULE] = test_module.name
@@ -281,11 +298,42 @@ module Datadog
         end
 
         def build_tracing_span_options(service, type, other_options = {})
-          other_options[:service] = service || @global_context.service
+          other_options[:service] = service || test_session_context&.service
           other_options[:type] = type
 
           other_options
         end
+
+        # one of:
+        #   1. Currrent test session from the Store::Process
+        #   2. Readonly copy of the remote test session (if test session was started by a parent process and local copy was created)
+        #   3. Remote test session as DRb::DRbObject link (in this case also local copy will be created)
+        def test_session_context
+          local_test_session = @process_context.active_test_session
+          return local_test_session if local_test_session
+
+          local_readonly_test_session = @process_context.readonly_test_session
+          return local_readonly_test_session if local_readonly_test_session
+
+          remote_test_session = @test_visibility_component.active_test_session
+          @process_context.set_readonly_test_session(remote_test_session)
+
+          remote_test_session
+        end
+
+        # works similar to test_session_context
+        def test_module_context
+          local_test_module = @process_context.active_test_module
+          return local_test_module if local_test_module
+
+          local_readonly_test_module = @process_context.readonly_test_module
+          return local_readonly_test_module if local_readonly_test_module
+
+          remote_test_module = @test_visibility_component.active_test_module
+          @process_context.set_readonly_test_module(remote_test_module)
+
+          remote_test_module
+        end
       end
     end
   end

data/lib/datadog/ci/test_visibility/null_component.rb

@@ -8,7 +8,9 @@ module Datadog
         def configure(_, _)
         end
 
-        def start_test_session(service: nil, tags: {}, estimated_total_tests_count: 0)
+        def start_test_session(
+          service: nil, tags: {}, estimated_total_tests_count: 0, distributed: false, local_test_suites_mode: true
+        )
           skip_tracing
         end
 
@@ -63,6 +65,10 @@ module Datadog
         def logical_test_session_name
         end
 
+        def client_process?
+          false
+        end
+
         private
 
         def skip_tracing(block = nil)

data/lib/datadog/ci/test_visibility/store/{local.rb → fiber_local.rb}

@@ -4,7 +4,7 @@ module Datadog
   module CI
     module TestVisibility
       module Store
-        class Local
+        class FiberLocal
           def initialize
             @key = :datadog_ci_active_test
 

data/lib/datadog/ci/test_visibility/store/{global.rb → process.rb}

@@ -1,11 +1,16 @@
 # frozen_string_literal: true
 
+require_relative "../../readonly_test_session"
+require_relative "../../readonly_test_module"
+
 module Datadog
   module CI
     module TestVisibility
       module Store
         # This context is shared between threads and represents the current test session and test module.
-        class Global
+        class Process
+          attr_reader :readonly_test_session, :readonly_test_module
+
           def initialize
             # we are using Monitor instead of Mutex because it is reentrant
             @mutex = Monitor.new
@@ -13,6 +18,11 @@ module Datadog
             @test_session = nil
             @test_module = nil
             @test_suites = {}
+
+            # small copies of id, name and some tags: store them in the current process to set session/module context
+            # for any spans faster
+            @readonly_test_session = nil
+            @readonly_test_module = nil
           end
 
           def fetch_or_activate_test_suite(test_suite_name, &block)
@@ -53,12 +63,6 @@ module Datadog
             @mutex.synchronize { @test_suites[test_suite_name] }
           end
 
-          def service
-            @mutex.synchronize do
-              @test_session&.service
-            end
-          end
-
           def stop_all_test_suites
             @mutex.synchronize do
               @test_suites.each_value(&:finish)
@@ -66,17 +70,6 @@ module Datadog
             end
           end
 
-          def inheritable_session_tags
-            @mutex.synchronize do
-              test_session = @test_session
-              if test_session
-                test_session.inheritable_tags
-              else
-                {}
-              end
-            end
-          end
-
           def deactivate_test_session!
             @mutex.synchronize { @test_session = nil }
           end
@@ -88,6 +81,18 @@ module Datadog
           def deactivate_test_suite!(test_suite_name)
             @mutex.synchronize { @test_suites.delete(test_suite_name) }
           end
+
+          def set_readonly_test_session(remote_test_session)
+            return if remote_test_session.nil?
+
+            @readonly_test_session = Datadog::CI::ReadonlyTestSession.new(remote_test_session)
+          end
+
+          def set_readonly_test_module(remote_test_module)
+            return if remote_test_module.nil?
+
+            @readonly_test_module = Datadog::CI::ReadonlyTestModule.new(remote_test_module)
+          end
         end
       end
     end

data/lib/datadog/ci/test_visibility/transport.rb

@@ -4,7 +4,6 @@ require "datadog/core/environment/identity"
 require "datadog/core/telemetry/logging"
 require "datadog/core/utils/only_once"
 
-require_relative "capabilities"
 require_relative "serializers/factories/test_level"
 
 require_relative "../ext/app_types"
@@ -117,7 +116,7 @@ module Datadog
           packer.write("library_version")
           packer.write(Datadog::CI::VERSION::STRING)
 
-          library_capabilities_tags = Capabilities.tags
+          library_capabilities_tags = Ext::Test::LibraryCapabilities::CAPABILITY_VERSIONS
 
           Ext::AppTypes::CI_SPAN_TYPES.each do |ci_span_type|
             packer.write(ci_span_type)

data/lib/datadog/ci/transport/api/agentless.rb

@@ -10,11 +10,12 @@ module Datadog
         class Agentless < Base
           attr_reader :api_key
 
-          def initialize(api_key:, citestcycle_url:, api_url:, citestcov_url:)
+          def initialize(api_key:, citestcycle_url:, api_url:, citestcov_url:, logs_intake_url:)
             @api_key = api_key
             @citestcycle_http = build_http_client(citestcycle_url, compress: true)
             @api_http = build_http_client(api_url, compress: false)
             @citestcov_http = build_http_client(citestcov_url, compress: true)
+            @logs_intake_http = build_http_client(logs_intake_url, compress: true)
           end
 
           def citestcycle_request(path:, payload:, headers: {}, verb: "post")
@@ -42,6 +43,12 @@ module Datadog
             perform_request(@citestcov_http, path: path, payload: @citestcov_payload, headers: headers, verb: verb)
           end
 
+          def logs_intake_request(path:, payload:, headers: {}, verb: "post")
+            super
+
+            perform_request(@logs_intake_http, path: path, payload: payload, headers: headers, verb: verb)
+          end
+
           private
 
           def perform_request(http_client, path:, payload:, headers:, verb:, accept_compressed_response: false)

data/lib/datadog/ci/transport/api/base.rb

@@ -38,6 +38,10 @@ module Datadog
             ].join("\r\n")
           end
 
+          def logs_intake_request(path:, payload:, headers: {}, verb: "post")
+            headers[Ext::Transport::HEADER_CONTENT_TYPE] ||= Ext::Transport::CONTENT_TYPE_JSON
+          end
+
           def headers_with_default(headers)
             request_headers = default_headers
             request_headers.merge!(headers)

data/lib/datadog/ci/transport/api/builder.rb

@@ -27,11 +27,15 @@ module Datadog
             citestcov_url = settings.ci.agentless_url ||
               "https://#{Ext::Transport::TEST_COVERAGE_INTAKE_HOST_PREFIX}.#{dd_site}:443"
 
+            logs_intake_url = settings.ci.agentless_url ||
+              "https://#{Ext::Transport::LOGS_INTAKE_HOST_PREFIX}.#{dd_site}:443"
+
             Agentless.new(
               api_key: settings.api_key,
               citestcycle_url: citestcycle_url,
               api_url: api_url,
-              citestcov_url: citestcov_url
+              citestcov_url: citestcov_url,
+              logs_intake_url: logs_intake_url
             )
           end
 

data/lib/datadog/ci/transport/api/evp_proxy.rb

@@ -46,6 +46,10 @@ module Datadog
             perform_request(@agent_intake_http, path: path, payload: @citestcov_payload, headers: headers, verb: verb)
           end
 
+          def logs_intake_request(path:, payload:, headers: {}, verb: "post")
+            raise NotImplementedError, "Logs intake is not supported in EVP proxy mode"
+          end
+
           private
 
           def perform_request(http_client, path:, payload:, headers:, verb:)

data/lib/datadog/ci/transport/http.rb

@@ -29,7 +29,7 @@ module Datadog
           @host = host
           @port = port
           @timeout = timeout
-          @ssl = ssl.nil? ? true : ssl
+          @ssl = ssl.nil? || ssl
           @compress = compress.nil? ? false : compress
         end
 

data/lib/datadog/ci/utils/file_storage.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "tempfile"
+
+module Datadog
+  module CI
+    module Utils
+      # FileStorage module provides functionality for storing and retrieving arbitrary Ruby objects in a temp file
+      # to share them between processes.
+      module FileStorage
+        TEMP_DIR = File.join(Dir.tmpdir, "datadog-ci-storage")
+
+        def self.store(key, value)
+          ensure_temp_dir_exists
+          file_path = file_path_for(key)
+
+          File.binwrite(file_path, Marshal.dump(value))
+
+          true
+        rescue => e
+          Datadog.logger.error("Failed to store data for key '#{key}': #{e.class} - #{e.message}")
+          false
+        end
+
+        def self.retrieve(key)
+          file_path = file_path_for(key)
+          return nil unless File.exist?(file_path)
+
+          Marshal.load(File.binread(file_path))
+        rescue => e
+          Datadog.logger.error("Failed to retrieve data for key '#{key}': #{e.class} - #{e.message}")
+          nil
+        end
+
+        def self.cleanup
+          return false unless Dir.exist?(TEMP_DIR)
+
+          FileUtils.rm_rf(TEMP_DIR)
+          true
+        rescue => e
+          Datadog.logger.error("Failed to cleanup storage directory: #{e.class} - #{e.message}")
+          false
+        end
+
+        def self.ensure_temp_dir_exists
+          FileUtils.mkdir_p(TEMP_DIR) unless Dir.exist?(TEMP_DIR)
+        end
+
+        def self.file_path_for(key)
+          sanitized_key = key.to_s.gsub(/[^a-zA-Z0-9_-]/, "_")
+          File.join(TEMP_DIR, "dd-ci-#{sanitized_key}.dat")
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/utils/stateful.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+require_relative "file_storage"
+
+module Datadog
+  module CI
+    module Utils
+      # Module for components that need to persist and restore state
+      module Stateful
+        # Store component state
+        def store_component_state
+          state = serialize_state
+
+          res = Utils::FileStorage.store(storage_key, state)
+          Datadog.logger.debug { "Stored component state (key=#{storage_key}): #{res}" }
+
+          res
+        end
+
+        # Load component state
+        def load_component_state
+          test_visibility_component = Datadog.send(:components).test_visibility
+          return false unless test_visibility_component.client_process?
+
+          state = Utils::FileStorage.retrieve(storage_key)
+          unless state
+            Datadog.logger.debug { "No component state found in file storage (key=#{storage_key})" }
+            return false
+          end
+
+          restore_state(state)
+          Datadog.logger.debug { "Loaded component state from file storage (key=#{storage_key})" }
+
+          true
+        end
+
+        # These methods must be implemented by including classes
+        def serialize_state
+          raise NotImplementedError, "Components must implement #serialize_state"
+        end
+
+        def restore_state(state)
+          raise NotImplementedError, "Components must implement #restore_state"
+        end
+
+        def storage_key
+          raise NotImplementedError, "Components must implement #storage_key"
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/version.rb

@@ -4,7 +4,7 @@ module Datadog
   module CI
     module VERSION
       MAJOR = 1
-      MINOR = 14
+      MINOR = 16
       PATCH = 0
       PRE = nil
       BUILD = nil

data/lib/datadog/ci.rb

@@ -415,19 +415,23 @@ end
 
 # Integrations
 
-# Test frameworks (manual instrumentation)
+# Test frameworks
 require_relative "ci/contrib/cucumber/integration"
 require_relative "ci/contrib/minitest/integration"
 require_relative "ci/contrib/rspec/integration"
 
-# Test runners (instrumented automatically when corresponding frameworks are instrumented)
+# Test runners
 require_relative "ci/contrib/knapsack/integration"
 require_relative "ci/contrib/ciqueue/integration"
+require_relative "ci/contrib/parallel_tests/integration"
 
-# Additional test libraries (auto instrumented later on test session start)
+# Additional test libraries (auto instrumented on test session start)
 require_relative "ci/contrib/selenium/integration"
 require_relative "ci/contrib/cuprite/integration"
 require_relative "ci/contrib/simplecov/integration"
+require_relative "ci/contrib/activesupport/integration"
+require_relative "ci/contrib/lograge/integration"
+require_relative "ci/contrib/semantic_logger/integration"
 
 # Configuration extensions
 require_relative "ci/configuration/extensions"