ddtrace 1.4.2 → 1.19.0

data/lib/datadog/core/remote/transport/http.rb

@@ -0,0 +1,179 @@
+# frozen_string_literal: true
+
+require 'uri'
+
+require_relative '../../environment/container'
+require_relative '../../environment/ext'
+require_relative '../../transport/ext'
+require_relative '../../transport/http/adapters/net'
+require_relative '../../transport/http/adapters/unix_socket'
+require_relative '../../transport/http/adapters/test'
+
+# TODO: Improve negotiation to allow per endpoint selection
+#
+# Since endpoint negotiation happens at the `API::Spec` level there can not be
+# a mix of endpoints at various versions or versionless without describing all
+# the possible combinations as specs. See http/api.
+#
+# Below should be:
+# require_relative '../../transport/http/api'
+require_relative 'http/api'
+
+# TODO: Decouple transport/http/builder
+#
+# See http/builder
+#
+# Below should be:
+# require_relative '../../transport/http/builder'
+require_relative 'http/builder'
+
+# TODO: Decouple transport/http
+#
+# Because a new transport is required for every (API, Client, Transport)
+# triplet and endpoints cannot be negotiated independently, there can not be a
+# single `default` transport, but only endpoint-specific ones.
+
+module Datadog
+  module Core
+    module Remote
+      module Transport
+        # Namespace for HTTP transport components
+        module HTTP
+          # NOTE: Due to... legacy reasons... This class likes having a default `AgentSettings` instance to fall back to.
+          # Because we generate this instance with an empty instance of `Settings`, the resulting `AgentSettings` below
+          # represents only settings specified via environment variables + the usual defaults.
+          #
+          # DO NOT USE THIS IN NEW CODE, as it ignores any settings specified by users via `Datadog.configure`.
+          DO_NOT_USE_ENVIRONMENT_AGENT_SETTINGS = Datadog::Core::Configuration::AgentSettingsResolver.call(
+            Datadog::Core::Configuration::Settings.new,
+            logger: nil,
+          )
+
+          module_function
+
+          # Builds a new Transport::HTTP::Client
+          def new(klass, &block)
+            Builder.new(&block).to_transport(klass)
+          end
+
+          # Builds a new Transport::HTTP::Client with default settings
+          # Pass a block to override any settings.
+          def root(
+            agent_settings: DO_NOT_USE_ENVIRONMENT_AGENT_SETTINGS,
+            **options
+          )
+            new(Core::Remote::Transport::Negotiation::Transport) do |transport|
+              transport.adapter(agent_settings)
+              transport.headers(default_headers)
+
+              apis = API.defaults
+
+              transport.api API::ROOT, apis[API::ROOT]
+
+              # Apply any settings given by options
+              unless options.empty?
+                transport.default_api = options[:api_version] if options.key?(:api_version)
+                transport.headers options[:headers] if options.key?(:headers)
+              end
+
+              if agent_settings.deprecated_for_removal_transport_configuration_proc
+                agent_settings.deprecated_for_removal_transport_configuration_proc.call(transport)
+              end
+
+              # Call block to apply any customization, if provided
+              yield(transport) if block_given?
+            end
+          end
+
+          # Builds a new Transport::HTTP::Client with default settings
+          # Pass a block to override any settings.
+          def v7(
+            agent_settings: DO_NOT_USE_ENVIRONMENT_AGENT_SETTINGS,
+            **options
+          )
+            new(Core::Remote::Transport::Config::Transport) do |transport|
+              transport.adapter(agent_settings)
+              transport.headers default_headers
+
+              apis = API.defaults
+
+              transport.api API::V7, apis[API::V7]
+
+              # Apply any settings given by options
+              unless options.empty?
+                transport.default_api = options[:api_version] if options.key?(:api_version)
+                transport.headers options[:headers] if options.key?(:headers)
+              end
+
+              if agent_settings.deprecated_for_removal_transport_configuration_proc
+                agent_settings.deprecated_for_removal_transport_configuration_proc.call(transport)
+              end
+
+              # Call block to apply any customization, if provided
+              yield(transport) if block_given?
+            end
+          end
+
+          def default_headers
+            {
+              Datadog::Core::Transport::Ext::HTTP::HEADER_CLIENT_COMPUTED_TOP_LEVEL => '1',
+              Datadog::Core::Transport::Ext::HTTP::HEADER_META_LANG => Datadog::Core::Environment::Ext::LANG,
+              Datadog::Core::Transport::Ext::HTTP::HEADER_META_LANG_VERSION =>
+                Datadog::Core::Environment::Ext::LANG_VERSION,
+              Datadog::Core::Transport::Ext::HTTP::HEADER_META_LANG_INTERPRETER =>
+                Datadog::Core::Environment::Ext::LANG_INTERPRETER,
+              Datadog::Core::Transport::Ext::HTTP::HEADER_META_TRACER_VERSION =>
+                Datadog::Core::Environment::Ext::TRACER_VERSION
+            }.tap do |headers|
+              # Add container ID, if present.
+              container_id = Datadog::Core::Environment::Container.container_id
+              headers[Datadog::Core::Transport::Ext::HTTP::HEADER_CONTAINER_ID] = container_id unless container_id.nil?
+            end
+          end
+
+          def default_adapter
+            Datadog::Core::Transport::Ext::HTTP::ADAPTER
+          end
+
+          def default_hostname(logger: Datadog.logger)
+            logger.warn(
+              'Deprecated for removal: Using #default_hostname for configuration is deprecated and will ' \
+                'be removed on a future ddtrace release.'
+            )
+
+            DO_NOT_USE_ENVIRONMENT_AGENT_SETTINGS.hostname
+          end
+
+          def default_port(logger: Datadog.logger)
+            logger.warn(
+              'Deprecated for removal: Using #default_hostname for configuration is deprecated and will ' \
+                'be removed on a future ddtrace release.'
+            )
+
+            DO_NOT_USE_ENVIRONMENT_AGENT_SETTINGS.port
+          end
+
+          def default_url(logger: Datadog.logger)
+            logger.warn(
+              'Deprecated for removal: Using #default_url for configuration is deprecated and will ' \
+                'be removed on a future ddtrace release.'
+            )
+
+            nil
+          end
+
+          # Add adapters to registry
+          Builder::REGISTRY.set(Datadog::Core::Transport::HTTP::Adapters::Net, Datadog::Core::Transport::Ext::HTTP::ADAPTER)
+          Builder::REGISTRY.set(
+            Datadog::Core::Transport::HTTP::Adapters::Test,
+            Datadog::Core::Transport::Ext::Test::ADAPTER
+          )
+          Builder::REGISTRY.set(
+            Datadog::Core::Transport::HTTP::Adapters::UnixSocket,
+            Datadog::Core::Transport::Ext::UnixSocket::ADAPTER
+          )
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/remote/transport/negotiation.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+require_relative '../../../core/transport/request'
+
+# TODO: Resolve conceptual conundrum
+#
+# It seems that through naming of `Transport::Traces::Transport` the transport
+# is specific to traces, which kind of matches the almost-generic-but-not-quite
+# implementation.
+#
+# This may be why `Transport::Traces::Transport` negotiates only considering
+# the `/vX/traces` path, but here we don't negotiate since we are at the root.
+#
+# In turn this means that API::Spec cannot describe multiple roots, or even
+# endpoints that happen to differ in version.
+#
+# Concepts such as Spec, API, Endpoint, and Transport should be clarified
+# before attempting further refactoring here, to attempt to resolve whether a
+# Transport - via its negotiated Spec - describes a function (implemented as
+# one or more endpoints) or whether the Spec describes the API towards the
+# agent as a whole, morphing through negotiation into the best available
+# version for each endpoint.
+
+module Datadog
+  module Core
+    module Remote
+      module Transport
+        module Negotiation
+          # Negotiation request
+          class Request < Datadog::Core::Transport::Request
+          end
+
+          # Negotiation response
+          module Response
+            attr_reader :version, :endpoints, :config
+          end
+
+          # Negotiation transport
+          class Transport
+            attr_reader :client, :apis, :default_api, :current_api_id
+
+            def initialize(apis, default_api)
+              @apis = apis
+
+              @client = HTTP::Client.new(current_api)
+            end
+
+            def send_info
+              request = Request.new
+
+              @client.send_info_payload(request)
+            end
+
+            def current_api
+              @apis[HTTP::API::ROOT]
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/remote/worker.rb

@@ -0,0 +1,99 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Core
+    module Remote
+      # Worker executes a block every interval on a separate Thread
+      class Worker
+        def initialize(interval:, &block)
+          @mutex = Mutex.new
+          @thr = nil
+
+          @starting = false
+          @stopping = false
+          @started = false
+
+          @interval = interval
+          raise ArgumentError, 'can not initialize a worker without a block' unless block
+
+          @block = block
+        end
+
+        def start
+          Datadog.logger.debug { 'remote worker starting' }
+
+          acquire_lock
+
+          return if @starting || @started
+
+          @starting = true
+
+          thread = Thread.new { poll(@interval) }
+          thread.name = self.class.name unless Gem::Version.new(RUBY_VERSION) < Gem::Version.new('2.3')
+          thread.thread_variable_set(:fork_safe, true)
+          @thr = thread
+
+          @started = true
+          @starting = false
+
+          Datadog.logger.debug { 'remote worker started' }
+        ensure
+          release_lock
+        end
+
+        def stop
+          Datadog.logger.debug { 'remote worker stopping' }
+
+          acquire_lock
+
+          @stopping = true
+
+          thread = @thr
+
+          if thread
+            thread.kill
+            thread.join
+          end
+
+          @started = false
+          @stopping = false
+          @thr = nil
+
+          Datadog.logger.debug { 'remote worker stopped' }
+        ensure
+          release_lock
+        end
+
+        def started?
+          @started
+        end
+
+        private
+
+        def acquire_lock
+          @mutex.lock
+        end
+
+        def release_lock
+          @mutex.unlock
+        end
+
+        def poll(interval)
+          loop do
+            break unless @mutex.synchronize { @starting || @started }
+
+            call
+
+            sleep(interval)
+          end
+        end
+
+        def call
+          Datadog.logger.debug { 'remote worker perform' }
+
+          @block.call
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/remote.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Core
+    # Module to interact with core remote component
+    module Remote
+      class << self
+        def active_remote
+          remote
+        end
+
+        private
+
+        def components
+          Datadog.send(:components)
+        end
+
+        def remote
+          components.remote
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/runtime/ext.rb

@@ -1,26 +1,36 @@
-# typed: true
+# frozen_string_literal: true
 
 module Datadog
   module Core
     module Runtime
       # @public_api
       module Ext
-        TAG_ID = 'runtime-id'.freeze
-        TAG_LANG = 'language'.freeze
-        TAG_PID = 'system.pid'.freeze
+        TAG_ID = 'runtime-id'
+        TAG_LANG = 'language'
+        TAG_PROCESS_ID = 'process_id'
 
         # Metrics
         # @public_api
         module Metrics
-          ENV_ENABLED = 'DD_RUNTIME_METRICS_ENABLED'.freeze
+          ENV_ENABLED = 'DD_RUNTIME_METRICS_ENABLED'
 
-          METRIC_CLASS_COUNT = 'runtime.ruby.class_count'.freeze
-          METRIC_GC_PREFIX = 'runtime.ruby.gc'.freeze
-          METRIC_THREAD_COUNT = 'runtime.ruby.thread_count'.freeze
-          METRIC_GLOBAL_CONSTANT_STATE = 'runtime.ruby.global_constant_state'.freeze
-          METRIC_GLOBAL_METHOD_STATE = 'runtime.ruby.global_method_state'.freeze
+          METRIC_CLASS_COUNT = 'runtime.ruby.class_count'
+          METRIC_GC_PREFIX = 'runtime.ruby.gc'
+          METRIC_THREAD_COUNT = 'runtime.ruby.thread_count'
+          METRIC_GLOBAL_CONSTANT_STATE = 'runtime.ruby.global_constant_state'
+          METRIC_GLOBAL_METHOD_STATE = 'runtime.ruby.global_method_state'
+          METRIC_CONSTANT_CACHE_INVALIDATIONS = 'runtime.ruby.constant_cache_invalidations'
+          METRIC_CONSTANT_CACHE_MISSES = 'runtime.ruby.constant_cache_misses'
+          METRIC_YJIT_CODE_GC_COUNT = 'runtime.ruby.yjit.code_gc_count'
+          METRIC_YJIT_CODE_REGION_SIZE = 'runtime.ruby.yjit.code_region_size'
+          METRIC_YJIT_FREED_CODE_SIZE = 'runtime.ruby.yjit.freed_code_size'
+          METRIC_YJIT_FREED_PAGE_COUNT = 'runtime.ruby.yjit.freed_page_count'
+          METRIC_YJIT_INLINE_CODE_SIZE = 'runtime.ruby.yjit.inline_code_size'
+          METRIC_YJIT_LIVE_PAGE_COUNT = 'runtime.ruby.yjit.live_page_count'
+          METRIC_YJIT_OBJECT_SHAPE_COUNT = 'runtime.ruby.yjit.object_shape_count'
+          METRIC_YJIT_OUTLINED_CODE_SIZE = 'runtime.ruby.yjit.outlined_code_size'
 
-          TAG_SERVICE = 'service'.freeze
+          TAG_SERVICE = 'service'
         end
       end
     end

data/lib/datadog/core/runtime/metrics.rb

@@ -1,5 +1,3 @@
-# typed: true
-
 require_relative 'ext'
 
 require_relative '../metrics/client'
@@ -7,6 +5,7 @@ require_relative '../environment/class_count'
 require_relative '../environment/gc'
 require_relative '../environment/thread_count'
 require_relative '../environment/vm_cache'
+require_relative '../environment/yjit'
 
 module Datadog
   module Core
@@ -57,17 +56,31 @@ module Datadog
 
           try_flush do
             if Core::Environment::VMCache.available?
-              gauge(
+              # Only on Ruby < 3.2
+              gauge_if_not_nil(
                 Core::Runtime::Ext::Metrics::METRIC_GLOBAL_CONSTANT_STATE,
                 Core::Environment::VMCache.global_constant_state
               )
 
-              # global_method_state is not available since Ruby >= 3.0,
-              # as method caching was moved to a per-class basis.
-              global_method_state = Core::Environment::VMCache.global_method_state
-              gauge(Core::Runtime::Ext::Metrics::METRIC_GLOBAL_METHOD_STATE, global_method_state) if global_method_state
+              # Only on Ruby 2.x
+              gauge_if_not_nil(
+                Core::Runtime::Ext::Metrics::METRIC_GLOBAL_METHOD_STATE,
+                Core::Environment::VMCache.global_method_state
+              )
+
+              # Only on Ruby >= 3.2
+              gauge_if_not_nil(
+                Core::Runtime::Ext::Metrics::METRIC_CONSTANT_CACHE_INVALIDATIONS,
+                Core::Environment::VMCache.constant_cache_invalidations
+              )
+              gauge_if_not_nil(
+                Core::Runtime::Ext::Metrics::METRIC_CONSTANT_CACHE_MISSES,
+                Core::Environment::VMCache.constant_cache_misses
+              )
             end
           end
+
+          flush_yjit_stats
         end
 
         def gc_metrics
@@ -120,6 +133,50 @@ module Datadog
         def to_metric_name(str)
           str.downcase.gsub(/[-\s]/, '_')
         end
+
+        def gauge_if_not_nil(metric_name, metric_value)
+          gauge(metric_name, metric_value) if metric_value
+        end
+
+        def flush_yjit_stats
+          # Only on Ruby >= 3.2
+          try_flush do
+            if Core::Environment::YJIT.available?
+              gauge_if_not_nil(
+                Core::Runtime::Ext::Metrics::METRIC_YJIT_CODE_GC_COUNT,
+                Core::Environment::YJIT.code_gc_count
+              )
+              gauge_if_not_nil(
+                Core::Runtime::Ext::Metrics::METRIC_YJIT_CODE_REGION_SIZE,
+                Core::Environment::YJIT.code_region_size
+              )
+              gauge_if_not_nil(
+                Core::Runtime::Ext::Metrics::METRIC_YJIT_FREED_CODE_SIZE,
+                Core::Environment::YJIT.freed_code_size
+              )
+              gauge_if_not_nil(
+                Core::Runtime::Ext::Metrics::METRIC_YJIT_FREED_PAGE_COUNT,
+                Core::Environment::YJIT.freed_page_count
+              )
+              gauge_if_not_nil(
+                Core::Runtime::Ext::Metrics::METRIC_YJIT_INLINE_CODE_SIZE,
+                Core::Environment::YJIT.inline_code_size
+              )
+              gauge_if_not_nil(
+                Core::Runtime::Ext::Metrics::METRIC_YJIT_LIVE_PAGE_COUNT,
+                Core::Environment::YJIT.live_page_count
+              )
+              gauge_if_not_nil(
+                Core::Runtime::Ext::Metrics::METRIC_YJIT_OBJECT_SHAPE_COUNT,
+                Core::Environment::YJIT.object_shape_count
+              )
+              gauge_if_not_nil(
+                Core::Runtime::Ext::Metrics::METRIC_YJIT_OUTLINED_CODE_SIZE,
+                Core::Environment::YJIT.outlined_code_size
+              )
+            end
+          end
+        end
       end
     end
   end

data/lib/datadog/core/telemetry/client.rb

@@ -1,4 +1,4 @@
-# typed: true
+# frozen_string_literal: true
 
 require_relative 'emitter'
 require_relative 'heartbeat'
@@ -7,7 +7,7 @@ require_relative '../utils/forking'
 module Datadog
   module Core
     module Telemetry
-      # Telemetry entrypoint, coordinates sending telemetry events at various points in app lifecyle
+      # Telemetry entrypoint, coordinates sending telemetry events at various points in app lifecycle.
       class Client
         attr_reader \
           :emitter,
@@ -18,12 +18,13 @@ module Datadog
         include Core::Utils::Forking
 
         # @param enabled [Boolean] Determines whether telemetry events should be sent to the API
-        def initialize(enabled: true)
+        # @param heartbeat_interval_seconds [Float] How frequently heartbeats will be reported, in seconds.
+        def initialize(heartbeat_interval_seconds:, enabled: true)
           @enabled = enabled
           @emitter = Emitter.new
           @stopped = false
           @unsupported = false
-          @worker = Telemetry::Heartbeat.new(enabled: @enabled) do
+          @worker = Telemetry::Heartbeat.new(enabled: @enabled, heartbeat_interval_seconds: heartbeat_interval_seconds) do
             heartbeat!
           end
         end
@@ -66,6 +67,13 @@ module Datadog
           @emitter.request(:'app-integrations-change')
         end
 
+        # Report configuration changes caused by Remote Configuration.
+        def client_configuration_change!(changes)
+          return if !@enabled || forked?
+
+          @emitter.request('app-client-configuration-change', data: { changes: changes, origin: 'remote_config' })
+        end
+
         private
 
         def heartbeat!

data/lib/datadog/core/telemetry/collector.rb

@@ -1,26 +1,26 @@
 # frozen_string_literal: true
 
-# typed: true
-
 require 'etc'
 
 require_relative '../configuration/agent_settings_resolver'
 require_relative '../environment/ext'
 require_relative '../environment/platform'
+require_relative '../utils/hash'
 require_relative 'v1/application'
 require_relative 'v1/dependency'
 require_relative 'v1/host'
+require_relative 'v1/install_signature'
 require_relative 'v1/integration'
 require_relative 'v1/product'
-require_relative '../../../ddtrace/transport/ext'
+require_relative '../transport/ext'
 
 module Datadog
   module Core
     module Telemetry
       # Module defining methods for collecting metadata for telemetry
-      # rubocop:disable Metrics/ModuleLength
       module Collector
         include Datadog::Core::Configuration
+        using Core::Utils::Hash::Refinement
 
         # Forms a hash of configuration key value pairs to be sent in the additional payload
         def additional_payload
@@ -44,19 +44,29 @@ module Datadog
 
         # Forms a hash of standard key value pairs to be sent in the app-started event configuration
         def configurations
-          configurations = {
-            DD_AGENT_HOST: Datadog.configuration.agent.host,
+          config = Datadog.configuration
+          hash = {
+            DD_AGENT_HOST: config.agent.host,
             DD_AGENT_TRANSPORT: agent_transport,
-            DD_TRACE_SAMPLE_RATE: format_configuration_value(Datadog.configuration.tracing.sampling.default_rate),
+            DD_TRACE_SAMPLE_RATE: format_configuration_value(config.tracing.sampling.default_rate),
+            DD_TRACE_REMOVE_INTEGRATION_SERVICE_NAMES_ENABLED: config.tracing.contrib.global_default_service_name.enabled
           }
-          compact_hash(configurations)
+          peer_service_mapping_str = ''
+          unless config.tracing.contrib.peer_service_mapping.empty?
+            peer_service_mapping = config.tracing.contrib.peer_service_mapping
+            peer_service_mapping_str = peer_service_mapping.map { |key, value| "#{key}:#{value}" }.join(',')
+          end
+          hash[:DD_TRACE_PEER_SERVICE_MAPPING] = peer_service_mapping_str
+          hash.compact!
+          hash
         end
 
         # Forms a telemetry app-started dependencies object
         def dependencies
           Gem.loaded_specs.collect do |name, loaded_gem|
             Datadog::Core::Telemetry::V1::Dependency.new(
-              name: name, version: loaded_gem.version.to_s, hash: loaded_gem.hash.to_s
+              # `hash` should be used when `version` is not available
+              name: name, version: loaded_gem.version.to_s, hash: nil
             )
           end
         end
@@ -72,6 +82,15 @@ module Datadog
           )
         end
 
+        # Forms a telemetry app-started install_signature object
+        def install_signature
+          Telemetry::V1::InstallSignature.new(
+            install_id: Datadog.configuration.dig('telemetry', 'install_id'),
+            install_type: Datadog.configuration.dig('telemetry', 'install_type'),
+            install_time: Datadog.configuration.dig('telemetry', 'install_time'),
+          )
+        end
+
         # Forms a telemetry app-started integrations object
         def integrations
           Datadog.registry.map do |integration|
@@ -136,7 +155,10 @@ module Datadog
             format_configuration_value(configuration.tracing.writer_options[:flush_interval])
           options['logger.instance'] = configuration.logger.instance.class.to_s
           options['appsec.enabled'] = configuration.dig('appsec', 'enabled') if configuration.respond_to?('appsec')
-          compact_hash(options)
+          options['tracing.opentelemetry.enabled'] = !defined?(Datadog::OpenTelemetry::LOADED).nil?
+          options['tracing.opentracing.enabled'] = !defined?(Datadog::OpenTracer::LOADED).nil?
+          options.compact!
+          options
         end
 
         def format_configuration_value(value)
@@ -149,11 +171,6 @@ module Datadog
           end
         end
 
-        # Manual implementation of hash.compact used because it is unsupported by older Ruby versions (<2.4)
-        def compact_hash(hash)
-          hash.delete_if { |_k, v| v.nil? }
-        end
-
         def env
           Datadog.configuration.env
         end
@@ -184,7 +201,7 @@ module Datadog
 
         def agent_transport
           adapter = Core::Configuration::AgentSettingsResolver.call(Datadog.configuration).adapter
-          if adapter == Datadog::Transport::Ext::UnixSocket::ADAPTER
+          if adapter == Datadog::Core::Transport::Ext::UnixSocket::ADAPTER
             'UDS'
           else
             'TCP'
@@ -224,11 +241,10 @@ module Datadog
             desc += ", Patchable? #{integration.klass.class.patchable?}"
             desc
           else
-            patch_error_result.to_s
+            patch_error_result.compact.to_s
           end
         end
       end
-      # rubocop:enable Metrics/ModuleLength
     end
   end
 end