ddtrace 1.18.0 → 1.20.0

data/lib/datadog/core/configuration/settings.rb

@@ -314,34 +314,89 @@ module Datadog
 
             # Can be used to enable/disable the Datadog::Profiling.allocation_count feature.
             #
-            # This feature is safe and enabled by default on Ruby 2.x, but has a few caveats on Ruby 3.x.
+            # This feature is now controlled via {:experimental_allocation_enabled}
+            option :allocation_counting_enabled do |o|
+              o.after_set do
+                Datadog.logger.warn(
+                  'The profiling.advanced.allocation_counting_enabled setting has been deprecated for removal and no ' \
+                  'longer does anything. Please remove it from your Datadog.configure block. ' \
+                  'Allocation counting is now controlled by the `experimental_allocation_enabled` setting instead.'
+                )
+              end
+            end
+
+            # Can be used to enable/disable collection of allocation profiles.
             #
-            # Caveat 1 (severe):
-            # On Ruby versions 3.0 (all), 3.1.0 to 3.1.3, and 3.2.0 to 3.2.2 this is disabled by default because it
-            # can trigger a VM bug that causes a segmentation fault during garbage collection of Ractors
-            # (https://bugs.ruby-lang.org/issues/18464). We don't recommend using this feature on such Rubies.
-            # This bug is fixed on Ruby versions 3.1.4, 3.2.3 and 3.3.0.
+            # This feature is alpha and disabled by default
             #
-            # Caveat 2 (annoyance):
-            # On all known versions of Ruby 3.x, due to https://bugs.ruby-lang.org/issues/19112, when a ractor gets
-            # garbage collected, Ruby will disable all active tracepoints, which this feature internally relies on.
-            # Thus this feature is only usable if you're not using Ractors.
+            # @warn This feature is not supported/safe in all Rubies. Details in {Datadog::Profiling::Component} but
+            #       in summary, this should be supported on Ruby 2.x, 3.1.4+, 3.2.3+ and 3.3.0+. Enabling it on
+            #       unsupported Rubies may result in unexpected behaviour, including crashes.
             #
-            # Caveat 3 (severe):
-            # Ruby 3.2.0 to 3.2.2 have a bug in the newobj tracepoint (https://bugs.ruby-lang.org/issues/19482,
-            # https://github.com/ruby/ruby/pull/7464) so that's an extra reason why it's not safe on those Rubies.
-            # This bug is fixed on Ruby versions 3.2.3 and 3.3.0.
+            # @default `DD_PROFILING_EXPERIMENTAL_ALLOCATION_ENABLED` environment variable as a boolean, otherwise `false`
+            option :experimental_allocation_enabled do |o|
+              o.type :bool
+              o.env 'DD_PROFILING_EXPERIMENTAL_ALLOCATION_ENABLED'
+              o.default false
+            end
+
+            # Can be used to enable/disable the collection of heap profiles.
             #
-            # @default `true` on Ruby 2.x and 3.1.4+, 3.2.3+ and 3.3.0+; `false` for Ruby 3.0 and unpatched Rubies.
-            option :allocation_counting_enabled do |o|
-              o.default do
-                RUBY_VERSION.start_with?('2.') ||
-                  (RUBY_VERSION.start_with?('3.1.') && RUBY_VERSION >= '3.1.4') ||
-                  (RUBY_VERSION.start_with?('3.2.') && RUBY_VERSION >= '3.2.3') ||
-                  RUBY_VERSION >= '3.3.'
+            # This feature is alpha and disabled by default
+            #
+            # @warn To enable heap profiling you are required to also enable allocation profiling.
+            #
+            # @default `DD_PROFILING_EXPERIMENTAL_HEAP_ENABLED` environment variable as a boolean, otherwise `false`
+            option :experimental_heap_enabled do |o|
+              o.type :bool
+              o.env 'DD_PROFILING_EXPERIMENTAL_HEAP_ENABLED'
+              o.default false
+            end
+
+            # Can be used to enable/disable the collection of heap size profiles.
+            #
+            # This feature is alpha and enabled by default when heap profiling is enabled.
+            #
+            # @warn To enable heap size profiling you are required to also enable allocation and heap profiling.
+            #
+            # @default `DD_PROFILING_EXPERIMENTAL_HEAP_SIZE_ENABLED` environment variable as a boolean, otherwise
+            # whatever the value of DD_PROFILING_EXPERIMENTAL_HEAP_ENABLED is.
+            option :experimental_heap_size_enabled do |o|
+              o.type :bool
+              o.env 'DD_PROFILING_EXPERIMENTAL_HEAP_SIZE_ENABLED'
+              o.default true # This gets ANDed with experimental_heap_enabled in the profiler component.
+            end
+
+            # Can be used to configure the allocation sampling rate: a sample will be collected every x allocations.
+            #
+            # This feature is now controlled via {:overhead_target_percentage}
+            option :experimental_allocation_sample_rate do |o|
+              o.after_set do
+                Datadog.logger.warn(
+                  'The profiling.advanced.experimental_allocation_sample_rate setting has been deprecated for removal ' \
+                  'and no longer does anything. Please remove it from your Datadog.configure block. ' \
+                  'Allocation sample rate is now handled by a dynamic sampler which will adjust the sampling rate to ' \
+                  'keep to the configured `profiling.advanced.overhead_target_percentage`.'
+                )
               end
             end
 
+            # Can be used to configure the heap sampling rate: a heap sample will be collected for every x allocation
+            # samples.
+            #
+            # The lower the value, the more accuracy in heap tracking but the bigger the overhead. In particular, a
+            # value of 1 will track ALL allocations samples for heap profiles.
+            #
+            # The effective heap sampling rate in terms of allocations (not allocation samples) can be calculated via
+            # effective_heap_sample_rate = allocation_sample_rate * heap_sample_rate.
+            #
+            # @default `DD_PROFILING_EXPERIMENTAL_HEAP_SAMPLE_RATE` environment variable, otherwise `10`.
+            option :experimental_heap_sample_rate do |o|
+              o.type :int
+              o.env 'DD_PROFILING_EXPERIMENTAL_HEAP_SAMPLE_RATE'
+              o.default 10
+            end
+
             # Can be used to disable checking which version of `libmysqlclient` is being used by the `mysql2` gem.
             #
             # This setting is only used when the `mysql2` gem is installed.
@@ -391,6 +446,34 @@ module Datadog
                 end
               end
             end
+
+            # Configures how much wall-time overhead the profiler targets. The profiler will dynamically adjust the
+            # interval between samples it takes so as to try and maintain the property that it spends no longer than
+            # this amount of wall-clock time profiling. For example, with the default value of 2%, the profiler will
+            # try and cause no more than 1.2 seconds per minute of overhead. Decreasing this value will reduce the
+            # accuracy of the data collected. Increasing will impact the application.
+            #
+            # We do not recommend tweaking this value.
+            #
+            # This value should be a percentage i.e. a number between 0 and 100, not 0 and 1.
+            #
+            # @default `DD_PROFILING_OVERHEAD_TARGET_PERCENTAGE` as a float, otherwise 2.0
+            option :overhead_target_percentage do |o|
+              o.type :float
+              o.env 'DD_PROFILING_OVERHEAD_TARGET_PERCENTAGE'
+              o.default 2.0
+            end
+
+            # Controls how often the profiler reports data, in seconds. Cannot be lower than 60 seconds.
+            #
+            # We do not recommend tweaking this value.
+            #
+            # @default `DD_PROFILING_UPLOAD_PERIOD` environment variable, otherwise 60
+            option :upload_period_seconds do |o|
+              o.type :int
+              o.env 'DD_PROFILING_UPLOAD_PERIOD'
+              o.default 60
+            end
           end
 
           # @public_api
@@ -592,6 +675,42 @@ module Datadog
             o.env Core::Telemetry::Ext::ENV_HEARTBEAT_INTERVAL
             o.default 60.0
           end
+
+          # The install id of the application.
+          #
+          # This method is used internally, by library injection.
+          #
+          # @default `DD_INSTRUMENTATION_INSTALL_ID` environment variable, otherwise `nil`.
+          # @return [String,nil]
+          # @!visibility private
+          option :install_id do |o|
+            o.type :string, nilable: true
+            o.env Core::Telemetry::Ext::ENV_INSTALL_ID
+          end
+
+          # The install type of the application.
+          #
+          # This method is used internally, by library injection.
+          #
+          # @default `DD_INSTRUMENTATION_INSTALL_TYPE` environment variable, otherwise `nil`.
+          # @return [String,nil]
+          # @!visibility private
+          option :install_type do |o|
+            o.type :string, nilable: true
+            o.env Core::Telemetry::Ext::ENV_INSTALL_TYPE
+          end
+
+          # The install time of the application.
+          #
+          # This method is used internally, by library injection.
+          #
+          # @default `DD_INSTRUMENTATION_INSTALL_TIME` environment variable, otherwise `nil`.
+          # @return [String,nil]
+          # @!visibility private
+          option :install_time do |o|
+            o.type :string, nilable: true
+            o.env Core::Telemetry::Ext::ENV_INSTALL_TIME
+          end
         end
 
         # Remote configuration
@@ -630,6 +749,19 @@ module Datadog
             o.default 5.0
           end
 
+          # Tune remote configuration boot timeout.
+          # Early operations such as requests are blocked until RC is ready. In
+          # order to not block the application indefinitely a timeout is
+          # enforced allowing requests to proceed with the local configuration.
+          #
+          # @default `DD_REMOTE_CONFIG_BOOT_TIMEOUT` environment variable, otherwise `1.0` seconds.
+          # @return [Float]
+          option :boot_timeout_seconds do |o|
+            o.env Core::Remote::Ext::ENV_BOOT_TIMEOUT_SECONDS
+            o.type :float
+            o.default 1.0
+          end
+
           # Declare service name to bind to remote configuration. Use when
           # DD_SERVICE does not match the correct integration for which remote
           # configuration applies.

data/lib/datadog/core/environment/class_count.rb

@@ -5,15 +5,15 @@ module Datadog
     module Environment
       # Retrieves number of classes from runtime
       module ClassCount
-        module_function
-
-        def value
+        def self.value
           ::ObjectSpace.count_objects[:T_CLASS]
         end
 
-        def available?
-          ::ObjectSpace.respond_to?(:count_objects) \
-           && ::ObjectSpace.count_objects.key?(:T_CLASS)
+        def self.available?
+          return @class_count_available if defined?(@class_count_available)
+
+          @class_count_available =
+            ::ObjectSpace.respond_to?(:count_objects) && ::ObjectSpace.count_objects.key?(:T_CLASS)
         end
       end
     end

data/lib/datadog/core/remote/component.rb

@@ -13,9 +13,7 @@ module Datadog
       # Configures the HTTP transport to communicate with the agent
       # to fetch and sync the remote configuration
       class Component
-        BARRIER_TIMEOUT = 1.0 # second
-
-        attr_reader :client
+        attr_reader :client, :healthy
 
         def initialize(settings, capabilities, agent_settings)
           transport_options = {}
@@ -24,14 +22,14 @@ module Datadog
           negotiation = Negotiation.new(settings, agent_settings)
           transport_v7 = Datadog::Core::Remote::Transport::HTTP.v7(**transport_options.dup)
 
-          @barrier = Barrier.new(BARRIER_TIMEOUT)
+          @barrier = Barrier.new(settings.remote.boot_timeout_seconds)
 
           @client = Client.new(transport_v7, capabilities)
-          healthy = false
+          @healthy = false
           Datadog.logger.debug { "new remote configuration client: #{@client.id}" }
 
           @worker = Worker.new(interval: settings.remote.poll_interval_seconds) do
-            unless healthy || negotiation.endpoint?('/v0.7/config')
+            unless @healthy || negotiation.endpoint?('/v0.7/config')
               @barrier.lift
 
               next
@@ -39,7 +37,7 @@ module Datadog
 
             begin
               @client.sync
-              healthy ||= true
+              @healthy ||= true
             rescue Client::SyncError => e
               Datadog.logger.error do
                 "remote worker client sync error: #{e.message} location: #{Array(e.backtrace).first}. skipping sync"
@@ -57,7 +55,7 @@ module Datadog
 
               # client state is unknown, state might be corrupted
               @client = Client.new(transport_v7, capabilities)
-              healthy = false
+              @healthy = false
               Datadog.logger.debug { "new remote configuration client: #{@client.id}" }
 
               # TODO: bail out if too many errors?
@@ -103,17 +101,32 @@ module Datadog
           def wait_once(timeout = nil)
             # TTAS (Test and Test-And-Set) optimisation
             # Since @once only ever goes from false to true, this is semantically valid
-            return if @once
+            return :pass if @once
 
             begin
               @mutex.lock
 
-              return if @once
+              return :pass if @once
 
               timeout ||= @timeout
 
-              # rbs/core has a bug, timeout type is incorrectly ?Integer
-              @condition.wait(@mutex, _ = timeout)
+              # - starting with Ruby 3.2, ConditionVariable#wait returns nil on
+              #   timeout and an integer otherwise
+              # - before Ruby 3.2, ConditionVariable returns itself
+              # so we have to rely on @once having been set
+              if RUBY_VERSION >= '3.2'
+                lifted = @condition.wait(@mutex, timeout)
+              else
+                @condition.wait(@mutex, timeout)
+                lifted = @once
+              end
+
+              if lifted
+                :lift
+              else
+                @once = true
+                :timeout
+              end
             ensure
               @mutex.unlock
             end

data/lib/datadog/core/remote/ext.rb

@@ -6,6 +6,7 @@ module Datadog
       module Ext
         ENV_ENABLED = 'DD_REMOTE_CONFIGURATION_ENABLED'
         ENV_POLL_INTERVAL_SECONDS = 'DD_REMOTE_CONFIG_POLL_INTERVAL_SECONDS'
+        ENV_BOOT_TIMEOUT_SECONDS = 'DD_REMOTE_CONFIG_BOOT_TIMEOUT_SECONDS'
       end
     end
   end

data/lib/datadog/core/remote/tie/tracing.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require_relative '../tie'
+
+module Datadog
+  module Core
+    module Remote
+      module Tie
+        # Extend Remote Configuration abilities with Tracing
+        module Tracing
+          # Tag per-request Remote Configuration metadata using Tracing
+          def self.tag(boot, span)
+            return if boot.nil?
+            return if span.nil?
+
+            return if Datadog::Core::Remote.active_remote.nil?
+
+            # TODO: this is not thread-consistent
+            ready = Datadog::Core::Remote.active_remote.healthy
+            status = ready ? 'ready' : 'disconnected'
+
+            span.set_tag('_dd.rc.client_id', Datadog::Core::Remote.active_remote.client.id)
+            span.set_tag('_dd.rc.status', status)
+
+            if boot.barrier != :pass
+              span.set_tag('_dd.rc.boot.time', boot.time)
+
+              if boot.barrier == :timeout
+                span.set_tag('_dd.rc.boot.timeout', true)
+              else
+                span.set_tag('_dd.rc.boot.ready', ready)
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/remote/tie.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Core
+    module Remote
+      # Provide Remote Configuration extensions to other components
+      module Tie
+        Boot = Struct.new(
+          :barrier,
+          :time,
+        )
+
+        def self.boot
+          return if Datadog::Core::Remote.active_remote.nil?
+
+          barrier = nil
+
+          t = Datadog::Core::Utils::Time.measure do
+            barrier = Datadog::Core::Remote.active_remote.barrier(:once)
+          end
+
+          Boot.new(barrier, t)
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/telemetry/collector.rb

@@ -9,6 +9,7 @@ require_relative '../utils/hash'
 require_relative 'v1/application'
 require_relative 'v1/dependency'
 require_relative 'v1/host'
+require_relative 'v1/install_signature'
 require_relative 'v1/integration'
 require_relative 'v1/product'
 require_relative '../transport/ext'
@@ -81,6 +82,15 @@ module Datadog
           )
         end
 
+        # Forms a telemetry app-started install_signature object
+        def install_signature
+          Telemetry::V1::InstallSignature.new(
+            install_id: Datadog.configuration.dig('telemetry', 'install_id'),
+            install_type: Datadog.configuration.dig('telemetry', 'install_type'),
+            install_time: Datadog.configuration.dig('telemetry', 'install_time'),
+          )
+        end
+
         # Forms a telemetry app-started integrations object
         def integrations
           Datadog.registry.map do |integration|

data/lib/datadog/core/telemetry/event.rb

@@ -60,7 +60,8 @@ module Datadog
             dependencies: dependencies,
             integrations: integrations,
             configuration: configurations,
-            additional_payload: additional_payload
+            additional_payload: additional_payload,
+            install_signature: install_signature
           )
         end
 

data/lib/datadog/core/telemetry/ext.rb

@@ -6,6 +6,9 @@ module Datadog
       module Ext
         ENV_ENABLED = 'DD_INSTRUMENTATION_TELEMETRY_ENABLED'
         ENV_HEARTBEAT_INTERVAL = 'DD_TELEMETRY_HEARTBEAT_INTERVAL'
+        ENV_INSTALL_ID = 'DD_INSTRUMENTATION_INSTALL_ID'
+        ENV_INSTALL_TYPE = 'DD_INSTRUMENTATION_INSTALL_TYPE'
+        ENV_INSTALL_TIME = 'DD_INSTRUMENTATION_INSTALL_TIME'
       end
     end
   end

data/lib/datadog/core/telemetry/v1/app_event.rb

@@ -10,18 +10,24 @@ module Datadog
             :additional_payload,
             :configuration,
             :dependencies,
+            :install_signature,
             :integrations
 
           # @param additional_payload [Array<Telemetry::V1::Configuration>] List of Additional payload to track (any key
           #   value not mentioned and doesn't fit under a metric)
           # @param configuration [Array<Telemetry::V1::Configuration>] List of Tracer related configuration data
           # @param dependencies [Array<Telemetry::V1::Dependency>] List of all loaded modules requested by the app
+          # @param install_signature [Telemetry::V1::InstallSignature] Install signature data
           # @param integrations [Array<Telemetry::V1::Integration>] List of integrations that are available within the app
           #   and applicable to be traced
-          def initialize(additional_payload: nil, configuration: nil, dependencies: nil, integrations: nil)
+          def initialize(
+            additional_payload: nil, configuration: nil, dependencies: nil, install_signature: nil,
+            integrations: nil
+          )
             @additional_payload = additional_payload
             @configuration = configuration
             @dependencies = dependencies
+            @install_signature = install_signature
             @integrations = integrations
           end
 
@@ -30,6 +36,7 @@ module Datadog
               hash[:additional_payload] = map_hash(@additional_payload) if @additional_payload
               hash[:configuration] = map_hash(@configuration) if @configuration
               hash[:dependencies] = map_array(@dependencies) if @dependencies
+              hash[:install_signature] = @install_signature.to_h if @install_signature
               hash[:integrations] = map_array(@integrations) if @integrations
             end
           end

data/lib/datadog/core/telemetry/v1/install_signature.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Core
+    module Telemetry
+      module V1
+        # Describes attributes for install signature
+        class InstallSignature
+          using Core::Utils::Hash::Refinement
+
+          attr_reader \
+            :install_id,
+            :install_type,
+            :install_time
+
+          # @param id [String,nil] Install ID
+          # @param type [String,nil] Install type
+          # @param type [String,nil] Install time
+          def initialize(install_id:, install_type:, install_time:)
+            @install_id = install_id
+            @install_type = install_type
+            @install_time = install_time
+          end
+
+          def to_h
+            hash = {
+              install_id: @install_id,
+              install_type: @install_type,
+              install_time: @install_time
+            }
+            hash.compact!
+            hash
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/opentelemetry/sdk/propagator.rb

@@ -29,9 +29,10 @@ module Datadog
           carrier, context: ::OpenTelemetry::Context.current,
           getter: ::OpenTelemetry::Context::Propagation.text_map_getter
         )
-          unless getter == ::OpenTelemetry::Context::Propagation.text_map_getter
+          if getter != ::OpenTelemetry::Context::Propagation.text_map_getter &&
+              getter != ::OpenTelemetry::Common::Propagation.rack_env_getter
             Datadog.logger.error(
-              'Custom getter is not supported. Please inform the `ddtrace` team at ' \
+              "Custom getter #{getter} is not supported. Please inform the `ddtrace` team at " \
             ' https://github.com/DataDog/dd-trace-rb of your use case so we can best support you. Using the default ' \
             'OpenTelemetry::Context::Propagation.text_map_getter as a fallback getter.'
             )

data/lib/datadog/opentelemetry.rb

@@ -8,7 +8,10 @@
 # This file activates the integrations of all OpenTelemetry
 # components supported by Datadog.
 
+# Load Tracing
 require_relative 'tracing'
+require_relative 'tracing/contrib'
+
 require_relative 'opentelemetry/api/context'
 
 # DEV: Should this be a Contrib integration, that depends on the `opentelemetry-sdk`

data/lib/datadog/profiling/collectors/cpu_and_wall_time_worker.rb

@@ -15,14 +15,14 @@ module Datadog
 
         def initialize(
           gc_profiling_enabled:,
-          allocation_counting_enabled:,
           no_signals_workaround_enabled:,
           thread_context_collector:,
-          idle_sampling_helper: IdleSamplingHelper.new,
+          dynamic_sampling_rate_overhead_target_percentage:,
+          allocation_profiling_enabled:,
           # **NOTE**: This should only be used for testing; disabling the dynamic sampling rate will increase the
           # profiler overhead!
           dynamic_sampling_rate_enabled: true,
-          allocation_sample_every: 0 # Currently only for testing; Setting this to > 0 can add a lot of overhead!
+          idle_sampling_helper: IdleSamplingHelper.new
         )
           unless dynamic_sampling_rate_enabled
             Datadog.logger.warn(
@@ -30,22 +30,15 @@ module Datadog
             )
           end
 
-          if allocation_counting_enabled && allocation_sample_every > 0
-            Datadog.logger.warn(
-              "Enabled experimental allocation profiling: allocation_sample_every=#{allocation_sample_every}. This is " \
-              'experimental, not recommended, and will increase overhead!'
-            )
-          end
-
           self.class._native_initialize(
             self,
             thread_context_collector,
             gc_profiling_enabled,
             idle_sampling_helper,
-            allocation_counting_enabled,
             no_signals_workaround_enabled,
             dynamic_sampling_rate_enabled,
-            allocation_sample_every,
+            dynamic_sampling_rate_overhead_target_percentage,
+            allocation_profiling_enabled,
           )
           @worker_thread = nil
           @failure_exception = nil