datadog 2.23.0 → 2.28.0

data/lib/datadog/profiling/component.rb

@@ -4,9 +4,6 @@ module Datadog
   module Profiling
     # Responsible for wiring up the Profiler for execution
     module Component
-      ALLOCATION_WITH_RACTORS_ONLY_ONCE = Datadog::Core::Utils::OnlyOnce.new
-      private_constant :ALLOCATION_WITH_RACTORS_ONLY_ONCE
-
       # Passing in a `nil` tracer is supported and will disable the following profiling features:
       # * Profiling in the trace viewer, as well as scoping a profile down to a span
       # * Endpoint aggregation in the profiler UX, including normalization (resource per endpoint call)
@@ -85,6 +82,10 @@ module Datadog
           Datadog::Profiling::Ext::DirMonkeyPatches.apply!
         end
 
+        if can_apply_exec_monkey_patch?(settings)
+          Datadog::Profiling::Ext::ExecMonkeyPatch.apply!
+        end
+
         [profiler, {profiling_enabled: true}]
       end
 
@@ -145,7 +146,7 @@ module Datadog
           logger.debug(
             "Using Ractors may result in GC profiling unexpectedly " \
             "stopping (https://bugs.ruby-lang.org/issues/19112). Note that this stop has no impact in your " \
-            "application stability or performance. This does not happen if Ractors are not used."
+            "application stability or performance. This issue is fixed on Ruby 4."
           )
         end
 
@@ -195,13 +196,11 @@ module Datadog
         # On all known versions of Ruby 3.x, due to https://bugs.ruby-lang.org/issues/19112, when a ractor gets
         # garbage collected, Ruby will disable all active tracepoints, which this feature internally relies on.
         elsif RUBY_VERSION.start_with?("3.")
-          ALLOCATION_WITH_RACTORS_ONLY_ONCE.run do
-            logger.info(
-              "Using Ractors may result in allocation profiling " \
-              "stopping (https://bugs.ruby-lang.org/issues/19112). Note that this stop has no impact in your " \
-              "application stability or performance. This does not happen if Ractors are not used."
-            )
-          end
+          logger.debug(
+            "Using Ractors may result in allocation profiling " \
+            "stopping (https://bugs.ruby-lang.org/issues/19112). Note that this stop has no impact in your " \
+            "application stability or performance. This issue is fixed on Ruby 4."
+          )
         end
 
         logger.debug("Enabled allocation profiling")
@@ -220,6 +219,12 @@ module Datadog
             "Please upgrade to Ruby >= 3.1 in order to use this feature. Heap profiling has been disabled."
           )
           return false
+        elsif RUBY_VERSION.start_with?("4.")
+          logger.warn(
+            "Datadog Ruby heap profiler is currently incompatible with Ruby 4. " \
+            "Heap profiling has been disabled."
+          )
+          return false
         end
 
         unless allocation_profiling_enabled
@@ -433,6 +438,15 @@ module Datadog
         settings.profiling.advanced.dir_interruption_workaround_enabled
       end
 
+      private_class_method def self.can_apply_exec_monkey_patch?(settings)
+        return false if RUBY_VERSION < "2.7"
+
+        # This file is 2.7+ only so we only require it here once we've checked the Ruby version
+        require "datadog/profiling/ext/exec_monkey_patch"
+
+        settings.profiling.advanced.shutdown_on_exec_enabled
+      end
+
       private_class_method def self.enable_gvl_profiling?(settings, logger)
         return false if RUBY_VERSION < "3.2"
 

data/lib/datadog/profiling/exporter.rb

@@ -70,6 +70,9 @@ module Datadog
 
         uncompressed_code_provenance = code_provenance_collector.refresh.generate_json if code_provenance_collector
 
+        process_tags = Datadog.configuration.experimental_propagate_process_tags_enabled ?
+          Core::Environment::Process.serialized : ''
+
         Flush.new(
           start: start,
           finish: finish,
@@ -80,6 +83,7 @@ module Datadog
             settings: Datadog.configuration,
             profile_seq: sequence_tracker.get_next,
           ).to_a,
+          process_tags: process_tags,
           internal_metadata: internal_metadata.merge(
             {
               worker_stats: worker_stats,

data/lib/datadog/profiling/ext/dir_monkey_patches.rb

@@ -30,6 +30,10 @@ module Datadog
       if RUBY_VERSION.start_with?("2.")
         # Monkey patches for Dir.singleton_class (Ruby 2 version). See DirMonkeyPatches above for more details.
         module DirClassMonkeyPatches
+          # Steep: Workaround that defines args and block only for Ruby 2.x.
+          # @type var args: ::Array[any]
+          # @type var block: ^(?) -> any | nil
+
           def [](*args, &block)
             Datadog::Profiling::Collectors::CpuAndWallTimeWorker._native_hold_signals
             super
@@ -148,6 +152,11 @@ module Datadog
       else
         # Monkey patches for Dir.singleton_class (Ruby 3 version). See DirMonkeyPatches above for more details.
         module DirClassMonkeyPatches
+          # Steep: Workaround that defines args, kwargs and block only for Ruby 3.x.
+          # @type var args: ::Array[any]
+          # @type var kwargs: ::Hash[::Symbol, any]
+          # @type var block: ^(?) -> any | nil
+
           def [](*args, **kwargs, &block)
             Datadog::Profiling::Collectors::CpuAndWallTimeWorker._native_hold_signals
             super
@@ -263,6 +272,10 @@ module Datadog
       if RUBY_VERSION.start_with?("2.")
         # Monkey patches for Dir (Ruby 2 version). See DirMonkeyPatches above for more details.
         module DirInstanceMonkeyPatches
+          # Steep: Workaround that defines args and block only for Ruby 2.x.
+          # @type var args: ::Array[any]
+          # @type var block: ^(?) -> any | nil
+
           # See note on methods that yield above.
           def each(*args, &block)
             if block
@@ -336,6 +349,11 @@ module Datadog
       else
         # Monkey patches for Dir (Ruby 3 version). See DirMonkeyPatches above for more details.
         module DirInstanceMonkeyPatches
+          # Steep: Workaround that defines args, kwargs and block only for Ruby 3.x.
+          # @type var args: ::Array[any]
+          # @type var kwargs: ::Hash[::Symbol, any]
+          # @type var block: ^(?) -> any | nil
+
           # See note on methods that yield above.
           def each(*args, **kwargs, &block)
             if block

data/lib/datadog/profiling/ext/exec_monkey_patch.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Profiling
+    module Ext
+      # The profiler gathers data by sending `SIGPROF` unix signals to Ruby application threads.
+      #
+      # When using `Kernel#exec` on Linux, it can happen that a signal sent before calling `exec` arrives after
+      # the new process is running, causing it to fail with the `Profiling timer expired` error message.
+      # To avoid this, the profiler installs a monkey patch on `Kernel#exec` to stop profiling before actually
+      # calling `exec`.
+      # This monkey patch is available for Ruby 2.7+; let us know if you need it on earlier Rubies.
+      # For more details see https://github.com/DataDog/dd-trace-rb/issues/5101 .
+      module ExecMonkeyPatch
+        def self.apply!
+          ::Object.prepend(ObjectMonkeyPatch)
+
+          true
+        end
+
+        module ObjectMonkeyPatch
+          private
+
+          def exec(...)
+            Datadog.send(:components, allow_initialization: false)&.profiler&.shutdown!(report_last_profile: false)
+            super
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/profiling/flush.rb

@@ -13,6 +13,7 @@ module Datadog
         :code_provenance_file_name,
         :code_provenance_data,
         :tags_as_array,
+        :process_tags,
         :internal_metadata_json,
         :info_json
 
@@ -23,6 +24,7 @@ module Datadog
         code_provenance_file_name:,
         code_provenance_data:,
         tags_as_array:,
+        process_tags:,
         internal_metadata:,
         info_json:
       )
@@ -32,6 +34,7 @@ module Datadog
         @code_provenance_file_name = code_provenance_file_name
         @code_provenance_data = code_provenance_data
         @tags_as_array = tags_as_array
+        @process_tags = process_tags
         @internal_metadata_json = JSON.generate(internal_metadata)
         @info_json = info_json
       end

data/lib/datadog/profiling/http_transport.rb

@@ -13,7 +13,10 @@ module Datadog
       def initialize(agent_settings:, site:, api_key:, upload_timeout_seconds:)
         @upload_timeout_milliseconds = (upload_timeout_seconds * 1_000).to_i
 
-        @exporter_configuration =
+        # Steep: multiple issues here
+        # first https://github.com/soutaro/steep/issues/363
+        # then https://github.com/soutaro/steep/issues/1603 (remove the .freeze to see it)
+        @exporter_configuration = # steep:ignore IncompatibleAssignment
           if agentless?(site, api_key)
             [:agentless, site, api_key].freeze
           else

data/lib/datadog/profiling/profiler.rb

@@ -31,10 +31,11 @@ module Datadog
         scheduler.start(on_failure_proc: proc { component_failed(:scheduler) })
       end
 
-      def shutdown!
+      def shutdown!(report_last_profile: true)
         Datadog.logger.debug("Shutting down profiler")
 
         stop_worker
+        scheduler.disable_reporting unless report_last_profile
         stop_scheduler
       end
 
@@ -57,11 +58,8 @@ module Datadog
         Datadog::Core::Telemetry::Logger
           .error("Detected issue with profiler (#{failed_component} component), stopping profiling")
 
-        # We explicitly not stop the crash tracker in this situation, under the assumption that, if a component failed,
-        # we're operating in a degraded state and crash tracking may still be helpful.
-
         if failed_component == :worker
-          scheduler.mark_profiler_failed
+          scheduler.disable_reporting
           stop_scheduler
         elsif failed_component == :scheduler
           stop_worker

data/lib/datadog/profiling/scheduler.rb

@@ -24,7 +24,7 @@ module Datadog
       attr_reader \
         :exporter,
         :transport,
-        :profiler_failed
+        :reporting_disabled
 
       public
 
@@ -36,7 +36,7 @@ module Datadog
       )
         @exporter = exporter
         @transport = transport
-        @profiler_failed = false
+        @reporting_disabled = false
         @stop_requested = false
 
         # Workers::Async::Thread settings
@@ -83,14 +83,15 @@ module Datadog
         true
       end
 
-      # This is called by the Profiler class whenever an issue happened in the profiler. This makes sure that even
-      # if there is data to be flushed, we don't try to flush it.
-      def mark_profiler_failed
-        @profiler_failed = true
+      # Called by the Profiler when it wants to prevent any further reporting,
+      # even if there is data to be flushed.
+      # Used when the profiler failed or we're otherwise in a hurry to shut down.
+      def disable_reporting
+        @reporting_disabled = true
       end
 
       def work_pending?
-        !profiler_failed && exporter.can_flush? && (run_loop? || !stop_requested?)
+        !reporting_disabled && exporter.can_flush? && (run_loop? || !stop_requested?)
       end
 
       def reset_after_fork

data/lib/datadog/profiling/tag_builder.rb

@@ -50,6 +50,7 @@ module Datadog
           FORM_FIELD_TAG_PROFILER_VERSION => profiler_version,
           'profile_seq' => profile_seq.to_s,
         )
+
         user_tag_keys = settings.tags.keys
         hash.keep_if { |tag| user_tag_keys.include?(tag) || ALLOWED_TAGS.include?(tag) }
         Core::Utils.encode_tags(hash)

data/lib/datadog/tracing/contrib/extensions.rb

@@ -220,12 +220,20 @@ module Datadog
             # `@instrumented_integrations` hash.
             # @!visibility private
             def instrumented_integrations
-              INSTRUMENTED_INTEGRATIONS_LOCK.synchronize { (@instrumented_integrations&.dup || {}).freeze }
+              INSTRUMENTED_INTEGRATIONS_LOCK.synchronize do
+                (if defined?(@instrumented_integrations)
+                   @instrumented_integrations&.dup
+                 end || {}).freeze
+              end
             end
 
             # @!visibility private
             def reset!
-              INSTRUMENTED_INTEGRATIONS_LOCK.synchronize { @instrumented_integrations&.clear }
+              INSTRUMENTED_INTEGRATIONS_LOCK.synchronize do
+                if defined?(@instrumented_integrations)
+                  @instrumented_integrations&.clear
+                end
+              end
               super
             end
 

data/lib/datadog/tracing/contrib/karafka/patcher.rb

@@ -25,35 +25,34 @@ module Datadog
           # @see https://github.com/karafka/karafka/blob/b06d1f7c17818e1605f80c2bb573454a33376b40/README.md?plain=1#L29-L35
           def each(&block)
             @messages_array.each do |message|
-              if configuration[:distributed_tracing]
+              trace_digest = if configuration[:distributed_tracing]
                 headers = if message.metadata.respond_to?(:raw_headers)
                   message.metadata.raw_headers
                 else
                   message.metadata.headers
                 end
-                trace_digest = Karafka.extract(headers)
-                Datadog::Tracing.continue_trace!(trace_digest) if trace_digest
+                Karafka.extract(headers)
               end
 
-              if Datadog::DataStreams.enabled?
-                begin
-                  headers = if message.metadata.respond_to?(:raw_headers)
-                    message.metadata.raw_headers
-                  else
-                    message.metadata.headers
+              Tracing.trace(Ext::SPAN_MESSAGE_CONSUME, continue_from: trace_digest) do |span, trace|
+                if Datadog::DataStreams.enabled?
+                  begin
+                    headers = if message.metadata.respond_to?(:raw_headers)
+                      message.metadata.raw_headers
+                    else
+                      message.metadata.headers
+                    end
+
+                    Datadog::DataStreams.set_consume_checkpoint(
+                      type: 'kafka',
+                      source: message.topic,
+                      auto_instrumentation: true
+                    ) { |key| headers[key] }
+                  rescue => e
+                    Datadog.logger.debug("Error setting DSM checkpoint: #{e.class}: #{e}")
                   end
-
-                  Datadog::DataStreams.set_consume_checkpoint(
-                    type: 'kafka',
-                    source: message.topic,
-                    auto_instrumentation: true
-                  ) { |key| headers[key] }
-                rescue => e
-                  Datadog.logger.debug("Error setting DSM checkpoint: #{e.class}: #{e}")
                 end
-              end
 
-              Tracing.trace(Ext::SPAN_MESSAGE_CONSUME) do |span|
                 span.set_tag(Ext::TAG_OFFSET, message.metadata.offset)
                 span.set_tag(Contrib::Ext::Messaging::TAG_DESTINATION, message.topic)
                 span.set_tag(Contrib::Ext::Messaging::TAG_SYSTEM, Ext::TAG_SYSTEM)
@@ -66,22 +65,12 @@ module Datadog
           end
         end
 
-        module AppPatch
-          ONLY_ONCE_PER_APP = Hash.new { |h, key| h[key] = Core::Utils::OnlyOnce.new }
-
-          def initialized!
-            ONLY_ONCE_PER_APP[self].run do
-              # Activate tracing on components related to Karafka (e.g. WaterDrop)
-              Contrib::Karafka::Framework.setup
-            end
-            super
-          end
-        end
-
         # Patcher enables patching of 'karafka' module.
         module Patcher
           include Contrib::Patcher
 
+          ACTIVATE_FRAMEWORK_ONLY_ONCE = Core::Utils::OnlyOnce.new
+
           module_function
 
           def target_version
@@ -91,10 +80,20 @@ module Datadog
           def patch
             require_relative 'monitor'
             require_relative 'framework'
+            require_relative '../waterdrop'
 
             ::Karafka::Instrumentation::Monitor.prepend(Monitor)
             ::Karafka::Messages::Messages.prepend(MessagesPatch)
-            ::Karafka::App.singleton_class.prepend(AppPatch)
+
+            if Contrib::WaterDrop::Integration.compatible?
+              ::Karafka.monitor.subscribe('app.initialized') do |event|
+                ACTIVATE_FRAMEWORK_ONLY_ONCE.run do
+                  Contrib::Karafka::Framework.setup
+                end
+
+                Contrib::WaterDrop::Patcher.add_middleware(::Karafka.producer)
+              end
+            end
           end
         end
       end

data/lib/datadog/tracing/contrib/status_range_matcher.rb

@@ -8,7 +8,8 @@ module Datadog
         attr_reader :ranges
 
         def initialize(ranges)
-          @ranges = Array(ranges)
+          # Steep: https://github.com/ruby/rbs/issues/1874
+          @ranges = Array(ranges) # steep:ignore IncompatibleAssignment
         end
 
         def +(other)

data/lib/datadog/tracing/contrib/utils/quantization/hash.rb

@@ -10,7 +10,9 @@ module Datadog
             PLACEHOLDER = '?'
             EXCLUDE_KEYS = [].freeze
             SHOW_KEYS = [].freeze
-            DEFAULT_OPTIONS = {
+
+            # Steep: https://github.com/soutaro/steep/issues/363
+            DEFAULT_OPTIONS = { # steep:ignore IncompatibleAssignment
               exclude: EXCLUDE_KEYS,
               show: SHOW_KEYS,
               placeholder: PLACEHOLDER

data/lib/datadog/tracing/contrib/waterdrop/patcher.rb

@@ -25,9 +25,7 @@ module Datadog
             ::WaterDrop::Producer.prepend(Producer)
             ::WaterDrop.instrumentation.subscribe('producer.configured') do |event|
               producer = event[:producer]
-
-              included_middlewares = producer.middleware.instance_variable_get(:@steps)
-              producer.middleware.append(Middleware) unless included_middlewares.include?(Middleware)
+              add_middleware(producer)
 
               if Datadog.configuration.data_streams.enabled
                 producer.monitor.subscribe('message.acknowledged') do |ack_event|
@@ -39,6 +37,11 @@ module Datadog
               end
             end
           end
+
+          def add_middleware(producer)
+            included_middlewares = producer.middleware.instance_variable_get(:@steps)
+            producer.middleware.append(Middleware) unless included_middlewares.include?(Middleware)
+          end
         end
       end
     end

data/lib/datadog/tracing/contrib/waterdrop.rb

@@ -12,12 +12,16 @@ module Datadog
         def self.inject(digest, data)
           raise 'Please invoke Datadog.configure at least once before calling this method' unless @propagation
 
+          # Steep: https://github.com/soutaro/steep/issues/477
+          # @type ivar @propagation: WaterDrop::Distributed::Propagation
           @propagation.inject!(digest, data)
         end
 
         def self.extract(data)
           raise 'Please invoke Datadog.configure at least once before calling this method' unless @propagation
 
+          # Steep: https://github.com/soutaro/steep/issues/477
+          # @type ivar @propagation: WaterDrop::Distributed::Propagation
           @propagation.extract(data)
         end
 

data/lib/datadog/tracing/diagnostics/environment_logger.rb

@@ -62,7 +62,7 @@ module Datadog
             # return `nil` with IO transport
             return unless transport.respond_to?(:client)
 
-            adapter = transport.client.api.adapter
+            adapter = transport.client.instance.adapter
             adapter.url
           end
 

data/lib/datadog/tracing/distributed/baggage.rb

@@ -175,8 +175,9 @@ module Datadog
 
           tags = {}
 
-          baggage_tag_keys.each do |key, _| # rubocop:disable Style/HashEachMethods
-            value = baggage[key]
+          baggage_tag_keys.each do |key, _|
+            # Steep: https://github.com/soutaro/steep/issues/2031
+            value = baggage[key] # steep:ignore ArgumentTypeMismatch
             next if value.nil? || value.empty?
 
             tags["baggage.#{key}"] = value

data/lib/datadog/tracing/remote.rb

@@ -7,8 +7,6 @@ module Datadog
   module Tracing
     # Remote configuration declaration
     module Remote
-      class ReadError < StandardError; end
-
       class << self
         PRODUCT = 'APM_TRACING'
 
@@ -71,13 +69,7 @@ module Datadog
         private
 
         def parse_content(content)
-          data = content.data.read
-
-          content.data.rewind
-
-          raise ReadError, 'EOF reached' if data.nil?
-
-          JSON.parse(data)
+          JSON.parse(content.data)
         end
       end
     end

data/lib/datadog/tracing/sampling/priority_sampler.rb

@@ -60,7 +60,9 @@ module Datadog
         ensure
           if trace.sampling_priority && trace.sampling_priority > 0
             # Don't modify decision if priority was set upstream.
-            if !distributed_sampling_priority && !trace.has_tag?(Tracing::Metadata::Ext::Distributed::TAG_DECISION_MAKER)
+            # Steep: https://github.com/soutaro/steep/issues/1971
+            if !distributed_sampling_priority && # steep:ignore FallbackAny
+                !trace.has_tag?(Tracing::Metadata::Ext::Distributed::TAG_DECISION_MAKER)
               # If no sampling priority being assigned at this point, a custom
               # sampler implementation is configured: this means the user has
               # full control over the sampling decision.

data/lib/datadog/tracing/sampling/rate_sampler.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 require_relative 'sampler'
-require_relative '../utils'
+require_relative '../../core/knuth_sampler'
 
 module Datadog
   module Tracing
@@ -9,46 +9,35 @@ module Datadog
       # {Datadog::Tracing::Sampling::RateSampler} is based on a sample rate.
       class RateSampler < Sampler
         KNUTH_FACTOR = 1111111111111111111
-        UINT64_MODULO = (1 << 64)
 
         # Initialize a {Datadog::Tracing::Sampling::RateSampler}.
         # This sampler keeps a random subset of the traces. Its main purpose is to
         # reduce the instrumentation footprint.
         #
         # @param sample_rate [Numeric] the sample rate between 0.0 and 1.0, inclusive.
-        #   0.0 means that no trace will be sampled; 1.0 means that all traces will be  sampled.
+        #   0.0 means that no trace will be sampled; 1.0 means that all traces will be sampled.
         def initialize(sample_rate = 1.0, decision: nil)
           super()
-
-          unless sample_rate >= 0.0 && sample_rate <= 1.0
-            Datadog.logger.warn('sample rate is not between 0 and 1, falling back to 1')
-            sample_rate = 1.0
-          end
-
-          self.sample_rate = sample_rate
-
+          @sampler = Core::KnuthSampler.new(sample_rate, knuth_factor: KNUTH_FACTOR)
           @decision = decision
         end
 
         def sample_rate(*_)
-          @sample_rate
+          @sampler.rate
         end
 
         def sample_rate=(sample_rate)
-          @sample_rate = sample_rate
-          @sampling_id_threshold = sample_rate * Tracing::Utils::EXTERNAL_MAX_ID
+          @sampler = Core::KnuthSampler.new(sample_rate, knuth_factor: KNUTH_FACTOR)
         end
 
         def sample?(trace)
-          ((trace.id * KNUTH_FACTOR) % UINT64_MODULO) <= @sampling_id_threshold
+          @sampler.sample?(trace.id)
         end
 
         def sample!(trace)
-          sampled = sample?(trace)
-
-          return false unless sampled
+          return false unless sample?(trace)
 
-          trace.sample_rate = @sample_rate
+          trace.sample_rate = sample_rate
           trace.set_tag(Tracing::Metadata::Ext::Distributed::TAG_DECISION_MAKER, @decision) if @decision
 
           true

data/lib/datadog/tracing/span.rb

@@ -80,7 +80,7 @@ module Datadog
 
         @meta = meta || {}
         @metrics = metrics || {}
-        @metastruct = metastruct || {}
+        @metastruct = metastruct || Metastruct.new
         @status = status || 0
 
         # start_time and end_time track wall clock. In Ruby, wall clock

data/lib/datadog/tracing/span_event.rb

@@ -69,8 +69,8 @@ module Datadog
 
       private
 
-      MIN_INT64_SIGNED = -2**63
-      MAX_INT64_SIGNED = 2 << 63 - 1
+      MIN_INT64_SIGNED = -2 << 62
+      MAX_INT64_SIGNED = (2 << 62) - 1
 
       # Checks the attributes hash to ensure it only contains serializable values.
       # Invalid values are removed from the hash.