ddtrace 1.12.1 → 1.14.0

data/lib/datadog/kit/identity.rb

@@ -6,84 +6,99 @@ module Datadog
   module Kit
     # Tracking identity via traces
     module Identity
-      # Attach user information to the trace
-      #
-      # @param trace [TraceOperation] Trace to attach data to. Defaults to
-      #   active trace.
-      # @param span [SpanOperation] Span to attach data to. Defaults to
-      #   active span on trace. Note that this should be a service entry span.
-      #   When AppSec is enabled, the expected span and trace are automatically
-      #   used as defaults.
-      # @param id [String] Mandatory. Username or client id extracted
-      #   from the access token or Authorization header in the inbound request
-      #   from outside the system.
-      # @param email [String] Email of the authenticated user associated
-      #   to the trace.
-      # @param name [String] User-friendly name. To be displayed in the
-      #   UI if set.
-      # @param session_id [String] Session ID of the authenticated user.
-      # @param role [String] Actual/assumed role the client is making
-      #   the request under extracted from token or application security
-      #   context.
-      # @param scope [String] Scopes or granted authorities the client
-      #   currently possesses extracted from token or application security
-      #   context. The value would come from the scope associated with an OAuth
-      #   2.0 Access Token or an attribute value in a SAML 2.0 Assertion.
-      # @param others [Hash<Symbol, String>] Additional free-form
-      #   user information to attach to the trace.
-      #
-      # rubocop:disable Metrics/CyclomaticComplexity
-      # rubocop:disable Metrics/PerceivedComplexity
-      # rubocop:disable Metrics/AbcSize
-      def self.set_user(
-        trace = nil, span = nil, id:, email: nil, name: nil, session_id: nil, role: nil, scope: nil, **others
-      )
-        raise ArgumentError, 'missing required key: :id' if id.nil?
+      class << self
+        # Attach user information to the trace
+        #
+        # @param trace [TraceOperation] Trace to attach data to. Defaults to
+        #   active trace.
+        # @param span [SpanOperation] Span to attach data to. Defaults to
+        #   active span on trace. Note that this should be a service entry span.
+        #   When AppSec is enabled, the expected span and trace are automatically
+        #   used as defaults.
+        # @param id [String] Mandatory. Username or client id extracted
+        #   from the access token or Authorization header in the inbound request
+        #   from outside the system.
+        # @param email [String] Email of the authenticated user associated
+        #   to the trace.
+        # @param name [String] User-friendly name. To be displayed in the
+        #   UI if set.
+        # @param session_id [String] Session ID of the authenticated user.
+        # @param role [String] Actual/assumed role the client is making
+        #   the request under extracted from token or application security
+        #   context.
+        # @param scope [String] Scopes or granted authorities the client
+        #   currently possesses extracted from token or application security
+        #   context. The value would come from the scope associated with an OAuth
+        #   2.0 Access Token or an attribute value in a SAML 2.0 Assertion.
+        # @param others [Hash<Symbol, String>] Additional free-form
+        #   user information to attach to the trace.
+        #
+        # rubocop:disable Metrics/CyclomaticComplexity
+        # rubocop:disable Metrics/PerceivedComplexity
+        def set_user(
+          trace = nil, span = nil, id:, email: nil, name: nil, session_id: nil, role: nil, scope: nil, **others
+        )
+          raise ArgumentError, 'missing required key: :id' if id.nil?
 
-        # enforce types
+          # enforce types
 
-        raise TypeError, ':id must be a String'         unless id.is_a?(String)
-        raise TypeError, ':email must be a String'      unless email.nil? || email.is_a?(String)
-        raise TypeError, ':name must be a String'       unless name.nil? || name.is_a?(String)
-        raise TypeError, ':session_id must be a String' unless session_id.nil? || session_id.is_a?(String)
-        raise TypeError, ':role must be a String'       unless role.nil? || role.is_a?(String)
-        raise TypeError, ':scope must be a String'      unless scope.nil? || scope.is_a?(String)
+          raise TypeError, ':id must be a String'         unless id.is_a?(String)
+          raise TypeError, ':email must be a String'      unless email.nil? || email.is_a?(String)
+          raise TypeError, ':name must be a String'       unless name.nil? || name.is_a?(String)
+          raise TypeError, ':session_id must be a String' unless session_id.nil? || session_id.is_a?(String)
+          raise TypeError, ':role must be a String'       unless role.nil? || role.is_a?(String)
+          raise TypeError, ':scope must be a String'      unless scope.nil? || scope.is_a?(String)
 
-        others.each do |k, v|
-          raise TypeError, "#{k.inspect} must be a String" unless v.nil? || v.is_a?(String)
-        end
+          others.each do |k, v|
+            raise TypeError, "#{k.inspect} must be a String" unless v.nil? || v.is_a?(String)
+          end
+
+          set_trace_and_span_context('set_user', trace, span) do |_active_trace, active_span|
+            # set tags once data is known consistent
+            active_span.set_tag('usr.id', id)
+            active_span.set_tag('usr.email', email)           unless email.nil?
+            active_span.set_tag('usr.name', name)             unless name.nil?
+            active_span.set_tag('usr.session_id', session_id) unless session_id.nil?
+            active_span.set_tag('usr.role', role)             unless role.nil?
+            active_span.set_tag('usr.scope', scope)           unless scope.nil?
 
-        if (appsec_scope = Datadog::AppSec.active_scope)
-          trace = appsec_scope.trace
-          span = appsec_scope.service_entry_span
+            others.each do |k, v|
+              active_span.set_tag("usr.#{k}", v) unless v.nil?
+            end
+
+            if Datadog::AppSec.active_scope
+              user = ::Datadog::AppSec::Instrumentation::Gateway::User.new(id)
+              ::Datadog::AppSec::Instrumentation.gateway.push('identity.set_user', user)
+            end
+          end
         end
+        # rubocop:enable Metrics/PerceivedComplexity
+        # rubocop:enable Metrics/CyclomaticComplexity
 
-        trace ||= Datadog::Tracing.active_trace
-        span ||= trace.active_span || Datadog::Tracing.active_span
+        private
 
-        raise ArgumentError, "span #{span.span_id} does not belong to trace #{trace.id}" if trace.id != span.trace_id
+        def set_trace_and_span_context(method, trace = nil, span = nil)
+          if (appsec_scope = Datadog::AppSec.active_scope)
+            trace = appsec_scope.trace
+            span = appsec_scope.service_entry_span
+          end
 
-        # set tags once data is known consistent
+          trace ||= Datadog::Tracing.active_trace
+          span ||= trace && trace.active_span || Datadog::Tracing.active_span
 
-        span.set_tag('usr.id', id)
-        span.set_tag('usr.email', email)           unless email.nil?
-        span.set_tag('usr.name', name)             unless name.nil?
-        span.set_tag('usr.session_id', session_id) unless session_id.nil?
-        span.set_tag('usr.role', role)             unless role.nil?
-        span.set_tag('usr.scope', scope)           unless scope.nil?
+          unless trace && span
+            Datadog.logger.debug(
+              "Tracing not enabled. Method ##{method} is a no-op. Please enable tracing if you want ##{method}"\
+              ' to track this events'
+            )
+            return
+          end
 
-        others.each do |k, v|
-          span.set_tag("usr.#{k}", v) unless v.nil?
-        end
+          raise ArgumentError, "span #{span.span_id} does not belong to trace #{trace.id}" if trace.id != span.trace_id
 
-        if appsec_scope
-          user = ::Datadog::AppSec::Instrumentation::Gateway::User.new(id)
-          ::Datadog::AppSec::Instrumentation.gateway.push('identity.set_user', user)
+          yield(trace, span)
         end
       end
-      # rubocop:enable Metrics/AbcSize
-      # rubocop:enable Metrics/PerceivedComplexity
-      # rubocop:enable Metrics/CyclomaticComplexity
     end
   end
 end

data/lib/datadog/profiling/collectors/cpu_and_wall_time_worker.rb

@@ -21,11 +21,13 @@ module Datadog
           gc_profiling_enabled:,
           allocation_counting_enabled:,
           no_signals_workaround_enabled:,
+          timeline_enabled:,
           thread_context_collector: ThreadContext.new(
             recorder: recorder,
             max_frames: max_frames,
             tracer: tracer,
             endpoint_collection_enabled: endpoint_collection_enabled,
+            timeline_enabled: timeline_enabled,
           ),
           idle_sampling_helper: IdleSamplingHelper.new,
           # **NOTE**: This should only be used for testing; disabling the dynamic sampling rate will increase the
@@ -76,6 +78,7 @@ module Datadog
                 )
               end
             end
+            @worker_thread.name = self.class.name # Repeated from above to make sure thread gets named asap
           end
 
           true

data/lib/datadog/profiling/collectors/idle_sampling_helper.rb

@@ -42,6 +42,7 @@ module Datadog
                 )
               end
             end
+            @worker_thread.name = self.class.name # Repeated from above to make sure thread gets named asap
           end
 
           true

data/lib/datadog/profiling/collectors/thread_context.rb

@@ -14,9 +14,16 @@ module Datadog
       #
       # Methods prefixed with _native_ are implemented in `collectors_thread_context.c`
       class ThreadContext
-        def initialize(recorder:, max_frames:, tracer:, endpoint_collection_enabled:)
+        def initialize(recorder:, max_frames:, tracer:, endpoint_collection_enabled:, timeline_enabled:)
           tracer_context_key = safely_extract_context_key_from(tracer)
-          self.class._native_initialize(self, recorder, max_frames, tracer_context_key, endpoint_collection_enabled)
+          self.class._native_initialize(
+            self,
+            recorder,
+            max_frames,
+            tracer_context_key,
+            endpoint_collection_enabled,
+            timeline_enabled,
+          )
         end
 
         def inspect

data/lib/datadog/profiling/component.rb

@@ -7,7 +7,10 @@ module Datadog
       # Passing in a `nil` tracer is supported and will disable the following profiling features:
       # * Code Hotspots panel in the trace viewer, as well as scoping a profile down to a span
       # * Endpoint aggregation in the profiler UX, including normalization (resource per endpoint call)
-      def self.build_profiler_component(settings:, agent_settings:, optional_tracer:)
+      def self.build_profiler_component(settings:, agent_settings:, optional_tracer:) # rubocop:disable Metrics/MethodLength
+        require_relative '../profiling/diagnostics/environment_logger'
+        Profiling::Diagnostics::EnvironmentLogger.collect_and_log!
+
         return unless settings.profiling.enabled
 
         # Workaround for weird dependency direction: the Core::Configuration::Components class currently has a
@@ -64,7 +67,13 @@ module Datadog
 
         # NOTE: Please update the Initialization section of ProfilingDevelopment.md with any changes to this method
 
+        no_signals_workaround_enabled = false
+        timeline_enabled = false
+
         if enable_new_profiler?(settings)
+          no_signals_workaround_enabled = no_signals_workaround_enabled?(settings)
+          timeline_enabled = settings.profiling.advanced.experimental_timeline_enabled
+
           recorder = Datadog::Profiling::StackRecorder.new(
             cpu_time_enabled: RUBY_PLATFORM.include?('linux'), # Only supported on Linux currently
             alloc_samples_enabled: false, # Always disabled for now -- work in progress
@@ -76,7 +85,8 @@ module Datadog
             endpoint_collection_enabled: settings.profiling.advanced.endpoint.collection.enabled,
             gc_profiling_enabled: enable_gc_profiling?(settings),
             allocation_counting_enabled: settings.profiling.advanced.allocation_counting_enabled,
-            no_signals_workaround_enabled: no_signals_workaround_enabled?(settings),
+            no_signals_workaround_enabled: no_signals_workaround_enabled,
+            timeline_enabled: timeline_enabled,
           )
         else
           load_pprof_support
@@ -85,7 +95,12 @@ module Datadog
           collector = build_profiler_oldstack_collector(settings, recorder, optional_tracer)
         end
 
-        exporter = build_profiler_exporter(settings, recorder)
+        internal_metadata = {
+          no_signals_workaround_enabled: no_signals_workaround_enabled,
+          timeline_enabled: timeline_enabled,
+        }.freeze
+
+        exporter = build_profiler_exporter(settings, recorder, internal_metadata: internal_metadata)
         transport = build_profiler_transport(settings, agent_settings)
         scheduler = Profiling::Scheduler.new(exporter: exporter, transport: transport)
 
@@ -96,11 +111,15 @@ module Datadog
         Profiling::OldRecorder.new([Profiling::Events::StackSample], settings.profiling.advanced.max_events)
       end
 
-      private_class_method def self.build_profiler_exporter(settings, recorder)
+      private_class_method def self.build_profiler_exporter(settings, recorder, internal_metadata:)
         code_provenance_collector =
           (Profiling::Collectors::CodeProvenance.new if settings.profiling.advanced.code_provenance_enabled)
 
-        Profiling::Exporter.new(pprof_recorder: recorder, code_provenance_collector: code_provenance_collector)
+        Profiling::Exporter.new(
+          pprof_recorder: recorder,
+          code_provenance_collector: code_provenance_collector,
+          internal_metadata: internal_metadata,
+        )
       end
 
       private_class_method def self.build_profiler_oldstack_collector(settings, old_recorder, tracer)
@@ -198,7 +217,7 @@ module Datadog
         if Gem.loaded_specs['mysql2'] && incompatible_libmysqlclient_version?(settings)
           Datadog.logger.warn(
             'Enabling the profiling "no signals" workaround because an incompatible version of the mysql2 gem is ' \
-            'installed. Profiling data will have lower quality.' \
+            'installed. Profiling data will have lower quality. ' \
             'To fix this, upgrade the libmysqlclient in your OS image to version 8.0.0 or above.'
           )
           return true
@@ -214,6 +233,16 @@ module Datadog
           return true
         end
 
+        if defined?(::PhusionPassenger)
+          Datadog.logger.warn(
+            'Enabling the profiling "no signals" workaround because the passenger web server is in use. ' \
+            'This is needed because passenger is currently incompatible with the normal working mode ' \
+            'of the profiler, as detailed in <https://github.com/DataDog/dd-trace-rb/issues/2976>. ' \
+            'Profiling data will have lower quality.'
+          )
+          return true
+        end
+
         false
       end
 
@@ -236,9 +265,22 @@ module Datadog
         begin
           require 'mysql2'
 
-          return true unless defined?(Mysql2::Client) && Mysql2::Client.respond_to?(:info)
-
-          libmysqlclient_version = Gem::Version.new(Mysql2::Client.info[:version])
+          # The mysql2-aurora gem likes to monkey patch itself in replacement of Mysql2::Client, and uses
+          # `method_missing` to delegate to the original BUT unfortunately does not implement `respond_to_missing?` and
+          # thus our `respond_to?(:info)` below was failing.
+          #
+          # But on the bright side, the gem does stash a reference to the original Mysql2::Client class in a constant,
+          # so if that constant exists, we use that for our probing.
+          mysql2_client_class =
+            if defined?(Mysql2::Aurora::ORIGINAL_CLIENT_CLASS)
+              Mysql2::Aurora::ORIGINAL_CLIENT_CLASS
+            elsif defined?(Mysql2::Client)
+              Mysql2::Client
+            end
+
+          return true unless mysql2_client_class && mysql2_client_class.respond_to?(:info)
+
+          libmysqlclient_version = Gem::Version.new(mysql2_client_class.info[:version])
 
           compatible = libmysqlclient_version >= Gem::Version.new('8.0.0')
 

data/lib/datadog/profiling/diagnostics/environment_logger.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require 'date'
+require 'json'
+require 'rbconfig'
+require_relative '../../core/diagnostics/environment_logger'
+
+module Datadog
+  module Profiling
+    module Diagnostics
+      # Collects and logs Profiling diagnostic information
+      module EnvironmentLogger
+        extend Core::Diagnostics::EnvironmentLogging
+
+        def self.collect_and_log!
+          log_once! do
+            data = EnvironmentCollector.collect_config!
+            log_configuration!('PROFILING', data.to_json)
+          end
+        rescue => e
+          logger.warn("Failed to collect profiling environment information: #{e} Location: #{Array(e.backtrace).first}")
+        end
+      end
+
+      # Collects environment information for Profiling diagnostic logging
+      module EnvironmentCollector
+        def self.collect_config!(*args)
+          {
+            profiling_enabled: profiling_enabled
+          }
+        end
+
+        def self.profiling_enabled
+          !!Datadog.configuration.profiling.enabled
+        end
+      end
+    end
+  end
+end

data/lib/datadog/profiling/exporter.rb

@@ -22,13 +22,15 @@ module Datadog
         :minimum_duration_seconds,
         :time_provider,
         :last_flush_finish_at,
-        :created_at
+        :created_at,
+        :internal_metadata
 
       public
 
       def initialize(
         pprof_recorder:,
         code_provenance_collector:,
+        internal_metadata:,
         minimum_duration_seconds: PROFILE_DURATION_THRESHOLD_SECONDS,
         time_provider: Time
       )
@@ -38,6 +40,7 @@ module Datadog
         @time_provider = time_provider
         @last_flush_finish_at = nil
         @created_at = time_provider.now.utc
+        @internal_metadata = internal_metadata
       end
 
       def flush
@@ -61,6 +64,7 @@ module Datadog
           code_provenance_file_name: Datadog::Profiling::Ext::Transport::HTTP::CODE_PROVENANCE_FILENAME,
           code_provenance_data: uncompressed_code_provenance,
           tags_as_array: Datadog::Profiling::TagBuilder.call(settings: Datadog.configuration).to_a,
+          internal_metadata: internal_metadata,
         )
       end
 

data/lib/datadog/profiling/flush.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'json'
+
 module Datadog
   module Profiling
     # Represents a collection of events of a specific type being flushed.
@@ -14,7 +16,8 @@ module Datadog
         :pprof_data, # gzipped pprof bytes
         :code_provenance_file_name,
         :code_provenance_data, # gzipped json bytes
-        :tags_as_array
+        :tags_as_array,
+        :internal_metadata_json
 
       def initialize(
         start:,
@@ -23,7 +26,8 @@ module Datadog
         pprof_data:,
         code_provenance_file_name:,
         code_provenance_data:,
-        tags_as_array:
+        tags_as_array:,
+        internal_metadata:
       )
         @start = start
         @finish = finish
@@ -32,6 +36,7 @@ module Datadog
         @code_provenance_file_name = code_provenance_file_name
         @code_provenance_data = code_provenance_data
         @tags_as_array = tags_as_array
+        @internal_metadata_json = JSON.fast_generate(internal_metadata.map { |k, v| [k, v.to_s] }.to_h)
       end
     end
   end

data/lib/datadog/profiling/http_transport.rb

@@ -40,6 +40,7 @@ module Datadog
           code_provenance_data: flush.code_provenance_data,
 
           tags_as_array: flush.tags_as_array,
+          internal_metadata_json: flush.internal_metadata_json,
         )
 
         if status == :ok
@@ -47,11 +48,14 @@ module Datadog
             Datadog.logger.debug('Successfully reported profiling data')
             true
           else
-            Datadog.logger.error("Failed to report profiling data: server returned unexpected HTTP #{result} status code")
+            Datadog.logger.error(
+              "Failed to report profiling data (#{config_without_api_key}): " \
+              "server returned unexpected HTTP #{result} status code"
+            )
             false
           end
         else
-          Datadog.logger.error("Failed to report profiling data: #{result}")
+          Datadog.logger.error("Failed to report profiling data (#{config_without_api_key}): #{result}")
           false
         end
       end
@@ -109,7 +113,8 @@ module Datadog
         pprof_data:,
         code_provenance_file_name:,
         code_provenance_data:,
-        tags_as_array:
+        tags_as_array:,
+        internal_metadata_json:
       )
         self.class._native_do_export(
           exporter_configuration,
@@ -123,8 +128,13 @@ module Datadog
           code_provenance_file_name,
           code_provenance_data,
           tags_as_array,
+          internal_metadata_json,
         )
       end
+
+      def config_without_api_key
+        [@exporter_configuration[0..1]].to_h
+      end
     end
   end
 end

data/lib/datadog/profiling/load_native_extension.rb

@@ -9,7 +9,13 @@
 # All code on this file is on-purpose at the top-level; this makes it so this file is executed only once,
 # the first time it gets required, to avoid any issues with the native extension being initialized more than once.
 
-require "ddtrace_profiling_loader.#{RUBY_VERSION}_#{RUBY_PLATFORM}"
+begin
+  require "ddtrace_profiling_loader.#{RUBY_VERSION}_#{RUBY_PLATFORM}"
+rescue LoadError => e
+  raise LoadError,
+    'Failed to load the profiling loader extension. To fix this, please remove and then reinstall ddtrace ' \
+    "(Details: #{e.message})"
+end
 
 extension_name = "ddtrace_profiling_native_extension.#{RUBY_VERSION}_#{RUBY_PLATFORM}"
 full_file_path = "#{__dir__}/../../#{extension_name}.#{RbConfig::CONFIG['DLEXT']}"

data/lib/datadog/profiling.rb

@@ -66,6 +66,11 @@ module Datadog
       nil
     end
 
+    def self.enabled?
+      profiler = Datadog.send(:components).profiler
+      !!(profiler.scheduler.running? if profiler)
+    end
+
     private_class_method def self.replace_noop_allocation_count
       def self.allocation_count # rubocop:disable Lint/DuplicateMethods, Lint/NestedMethodDefinition (On purpose!)
         Datadog::Profiling::Collectors::CpuAndWallTimeWorker._native_allocation_count
@@ -178,6 +183,10 @@ module Datadog
       end
     end
 
+    # All requires for the profiler should be directly added here; and everything should be loaded eagerly.
+    # (Currently there's a few exceptions for the old profiler, but we should avoid other exceptions.)
+    #
+    # All of the profiler should be loaded and ready to go when this method returns `true`.
     private_class_method def self.load_profiling
       return false unless supported?
 
@@ -189,6 +198,7 @@ module Datadog
       require_relative 'profiling/collectors/old_stack'
       require_relative 'profiling/collectors/stack'
       require_relative 'profiling/collectors/thread_context'
+      require_relative 'profiling/diagnostics/environment_logger'
       require_relative 'profiling/stack_recorder'
       require_relative 'profiling/old_recorder'
       require_relative 'profiling/exporter'
@@ -197,7 +207,8 @@ module Datadog
       require_relative 'profiling/profiler'
       require_relative 'profiling/native_extension'
       require_relative 'profiling/trace_identifiers/helper'
-      # This file is loaded in Profiling::Component#load_pprof_support; see notes there for why
+      # This file is no longer eagerly loaded as a workaround for an issue. It only gets loaded dynamically if the old
+      # profiler is in use. See Profiling::Component#load_pprof_support for more details.
       # require_relative 'profiling/pprof/pprof_pb'
       require_relative 'profiling/tag_builder'
       require_relative 'profiling/http_transport'