datadog 2.18.0 → 2.19.0

data/ext/datadog_profiling_native_extension/private_vm_api_access.c

@@ -212,21 +212,6 @@ uint64_t native_thread_id_for(VALUE thread) {
   #endif
 }
 
-// Returns the stack depth by using the same approach as rb_profile_frames and backtrace_each: get the positions
-// of the end and current frame pointers and subtracting them.
-ptrdiff_t stack_depth_for(VALUE thread) {
-  const rb_execution_context_t *ec = thread_struct_from_object(thread)->ec;
-  const rb_control_frame_t *cfp = ec->cfp, *end_cfp = RUBY_VM_END_CONTROL_FRAME(ec);
-
-  if (end_cfp == NULL) return 0;
-
-  // Skip dummy frame, as seen in `backtrace_each` (`vm_backtrace.c`) and our custom rb_profile_frames
-  // ( https://github.com/ruby/ruby/blob/4bd38e8120f2fdfdd47a34211720e048502377f1/vm_backtrace.c#L890-L914 )
-  end_cfp = RUBY_VM_NEXT_CONTROL_FRAME(end_cfp);
-
-  return end_cfp <= cfp ? 0 : end_cfp - cfp - 1;
-}
-
 // This was renamed in Ruby 3.2
 #if !defined(ccan_list_for_each) && defined(list_for_each)
   #define ccan_list_for_each list_for_each
@@ -360,11 +345,16 @@ calc_pos(const rb_iseq_t *iseq, const VALUE *pc, int *lineno, int *node_id)
         }
 #endif
 
-        // In PROF-11475 we spotted a crash when calling `rb_iseq_line_no` from this method. We couldn't reproduce or
-        // figure out the root cause, but "just in case", we're validating that the iseq looks valid and that the
-        // `n` used for the position is also sane, and if they don't look good, we don't calculate the line, rather
-        // than potentially trigger any issues.
-        if (RB_UNLIKELY(!RB_TYPE_P((VALUE) iseq, T_IMEMO) || n < 0 || n > ISEQ_BODY(iseq)->iseq_size)) return 0;
+        // In PROF-11475 we spotted a crash when calling `rb_iseq_line_no` from this method.
+        // We were only able to reproduce this issue on Ruby 2.6 and 2.7, not 2.5 or the 3.x series (tried 3.0, 3.2 and 3.4).
+        // Note that going out of bounds doesn't crash every time, as usual with C we may just read garbage or get lucky.
+        //
+        // For those problematic Rubies, we observed that when we try to take a sample in the middle of processing the
+        // VM `LEAVE` instruction, the value of `n` can violate the documented assumptions above and be
+        // `n > ISEQ_BODY(iseq)->iseq_size)`.
+        //
+        // To work around this and any other potential issues, we validate here that the bytecode position is sane.
+        if (RB_UNLIKELY(n < 0 || n > ISEQ_BODY(iseq)->iseq_size)) return 0;
 
         if (lineno) *lineno = rb_iseq_line_no(iseq, pos);
 #ifdef USE_ISEQ_NODE_ID
@@ -410,6 +400,9 @@ calc_lineno(const rb_iseq_t *iseq, const VALUE *pc)
 // * Add frame_flags.same_frame and logic to skip redoing work if the buffer already contains the same data we're collecting
 // * Skipped use of rb_callable_method_entry_t (cme) for Ruby frames as it doesn't impact us.
 // * Imported fix from https://github.com/ruby/ruby/pull/8280 to keep us closer to upstream
+// * Added potential fix for https://github.com/ruby/ruby/pull/13643 (this one is a just-in-case, unclear if it happens
+//   for ddtrace)
+// * Reversed order of iteration to better enable caching
 //
 // What is rb_profile_frames?
 // `rb_profile_frames` is a Ruby VM debug API added for use by profilers for sampling the stack trace of a Ruby thread.
@@ -445,6 +438,16 @@ int ddtrace_rb_profile_frames(VALUE thread, int start, int limit, frame_info *st
     // support sampling any thread (including the current) passed as an argument
     rb_thread_t *th = thread_struct_from_object(thread);
     const rb_execution_context_t *ec = th->ec;
+
+    // As of this writing, we don't support profiling with MN enabled, and this only happens in that mode, but as we
+    // probably want to experiment with it in the future, I've decided to import https://github.com/ruby/ruby/pull/9310
+    // here.
+    if (ec == NULL) return 0;
+
+    // I suspect this won't happen for ddtrace, but just-in-case we've imported a potential fix for
+    // https://github.com/ruby/ruby/pull/13643 by assuming that these can be NULL/zero with the cfp being non-NULL yet.
+    if (ec->vm_stack == NULL || ec->vm_stack_size == 0) return 0;
+
     const rb_control_frame_t *cfp = ec->cfp, *end_cfp = RUBY_VM_END_CONTROL_FRAME(ec);
     #ifndef NO_JIT_RETURN
       const rb_control_frame_t *top = cfp;
@@ -462,11 +465,6 @@ int ddtrace_rb_profile_frames(VALUE thread, int start, int limit, frame_info *st
     // it from https://github.com/ruby/ruby/pull/7116 in a "just in case" kind of mindset.
     if (cfp == NULL) return 0;
 
-    // As of this writing, we don't support profiling with MN enabled, and this only happens in that mode, but as we
-    // probably want to experiment with it in the future, I've decided to import https://github.com/ruby/ruby/pull/9310
-    // here.
-    if (ec == NULL) return 0;
-
     // Fix: Skip dummy frame that shows up in main thread.
     //
     // According to a comment in `backtrace_each` (`vm_backtrace.c`), there's two dummy frames that we should ignore
@@ -483,7 +481,20 @@ int ddtrace_rb_profile_frames(VALUE thread, int start, int limit, frame_info *st
     // See comment on `record_placeholder_stack_in_native_code` for a full explanation of what this means (and why we don't just return 0)
     if (end_cfp <= cfp) return PLACEHOLDER_STACK_IN_NATIVE_CODE;
 
-    for (i=0; i<limit && cfp != end_cfp; cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp)) {
+    // This is the position just after the top of the stack -- e.g. where a new frame pushed on the stack would end up.
+    const rb_control_frame_t *top_sentinel = RUBY_VM_NEXT_CONTROL_FRAME(cfp);
+
+    // We iterate the stack from bottom (beginning of thread) to the top (currently-active frame). This is different
+    // from upstream rb_profile_frames, but actually matches what `backtrace_each` does (yes, different Ruby VM APIs
+    // iterate in different directions).
+    // We do this to better take advantage of the `same_frame` caching mechanism: By starting from the bottom of the
+    // stack towards the top, we can usually keep most of the stack intact when the code is only going up and down
+    // a few methods at the top. Before this change, the cache was really only useful if between samples the app had
+    // not moved from the current stack, as adding or removing one frame would invalidate the existing cache (because
+    // every position would shift).
+    cfp = RUBY_VM_NEXT_CONTROL_FRAME(end_cfp);
+
+    for (i=0; i<limit && cfp != top_sentinel; cfp = RUBY_VM_NEXT_CONTROL_FRAME(cfp)) {
         if (cfp->iseq && !cfp->pc) {
           // Fix: Do nothing -- this frame should not be used
           //

data/ext/datadog_profiling_native_extension/private_vm_api_access.h

@@ -41,7 +41,6 @@ rb_nativethread_id_t pthread_id_for(VALUE thread);
 bool is_current_thread_holding_the_gvl(void);
 current_gvl_owner gvl_owner(void);
 uint64_t native_thread_id_for(VALUE thread);
-ptrdiff_t stack_depth_for(VALUE thread);
 void ddtrace_thread_list(VALUE result_array);
 bool is_thread_alive(VALUE thread);
 VALUE thread_name_for(VALUE thread);

data/ext/datadog_profiling_native_extension/ruby_helpers.h

@@ -72,7 +72,7 @@ NORETURN(void raise_syserr(
 // reference a valid object (in which case value is not changed).
 //
 // Note: GVL can be released and other threads may get to run before this method returns
-bool ruby_ref_from_id(size_t id, VALUE *value);
+bool ruby_ref_from_id(VALUE obj_id, VALUE *value);
 
 // Native wrapper to get the approximate/estimated current size of the passed
 // object.

data/lib/datadog/appsec/api_security/route_extractor.rb

@@ -10,6 +10,7 @@ module Datadog
         GRAPE_ROUTE_KEY = 'grape.routing_args'
         RAILS_ROUTE_KEY = 'action_dispatch.route_uri_pattern'
         RAILS_ROUTES_KEY = 'action_dispatch.routes'
+        RAILS_PATH_PARAMS_KEY = 'action_dispatch.request.path_parameters'
         RAILS_FORMAT_SUFFIX = '(.:format)'
 
         # HACK: We rely on the fact that each contrib will modify `request.env`
@@ -36,6 +37,11 @@ module Datadog
         #         "/users/:id(.:format)"
         #
         # WARNING: This method works only *after* the request has been routed.
+        #
+        # WARNING: In Rails > 7.1 when a route was not found,
+        #          action_dispatch.route_uri_pattern will not be set.
+        #          In Rails < 7.1 it also will not be set even if a route was found,
+        #          but in this case  action_dispatch.request.path_parameters won't be empty.
         def self.route_pattern(request)
           if request.env.key?(GRAPE_ROUTE_KEY)
             pattern = request.env[GRAPE_ROUTE_KEY][:route_info]&.pattern&.origin
@@ -45,7 +51,7 @@ module Datadog
             "#{request.script_name}#{pattern}"
           elsif request.env.key?(RAILS_ROUTE_KEY)
             request.env[RAILS_ROUTE_KEY].delete_suffix(RAILS_FORMAT_SUFFIX)
-          elsif request.env.key?(RAILS_ROUTES_KEY)
+          elsif request.env.key?(RAILS_ROUTES_KEY) && !request.env.fetch(RAILS_PATH_PARAMS_KEY, {}).empty?
             pattern = request.env[RAILS_ROUTES_KEY].router
               .recognize(request) { |route, _| break route.path.spec.to_s }
 

data/lib/datadog/appsec/instrumentation/gateway/argument.rb

@@ -13,7 +13,7 @@ module Datadog
         class User < Argument
           attr_reader :id, :login, :session_id
 
-          def initialize(id, login = nil, session_id = nil)
+          def initialize(id = nil, login = nil, session_id = nil)
             super()
 
             @id = id

data/lib/datadog/core/configuration/settings.rb

@@ -554,6 +554,26 @@ module Datadog
               o.env 'DD_PROFILING_NATIVE_FILENAMES_ENABLED'
               o.default true
             end
+
+            # Controls if the profiler should sample directly from the signal handler.
+            # Sampling directly from the signal handler improves accuracy of the data collected.
+            #
+            # We recommend using this setting with Ruby 3.2.5+ / Ruby 3.3.4+ and above
+            # as they include additional safety measures added in https://github.com/ruby/ruby/pull/11036.
+            # We have not validated it thoroughly with earlier versions, but in practice it should work on Ruby 3.0+
+            # (the key change was https://github.com/ruby/ruby/pull/3296).
+            #
+            # Enabling this on Ruby 2 is not recommended as it may cause VM crashes and/or incorrect data.
+            #
+            # @default true on Ruby 3.2.5+ / Ruby 3.3.4+, false on older Rubies
+            option :sighandler_sampling_enabled do |o|
+              o.type :bool
+              o.env 'DD_PROFILING_SIGHANDLER_SAMPLING_ENABLED'
+              o.default do
+                Gem::Version.new(RUBY_VERSION) >= Gem::Version.new('3.2.5') &&
+                  !(RUBY_VERSION.start_with?('3.3.') && Gem::Version.new(RUBY_VERSION) < Gem::Version.new('3.3.4'))
+              end
+            end
           end
 
           # @public_api

data/lib/datadog/core/telemetry/component.rb

@@ -62,7 +62,9 @@ module Datadog
           return unless @enabled
 
           @transport = if settings.telemetry.agentless_enabled
-            agent_settings = Core::Configuration::AgentlessSettingsResolver.call(
+            # We don't touch the `agent_settings` since we still want the telemetry payloads to refer to the original
+            # settings, even though the telemetry itself may be using a different path.
+            telemetry_specific_agent_settings = Core::Configuration::AgentlessSettingsResolver.call(
               settings,
               host_prefix: 'instrumentation-telemetry-intake',
               url_override: settings.telemetry.agentless_url_override,
@@ -70,7 +72,7 @@ module Datadog
               logger: logger,
             )
             Telemetry::Transport::HTTP.agentless_telemetry(
-              agent_settings: agent_settings,
+              agent_settings: telemetry_specific_agent_settings,
               logger: logger,
               # api_key should have already validated to be
               # not nil by +build+ method above.
@@ -96,6 +98,8 @@ module Datadog
             logger: logger,
             shutdown_timeout: settings.telemetry.shutdown_timeout_seconds,
           )
+
+          @agent_settings = agent_settings
         end
 
         def disable!
@@ -107,9 +111,9 @@ module Datadog
           return if !@enabled
 
           initial_event = if initial_event_is_change
-            Event::SynthAppClientConfigurationChange.new
+            Event::SynthAppClientConfigurationChange.new(agent_settings: @agent_settings)
           else
-            Event::AppStarted.new
+            Event::AppStarted.new(agent_settings: @agent_settings)
           end
 
           @worker.start(initial_event)

data/lib/datadog/core/telemetry/event/app_started.rb

@@ -8,6 +8,10 @@ module Datadog
       module Event
         # Telemetry class for the 'app-started' event
         class AppStarted < Base
+          def initialize(agent_settings:)
+            @agent_settings = agent_settings
+          end
+
           def type
             'app-started'
           end
@@ -96,7 +100,7 @@ module Datadog
               ),
 
               # Mix of env var, programmatic and default config, so we use unknown
-              conf_value('DD_AGENT_TRANSPORT', agent_transport(config), seq_id, 'unknown'),
+              conf_value('DD_AGENT_TRANSPORT', agent_transport, seq_id, 'unknown'),
 
               # writer_options is defined as an option that has a Hash value.
               conf_value(
@@ -162,6 +166,20 @@ module Datadog
               )
             end
 
+            instrumentation_source = if Datadog.const_defined?(:SingleStepInstrument, false) &&
+                Datadog::SingleStepInstrument.const_defined?(:LOADED, false) &&
+                Datadog::SingleStepInstrument::LOADED
+              'ssi'
+            else
+              'manual'
+            end
+            # Track ssi configurations
+            list.push(
+              conf_value('instrumentation_source', instrumentation_source, seq_id, 'code'),
+              conf_value('DD_INJECT_FORCE', Core::Environment::VariableHelpers.env_to_bool('DD_INJECT_FORCE', false), seq_id, 'env_var'),
+              conf_value('DD_INJECTION_ENABLED', ENV['DD_INJECTION_ENABLED'] || '', seq_id, 'env_var'),
+            )
+
             # Add some more custom additional payload values here
             if config.logger.instance
               list << conf_value(
@@ -200,8 +218,8 @@ module Datadog
           # standard:enable Metrics/AbcSize
           # standard:enable Metrics/MethodLength
 
-          def agent_transport(config)
-            adapter = Core::Configuration::AgentSettingsResolver.call(config).adapter
+          def agent_transport
+            adapter = @agent_settings.adapter
             if adapter == Datadog::Core::Transport::Ext::UnixSocket::ADAPTER
               'UDS'
             else

data/lib/datadog/di/instrumenter.rb

@@ -121,9 +121,8 @@ module Datadog
             if rate_limiter.nil? || rate_limiter.allow?
               # Arguments may be mutated by the method, therefore
               # they need to be serialized prior to method invocation.
-              entry_args = if probe.capture_snapshot?
-                instance_vars = Instrumenter.get_instance_variables(self)
-                serializer.serialize_args(args, kwargs, instance_vars,
+              serialized_entry_args = if probe.capture_snapshot?
+                serializer.serialize_args(args, kwargs, self,
                   depth: probe.max_capture_depth || settings.dynamic_instrumentation.max_capture_depth,
                   attribute_count: probe.max_capture_attribute_count || settings.dynamic_instrumentation.max_capture_attribute_count)
               end
@@ -160,9 +159,10 @@ module Datadog
               caller_locs = method_frame + caller_locations # steep:ignore
               # TODO capture arguments at exit
               # & is to stop steep complaints, block is always present here.
-              block&.call(probe: probe, rv: rv, duration: duration, caller_locations: caller_locs,
-                instance_vars: probe.capture_snapshot? ? Instrumenter.get_instance_variables(self) : nil,
-                serialized_entry_args: entry_args)
+              block&.call(probe: probe, rv: rv,
+                duration: duration, caller_locations: caller_locs,
+                target_self: self,
+                serialized_entry_args: serialized_entry_args)
               rv
             else
               # stop standard from trying to mess up my code
@@ -311,19 +311,20 @@ module Datadog
               probe.file == tp.path || probe.file_matches?(tp.path)
             )
               if rate_limiter.nil? || rate_limiter.allow?
-                locals = if probe.capture_snapshot?
+                serialized_locals = if probe.capture_snapshot?
                   serializer.serialize_vars(Instrumenter.get_local_variables(tp),
                     depth: probe.max_capture_depth || settings.dynamic_instrumentation.max_capture_depth,
                     attribute_count: probe.max_capture_attribute_count || settings.dynamic_instrumentation.max_capture_attribute_count,)
                 end
-                instance_vars = if probe.capture_snapshot?
-                  serializer.serialize_vars(Instrumenter.get_instance_variables(tp.self),
+                if probe.capture_snapshot?
+                  serializer.serialize_value(tp.self,
                     depth: probe.max_capture_depth || settings.dynamic_instrumentation.max_capture_depth,
                     attribute_count: probe.max_capture_attribute_count || settings.dynamic_instrumentation.max_capture_attribute_count,)
                 end
                 # & is to stop steep complaints, block is always present here.
                 block&.call(probe: probe,
-                  locals: locals, instance_vars: instance_vars,
+                  serialized_locals: serialized_locals,
+                  target_self: tp.self,
                   path: tp.path, caller_locations: caller_locations)
               end
             end
@@ -397,14 +398,6 @@ module Datadog
       end
 
       class << self
-        def get_instance_variables(object)
-          {}.tap do |hash|
-            object.instance_variables.each do |var|
-              hash[var] = object.instance_variable_get(var)
-            end
-          end
-        end
-
         def get_local_variables(trace_point)
           # binding appears to be constructed on access, therefore
           # 1) we should attempt to cache it and

data/lib/datadog/di/probe_notification_builder.rb

@@ -42,9 +42,9 @@ module Datadog
       # path is the actual path of the instrumented file.
       def build_executed(probe,
         path: nil, rv: nil, duration: nil, caller_locations: nil,
-        locals: nil, args: nil, kwargs: nil, instance_vars: nil,
+        serialized_locals: nil, args: nil, kwargs: nil, target_self: nil,
         serialized_entry_args: nil)
-        build_snapshot(probe, rv: rv, locals: locals,
+        build_snapshot(probe, rv: rv, serialized_locals: serialized_locals,
           # Actual path of the instrumented file.
           path: path,
           duration: duration,
@@ -52,14 +52,23 @@ module Datadog
           # this should be all frames for enriched probes and no frames for
           # non-enriched probes?
           caller_locations: caller_locations,
-          args: args, kwargs: kwargs, instance_vars: instance_vars,
+          args: args, kwargs: kwargs,
+          target_self: target_self,
           serialized_entry_args: serialized_entry_args)
       end
 
-      def build_snapshot(probe, rv: nil, locals: nil, path: nil,
+      def build_snapshot(probe, rv: nil, serialized_locals: nil, path: nil,
+        # In Ruby everything is a method, therefore we should always have
+        # a target self. However, if we are not capturing a snapshot,
+        # there is no need to pass in the target self.
+        target_self: nil,
         duration: nil, caller_locations: nil,
-        args: nil, kwargs: nil, instance_vars: nil,
+        args: nil, kwargs: nil,
         serialized_entry_args: nil)
+        if probe.capture_snapshot? && !target_self
+          raise ArgumentError, "Asked to build snapshot with snapshot capture but target_self is nil"
+        end
+
         # TODO also verify that non-capturing probe does not pass
         # snapshot or vars/args into this method
         captures = if probe.capture_snapshot?
@@ -68,25 +77,18 @@ module Datadog
               "@return": serializer.serialize_value(rv,
                 depth: probe.max_capture_depth || settings.dynamic_instrumentation.max_capture_depth,
                 attribute_count: probe.max_capture_attribute_count || settings.dynamic_instrumentation.max_capture_attribute_count),
+              self: serializer.serialize_value(target_self),
             }
-            if instance_vars
-              return_arguments.update(
-                serializer.serialize_vars(instance_vars,
-                  depth: probe.max_capture_depth || settings.dynamic_instrumentation.max_capture_depth,
-                  attribute_count: probe.max_capture_attribute_count || settings.dynamic_instrumentation.max_capture_attribute_count,)
-              )
-            end
             {
               entry: {
                 # standard:disable all
                 arguments: if serialized_entry_args
                   serialized_entry_args
                 else
-                  (args || kwargs) && serializer.serialize_args(args, kwargs, instance_vars,
+                  (args || kwargs) && serializer.serialize_args(args, kwargs, target_self,
                     depth: probe.max_capture_depth || settings.dynamic_instrumentation.max_capture_depth,
                     attribute_count: probe.max_capture_attribute_count || settings.dynamic_instrumentation.max_capture_attribute_count)
                 end,
-                throwable: nil,
                 # standard:enable all
               },
               return: {
@@ -96,8 +98,11 @@ module Datadog
             }
           elsif probe.line?
             {
-              lines: locals && {
-                probe.line_no => {locals: locals.merge(instance_vars || {})},
+              lines: serialized_locals && {
+                probe.line_no => {
+                  locals: serialized_locals,
+                  arguments: {self: serializer.serialize_value(target_self)},
+                },
               },
             }
           end

data/lib/datadog/di/serializer.rb

@@ -36,6 +36,10 @@ module Datadog
     # efficient but there would be additional overhead from passing this
     # parameter all the time and the API would get more complex.
     #
+    # Note: "self" cannot be used as a parameter name in Ruby, therefore
+    # there should never be a conflict between instance variable
+    # serialization and method parameters.
+    #
     # @api private
     class Serializer
       # Third-party library integration / custom serializers.
@@ -86,7 +90,7 @@ module Datadog
       # Instance variables are technically a hash just like kwargs,
       # we take them as a separate parameter to avoid a hash merge
       # in upstream code.
-      def serialize_args(args, kwargs, instance_vars,
+      def serialize_args(args, kwargs, target_self,
         depth: settings.dynamic_instrumentation.max_capture_depth,
         attribute_count: settings.dynamic_instrumentation.max_capture_attribute_count)
         counter = 0
@@ -95,7 +99,7 @@ module Datadog
           # Conversion to symbol is needed here to put args ahead of
           # kwargs when they are merged below.
           c[:"arg#{counter}"] = value
-        end.update(kwargs).update(instance_vars)
+        end.update(kwargs).update(self: target_self)
         serialize_vars(combined, depth: depth, attribute_count: attribute_count)
       end
 

data/lib/datadog/di.rb

@@ -35,9 +35,3 @@ module Datadog
     end
   end
 end
-
-# Line probes will not work on Ruby < 2.6 because of lack of :script_compiled
-# trace point. Activate DI automatically on supported Ruby versions but
-# always load its settings so that, for example, turning DI off when
-# we are on Ruby 2.5 does not produce exceptions.
-require_relative 'di/boot' if RUBY_VERSION >= '2.6' && RUBY_ENGINE != 'jruby'