datadog 2.18.0 → 2.20.0

data/lib/datadog/core/configuration/option.rb

@@ -22,9 +22,18 @@ module Datadog
           # Represents an Option precedence level.
           # Each precedence has a `numeric` value; higher values means higher precedence.
           # `name` is for inspection purposes only.
-          Value = Struct.new(:numeric, :name, :origin) do
+
+          class Value
             include Comparable
 
+            attr_accessor :numeric, :name, :origin
+
+            def initialize(numeric, name, origin)
+              @numeric = numeric
+              @name = name
+              @origin = origin
+            end
+
             def <=>(other)
               return nil unless other.is_a?(Value)
 
@@ -172,20 +181,17 @@ module Datadog
         private
 
         def coerce_env_variable(value)
-          return context_exec(value, &@definition.env_parser) if @definition.env_parser
+          env_parser = @definition.env_parser
+          return context_exec(value, &env_parser) if env_parser
 
           case @definition.type
           when :hash
             values = value.split(',') # By default we only want to support comma separated strings
 
-            values.map! do |v|
+            values.each_with_object({}) do |v, hash| # $ Hash[String, String]
               v.gsub!(/\A[\s,]*|[\s,]*\Z/, '')
+              next if v.empty?
 
-              v.empty? ? nil : v
-            end
-
-            values.compact!
-            values.each.with_object({}) do |v, hash|
               pair = v.split(':', 2)
               hash[pair[0]] = pair[1]
             end
@@ -196,14 +202,12 @@ module Datadog
           when :array
             values = value.split(',')
 
-            values.map! do |v|
+            values.each_with_object([]) do |v, arr| # $ Array[String]
               v.gsub!(/\A[\s,]*|[\s,]*\Z/, '')
+              next if v.empty?
 
-              v.empty? ? nil : v
+              arr << v
             end
-
-            values.compact!
-            values
           when :bool
             string_value = value.strip
             string_value = string_value.downcase
@@ -329,18 +333,22 @@ module Datadog
           resolved_env = nil
 
           if definition.env
-            Array(definition.env).each do |env|
-              next if env_vars[env].nil?
+            # @type var env_and_aliases: Array[String]
+            env_and_aliases = Array(definition.env)
+            env_and_aliases.each do |env|
+              env_value = env_vars[env]
+              next if env_value.nil?
 
               resolved_env = env
-              value = coerce_env_variable(env_vars[env])
+              value = coerce_env_variable(env_value)
               break
             end
           end
 
-          if value.nil? && definition.deprecated_env && env_vars[definition.deprecated_env]
+          deprecated_env = definition.deprecated_env ? env_vars[definition.deprecated_env] : nil
+          if value.nil? && deprecated_env
             resolved_env = definition.deprecated_env
-            value = coerce_env_variable(env_vars[definition.deprecated_env])
+            value = coerce_env_variable(deprecated_env)
 
             Datadog::Core.log_deprecation do
               "#{definition.deprecated_env} #{source} is deprecated, use #{definition.env} instead."
@@ -349,9 +357,10 @@ module Datadog
 
           [value, resolved_env]
         rescue ArgumentError
+          env_value = resolved_env ? env_vars[resolved_env] : nil
           raise ArgumentError,
-            "Expected #{source} #{resolved_env} to be a #{@definition.type}, " \
-                              "but '#{env_vars[resolved_env]}' was provided"
+            "Expected #{source} #{resolved_env} to be a #{definition.type}, " \
+            "but '#{env_value}' was provided"
         end
 
         # Anchor object that represents a value that is not set.

data/lib/datadog/core/configuration/option_definition.rb

@@ -22,7 +22,7 @@ module Datadog
           :type,
           :type_options
 
-        def initialize(name, meta = {}, &block)
+        def initialize(name, meta, &block)
           @default = meta[:default]
           @default_proc = meta[:default_proc]
           @env = meta[:env]
@@ -44,7 +44,7 @@ module Datadog
         # Acts as DSL for building OptionDefinitions
         # @public_api
         class Builder
-          class InvalidOptionError < StandardError; end
+          InvalidOptionError = Class.new(StandardError)
 
           attr_reader \
             :helpers

data/lib/datadog/core/configuration/options.rb

@@ -40,14 +40,19 @@ module Datadog
 
           def default_helpers(name)
             option_name = name.to_sym
-
+            # @type var opt_getter: Configuration::OptionDefinition::helper_proc
+            opt_getter = proc do
+              # These Procs uses `get/set_option`, but we only add them to the OptionDefinition helpers here.
+              # Steep is right that these methods are not defined, but we only run these Procs in instance context.
+              get_option(option_name) # steep:ignore NoMethod
+            end
+            # @type var opt_setter: Configuration::OptionDefinition::helper_proc
+            opt_setter = proc do |value|
+              set_option(option_name, value) # steep:ignore NoMethod
+            end
             {
-              option_name.to_sym => proc do
-                get_option(option_name)
-              end,
-              :"#{option_name}=" => proc do |value|
-                set_option(option_name, value)
-              end
+              option_name.to_sym => opt_getter,
+              :"#{option_name}=" => opt_setter
             }
           end
 
@@ -113,6 +118,7 @@ module Datadog
 
             assert_valid_option!(name)
             definition = self.class.options[name]
+            # @type self: Configuration::Options::GenericSettingsClass
             options[name] = definition.build(self)
           end
 

data/lib/datadog/core/configuration/settings.rb

@@ -554,6 +554,26 @@ module Datadog
               o.env 'DD_PROFILING_NATIVE_FILENAMES_ENABLED'
               o.default true
             end
+
+            # Controls if the profiler should sample directly from the signal handler.
+            # Sampling directly from the signal handler improves accuracy of the data collected.
+            #
+            # We recommend using this setting with Ruby 3.2.5+ / Ruby 3.3.4+ and above
+            # as they include additional safety measures added in https://github.com/ruby/ruby/pull/11036.
+            # We have not validated it thoroughly with earlier versions, but in practice it should work on Ruby 3.0+
+            # (the key change was https://github.com/ruby/ruby/pull/3296).
+            #
+            # Enabling this on Ruby 2 is not recommended as it may cause VM crashes and/or incorrect data.
+            #
+            # @default true on Ruby 3.2.5+ / Ruby 3.3.4+, false on older Rubies
+            option :sighandler_sampling_enabled do |o|
+              o.type :bool
+              o.env 'DD_PROFILING_SIGHANDLER_SAMPLING_ENABLED'
+              o.default do
+                Gem::Version.new(RUBY_VERSION) >= Gem::Version.new('3.2.5') &&
+                  !(RUBY_VERSION.start_with?('3.3.') && Gem::Version.new(RUBY_VERSION) < Gem::Version.new('3.3.4'))
+              end
+            end
           end
 
           # @public_api

data/lib/datadog/core/telemetry/component.rb

@@ -62,7 +62,9 @@ module Datadog
           return unless @enabled
 
           @transport = if settings.telemetry.agentless_enabled
-            agent_settings = Core::Configuration::AgentlessSettingsResolver.call(
+            # We don't touch the `agent_settings` since we still want the telemetry payloads to refer to the original
+            # settings, even though the telemetry itself may be using a different path.
+            telemetry_specific_agent_settings = Core::Configuration::AgentlessSettingsResolver.call(
               settings,
               host_prefix: 'instrumentation-telemetry-intake',
               url_override: settings.telemetry.agentless_url_override,
@@ -70,7 +72,7 @@ module Datadog
               logger: logger,
             )
             Telemetry::Transport::HTTP.agentless_telemetry(
-              agent_settings: agent_settings,
+              agent_settings: telemetry_specific_agent_settings,
               logger: logger,
               # api_key should have already validated to be
               # not nil by +build+ method above.
@@ -96,6 +98,8 @@ module Datadog
             logger: logger,
             shutdown_timeout: settings.telemetry.shutdown_timeout_seconds,
           )
+
+          @agent_settings = agent_settings
         end
 
         def disable!
@@ -107,9 +111,9 @@ module Datadog
           return if !@enabled
 
           initial_event = if initial_event_is_change
-            Event::SynthAppClientConfigurationChange.new
+            Event::SynthAppClientConfigurationChange.new(agent_settings: @agent_settings)
           else
-            Event::AppStarted.new
+            Event::AppStarted.new(agent_settings: @agent_settings)
           end
 
           @worker.start(initial_event)

data/lib/datadog/core/telemetry/event/app_started.rb

@@ -8,6 +8,10 @@ module Datadog
       module Event
         # Telemetry class for the 'app-started' event
         class AppStarted < Base
+          def initialize(agent_settings:)
+            @agent_settings = agent_settings
+          end
+
           def type
             'app-started'
           end
@@ -96,7 +100,7 @@ module Datadog
               ),
 
               # Mix of env var, programmatic and default config, so we use unknown
-              conf_value('DD_AGENT_TRANSPORT', agent_transport(config), seq_id, 'unknown'),
+              conf_value('DD_AGENT_TRANSPORT', agent_transport, seq_id, 'unknown'),
 
               # writer_options is defined as an option that has a Hash value.
               conf_value(
@@ -162,6 +166,20 @@ module Datadog
               )
             end
 
+            instrumentation_source = if Datadog.const_defined?(:SingleStepInstrument, false) &&
+                Datadog::SingleStepInstrument.const_defined?(:LOADED, false) &&
+                Datadog::SingleStepInstrument::LOADED
+              'ssi'
+            else
+              'manual'
+            end
+            # Track ssi configurations
+            list.push(
+              conf_value('instrumentation_source', instrumentation_source, seq_id, 'code'),
+              conf_value('DD_INJECT_FORCE', Core::Environment::VariableHelpers.env_to_bool('DD_INJECT_FORCE', false), seq_id, 'env_var'),
+              conf_value('DD_INJECTION_ENABLED', ENV['DD_INJECTION_ENABLED'] || '', seq_id, 'env_var'),
+            )
+
             # Add some more custom additional payload values here
             if config.logger.instance
               list << conf_value(
@@ -200,8 +218,8 @@ module Datadog
           # standard:enable Metrics/AbcSize
           # standard:enable Metrics/MethodLength
 
-          def agent_transport(config)
-            adapter = Core::Configuration::AgentSettingsResolver.call(config).adapter
+          def agent_transport
+            adapter = @agent_settings.adapter
             if adapter == Datadog::Core::Transport::Ext::UnixSocket::ADAPTER
               'UDS'
             else

data/lib/datadog/di/boot.rb

@@ -32,3 +32,10 @@ end
 require_relative 'contrib'
 
 Datadog::DI::Contrib.load_now_or_later
+
+if %w[1 true yes].include?(ENV['DD_DYNAMIC_INSTRUMENTATION_ENABLED']) # steep:ignore
+  if ENV['DD_DYNAMIC_INSTRUMENTATION_PROBE_FILE']
+    require_relative 'probe_file_loader'
+    Datadog::DI::ProbeFileLoader.load_now_or_later
+  end
+end

data/lib/datadog/di/component.rb

@@ -112,6 +112,13 @@ module Datadog
         probe_manager.close
         probe_notifier_worker.stop
       end
+
+      def parse_probe_spec_and_notify(probe_spec)
+        probe = ProbeBuilder.build_from_remote_config(probe_spec)
+        payload = probe_notification_builder.build_received(probe)
+        probe_notifier_worker.add_status(payload)
+        probe
+      end
     end
   end
 end

data/lib/datadog/di/instrumenter.rb

@@ -121,9 +121,8 @@ module Datadog
             if rate_limiter.nil? || rate_limiter.allow?
               # Arguments may be mutated by the method, therefore
               # they need to be serialized prior to method invocation.
-              entry_args = if probe.capture_snapshot?
-                instance_vars = Instrumenter.get_instance_variables(self)
-                serializer.serialize_args(args, kwargs, instance_vars,
+              serialized_entry_args = if probe.capture_snapshot?
+                serializer.serialize_args(args, kwargs, self,
                   depth: probe.max_capture_depth || settings.dynamic_instrumentation.max_capture_depth,
                   attribute_count: probe.max_capture_attribute_count || settings.dynamic_instrumentation.max_capture_attribute_count)
               end
@@ -160,9 +159,10 @@ module Datadog
               caller_locs = method_frame + caller_locations # steep:ignore
               # TODO capture arguments at exit
               # & is to stop steep complaints, block is always present here.
-              block&.call(probe: probe, rv: rv, duration: duration, caller_locations: caller_locs,
-                instance_vars: probe.capture_snapshot? ? Instrumenter.get_instance_variables(self) : nil,
-                serialized_entry_args: entry_args)
+              block&.call(probe: probe, rv: rv,
+                duration: duration, caller_locations: caller_locs,
+                target_self: self,
+                serialized_entry_args: serialized_entry_args)
               rv
             else
               # stop standard from trying to mess up my code
@@ -311,19 +311,20 @@ module Datadog
               probe.file == tp.path || probe.file_matches?(tp.path)
             )
               if rate_limiter.nil? || rate_limiter.allow?
-                locals = if probe.capture_snapshot?
+                serialized_locals = if probe.capture_snapshot?
                   serializer.serialize_vars(Instrumenter.get_local_variables(tp),
                     depth: probe.max_capture_depth || settings.dynamic_instrumentation.max_capture_depth,
                     attribute_count: probe.max_capture_attribute_count || settings.dynamic_instrumentation.max_capture_attribute_count,)
                 end
-                instance_vars = if probe.capture_snapshot?
-                  serializer.serialize_vars(Instrumenter.get_instance_variables(tp.self),
+                if probe.capture_snapshot?
+                  serializer.serialize_value(tp.self,
                     depth: probe.max_capture_depth || settings.dynamic_instrumentation.max_capture_depth,
                     attribute_count: probe.max_capture_attribute_count || settings.dynamic_instrumentation.max_capture_attribute_count,)
                 end
                 # & is to stop steep complaints, block is always present here.
                 block&.call(probe: probe,
-                  locals: locals, instance_vars: instance_vars,
+                  serialized_locals: serialized_locals,
+                  target_self: tp.self,
                   path: tp.path, caller_locations: caller_locations)
               end
             end
@@ -397,14 +398,6 @@ module Datadog
       end
 
       class << self
-        def get_instance_variables(object)
-          {}.tap do |hash|
-            object.instance_variables.each do |var|
-              hash[var] = object.instance_variable_get(var)
-            end
-          end
-        end
-
         def get_local_variables(trace_point)
           # binding appears to be constructed on access, therefore
           # 1) we should attempt to cache it and

data/lib/datadog/di/probe_file_loader/railtie.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Datadog
+  module DI
+    module ProbeFileLoader
+      # Railtie class initializes dynamic instrumentation contrib code
+      # in Rails environments.
+      class Railtie < Rails::Railtie
+        initializer 'datadog.dynamic_instrumentation.load_probe_file' do |app| # steep:ignore
+          ProbeFileLoader.load_now
+        end
+      end
+    end
+  end
+end

data/lib/datadog/di/probe_file_loader.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module Datadog
+  module DI
+    module ProbeFileLoader
+      module_function def load_now_or_later
+        if Datadog::Core::Contrib::Rails::Utils.railtie_supported?
+          Datadog.logger.debug('di: loading probe_file_loader/railtie')
+          require_relative 'probe_file_loader/railtie'
+        else
+          load_now
+        end
+      end
+
+      # This method can be called more than once, to attempt to load
+      # DI components that depend on third-party libraries after additional
+      # dependencies are loaded (or potentially loaded).
+      module_function def load_now
+        should_propagate = false
+
+        probe_file_path = ENV['DD_DYNAMIC_INSTRUMENTATION_PROBE_FILE']
+        if probe_file_path.nil? || probe_file_path.empty?
+          return
+        end
+
+        # We want to initialize the component tree here if it was not already
+        # initialized.
+        component = Datadog::DI.component
+        return unless component
+
+        begin
+          probe_specs = File.open(probe_file_path) do |f|
+            # The probe file should contain an array, JSON.parse does not work
+            JSON.load(f) # standard:disable Security/JSONLoad
+          end
+
+          probe_specs.each do |probe_spec|
+            probe = component.parse_probe_spec_and_notify(probe_spec)
+            component.logger.debug { "di: received #{probe.type} probe at #{probe.location} (#{probe.id}) via probe file: #{probe_file_path}" }
+
+            begin
+              component.probe_manager.add_probe(probe)
+            rescue DI::Error::DITargetNotInRegistry => exc
+              component.telemetry&.report(exc, description: "Line probe is targeting a loaded file that is not in code tracker")
+
+              payload = component.probe_notification_builder.build_errored(probe, exc)
+              component.probe_notifier_worker.add_status(payload)
+            rescue => exc
+              raise if component.settings.dynamic_instrumentation.internal.propagate_all_exceptions
+
+              component.logger.debug { "di: unhandled exception adding #{probe.type} probe at #{probe.location} (#{probe.id}) in DI probe file loader: #{exc.class}: #{exc}" }
+              component.telemetry&.report(exc, description: "Unhandled exception adding probe in DI probe file loader")
+
+              # TODO test this path
+              payload = component.probe_notification_builder.build_errored(probe, exc)
+              component.probe_notifier_worker.add_status(payload)
+            end
+          end
+        rescue => exc
+          if component.settings.dynamic_instrumentation.internal.propagate_all_exceptions
+            should_propagate = true
+            raise
+          end
+
+          component.logger.debug { "di: unhandled exception handling a probe in DI probe file loader: #{exc.class}: #{exc}" }
+          component.telemetry&.report(exc, description: "Unhandled exception handling probe in DI probe file loader")
+        end
+      rescue
+        # We should generally never get here, but if component tree
+        # initialization fails for some unexpected reason, don't nuke
+        # the customer application.
+        #
+        # For the same reason, we do not check
+        # component.settings.dynamic_instrumentation.internal.propagate_all_exceptions
+        # here again, but rely on the local variable storing that value.
+        raise if should_propagate
+      end
+    end
+  end
+end

data/lib/datadog/di/probe_notification_builder.rb

@@ -42,9 +42,9 @@ module Datadog
       # path is the actual path of the instrumented file.
       def build_executed(probe,
         path: nil, rv: nil, duration: nil, caller_locations: nil,
-        locals: nil, args: nil, kwargs: nil, instance_vars: nil,
+        serialized_locals: nil, args: nil, kwargs: nil, target_self: nil,
         serialized_entry_args: nil)
-        build_snapshot(probe, rv: rv, locals: locals,
+        build_snapshot(probe, rv: rv, serialized_locals: serialized_locals,
           # Actual path of the instrumented file.
           path: path,
           duration: duration,
@@ -52,14 +52,23 @@ module Datadog
           # this should be all frames for enriched probes and no frames for
           # non-enriched probes?
           caller_locations: caller_locations,
-          args: args, kwargs: kwargs, instance_vars: instance_vars,
+          args: args, kwargs: kwargs,
+          target_self: target_self,
           serialized_entry_args: serialized_entry_args)
       end
 
-      def build_snapshot(probe, rv: nil, locals: nil, path: nil,
+      def build_snapshot(probe, rv: nil, serialized_locals: nil, path: nil,
+        # In Ruby everything is a method, therefore we should always have
+        # a target self. However, if we are not capturing a snapshot,
+        # there is no need to pass in the target self.
+        target_self: nil,
         duration: nil, caller_locations: nil,
-        args: nil, kwargs: nil, instance_vars: nil,
+        args: nil, kwargs: nil,
         serialized_entry_args: nil)
+        if probe.capture_snapshot? && !target_self
+          raise ArgumentError, "Asked to build snapshot with snapshot capture but target_self is nil"
+        end
+
         # TODO also verify that non-capturing probe does not pass
         # snapshot or vars/args into this method
         captures = if probe.capture_snapshot?
@@ -68,25 +77,18 @@ module Datadog
               "@return": serializer.serialize_value(rv,
                 depth: probe.max_capture_depth || settings.dynamic_instrumentation.max_capture_depth,
                 attribute_count: probe.max_capture_attribute_count || settings.dynamic_instrumentation.max_capture_attribute_count),
+              self: serializer.serialize_value(target_self),
             }
-            if instance_vars
-              return_arguments.update(
-                serializer.serialize_vars(instance_vars,
-                  depth: probe.max_capture_depth || settings.dynamic_instrumentation.max_capture_depth,
-                  attribute_count: probe.max_capture_attribute_count || settings.dynamic_instrumentation.max_capture_attribute_count,)
-              )
-            end
             {
               entry: {
                 # standard:disable all
                 arguments: if serialized_entry_args
                   serialized_entry_args
                 else
-                  (args || kwargs) && serializer.serialize_args(args, kwargs, instance_vars,
+                  (args || kwargs) && serializer.serialize_args(args, kwargs, target_self,
                     depth: probe.max_capture_depth || settings.dynamic_instrumentation.max_capture_depth,
                     attribute_count: probe.max_capture_attribute_count || settings.dynamic_instrumentation.max_capture_attribute_count)
                 end,
-                throwable: nil,
                 # standard:enable all
               },
               return: {
@@ -96,8 +98,11 @@ module Datadog
             }
           elsif probe.line?
             {
-              lines: locals && {
-                probe.line_no => {locals: locals.merge(instance_vars || {})},
+              lines: serialized_locals && {
+                probe.line_no => {
+                  locals: serialized_locals,
+                  arguments: {self: serializer.serialize_value(target_self)},
+                },
               },
             }
           end

data/lib/datadog/di/remote.rb

@@ -43,6 +43,8 @@ module Datadog
             if component
 
               probe_manager = component.probe_manager
+              probe_notification_builder = component.probe_notification_builder
+              probe_notifier_worker = component.probe_notifier_worker
 
               current_probe_ids = {}
               repository.contents.each do |content|
@@ -50,11 +52,7 @@ module Datadog
                 when PRODUCT
                   begin
                     probe_spec = parse_content(content)
-                    probe = ProbeBuilder.build_from_remote_config(probe_spec)
-                    probe_notification_builder = component.probe_notification_builder
-                    payload = probe_notification_builder.build_received(probe)
-                    probe_notifier_worker = component.probe_notifier_worker
-                    probe_notifier_worker.add_status(payload)
+                    probe = component.parse_probe_spec_and_notify(probe_spec)
                     component.logger.debug { "di: received #{probe.type} probe at #{probe.location} (#{probe.id}) via RC" }
 
                     begin

data/lib/datadog/di/serializer.rb

@@ -36,6 +36,10 @@ module Datadog
     # efficient but there would be additional overhead from passing this
     # parameter all the time and the API would get more complex.
     #
+    # Note: "self" cannot be used as a parameter name in Ruby, therefore
+    # there should never be a conflict between instance variable
+    # serialization and method parameters.
+    #
     # @api private
     class Serializer
       # Third-party library integration / custom serializers.
@@ -86,7 +90,7 @@ module Datadog
       # Instance variables are technically a hash just like kwargs,
       # we take them as a separate parameter to avoid a hash merge
       # in upstream code.
-      def serialize_args(args, kwargs, instance_vars,
+      def serialize_args(args, kwargs, target_self,
         depth: settings.dynamic_instrumentation.max_capture_depth,
         attribute_count: settings.dynamic_instrumentation.max_capture_attribute_count)
         counter = 0
@@ -95,7 +99,7 @@ module Datadog
           # Conversion to symbol is needed here to put args ahead of
           # kwargs when they are merged below.
           c[:"arg#{counter}"] = value
-        end.update(kwargs).update(instance_vars)
+        end.update(kwargs).update(self: target_self)
         serialize_vars(combined, depth: depth, attribute_count: attribute_count)
       end
 

data/lib/datadog/di.rb

@@ -19,7 +19,6 @@ module Datadog
     Extensions.activate!
 
     class << self
-
       # This method is called from DI Remote handler to issue DI operations
       # to the probe manager (add or remove probes).
       #
@@ -35,9 +34,3 @@ module Datadog
     end
   end
 end
-
-# Line probes will not work on Ruby < 2.6 because of lack of :script_compiled
-# trace point. Activate DI automatically on supported Ruby versions but
-# always load its settings so that, for example, turning DI off when
-# we are on Ruby 2.5 does not produce exceptions.
-require_relative 'di/boot' if RUBY_VERSION >= '2.6' && RUBY_ENGINE != 'jruby'