datadog 2.12.1 → 2.19.0

data/lib/datadog/di/instrumenter.rb

@@ -52,6 +52,17 @@ module Datadog
     # (however, Probe instances can be replaced by OpenStruct instances
     # providing the same interface with not much effort).
     #
+    # Instrumenter (this class) is responsible for building snapshots.
+    # This is because to capture values on method entry, those values need to
+    # be duplicated or serialized into immutable values to prevent their
+    # modification by the instrumented method. Therefore this class must
+    # do at least some serialization/snapshot building and to keep the code
+    # well-encapsulated, all serialization/snapshot building should thus be
+    # initiated from this class rather than downstream code.
+    #
+    # As a consequence of Instrumenter building snapshots, it should not
+    # expose TracePoint objects to any downstream code.
+    #
     # @api private
     class Instrumenter
       def initialize(settings, serializer, logger, code_tracker: nil, telemetry: nil)
@@ -110,8 +121,8 @@ module Datadog
             if rate_limiter.nil? || rate_limiter.allow?
               # Arguments may be mutated by the method, therefore
               # they need to be serialized prior to method invocation.
-              entry_args = if probe.capture_snapshot?
-                serializer.serialize_args(args, kwargs,
+              serialized_entry_args = if probe.capture_snapshot?
+                serializer.serialize_args(args, kwargs, self,
                   depth: probe.max_capture_depth || settings.dynamic_instrumentation.max_capture_depth,
                   attribute_count: probe.max_capture_attribute_count || settings.dynamic_instrumentation.max_capture_attribute_count)
               end
@@ -148,8 +159,10 @@ module Datadog
               caller_locs = method_frame + caller_locations # steep:ignore
               # TODO capture arguments at exit
               # & is to stop steep complaints, block is always present here.
-              block&.call(probe: probe, rv: rv, duration: duration, caller_locations: caller_locs,
-                serialized_entry_args: entry_args)
+              block&.call(probe: probe, rv: rv,
+                duration: duration, caller_locations: caller_locs,
+                target_self: self,
+                serialized_entry_args: serialized_entry_args)
               rv
             else
               # stop standard from trying to mess up my code
@@ -298,8 +311,21 @@ module Datadog
               probe.file == tp.path || probe.file_matches?(tp.path)
             )
               if rate_limiter.nil? || rate_limiter.allow?
+                serialized_locals = if probe.capture_snapshot?
+                  serializer.serialize_vars(Instrumenter.get_local_variables(tp),
+                    depth: probe.max_capture_depth || settings.dynamic_instrumentation.max_capture_depth,
+                    attribute_count: probe.max_capture_attribute_count || settings.dynamic_instrumentation.max_capture_attribute_count,)
+                end
+                if probe.capture_snapshot?
+                  serializer.serialize_value(tp.self,
+                    depth: probe.max_capture_depth || settings.dynamic_instrumentation.max_capture_depth,
+                    attribute_count: probe.max_capture_attribute_count || settings.dynamic_instrumentation.max_capture_attribute_count,)
+                end
                 # & is to stop steep complaints, block is always present here.
-                block&.call(probe: probe, trace_point: tp, caller_locations: caller_locations)
+                block&.call(probe: probe,
+                  serialized_locals: serialized_locals,
+                  target_self: tp.self,
+                  path: tp.path, caller_locations: caller_locations)
               end
             end
           rescue => exc
@@ -371,6 +397,23 @@ module Datadog
         end
       end
 
+      class << self
+        def get_local_variables(trace_point)
+          # binding appears to be constructed on access, therefore
+          # 1) we should attempt to cache it and
+          # 2) we should not call +binding+ until we actually need variable values.
+          binding = trace_point.binding
+
+          # steep hack - should never happen
+          return {} unless binding
+
+          binding.local_variables.each_with_object({}) do |name, map|
+            value = binding.local_variable_get(name)
+            map[name] = value
+          end
+        end
+      end
+
       private
 
       attr_reader :lock

data/lib/datadog/di/probe_notification_builder.rb

@@ -39,60 +39,70 @@ module Datadog
       end
 
       # Duration is in seconds.
+      # path is the actual path of the instrumented file.
       def build_executed(probe,
-        trace_point: nil, rv: nil, duration: nil, caller_locations: nil,
-        args: nil, kwargs: nil, serialized_entry_args: nil)
-        snapshot = if probe.line? && probe.capture_snapshot?
-          if trace_point.nil?
-            raise "Cannot create snapshot because there is no trace point"
-          end
-          get_local_variables(trace_point)
-        end
-        # TODO check how many stack frames we should be keeping/sending,
-        # this should be all frames for enriched probes and no frames for
-        # non-enriched probes?
-        build_snapshot(probe, rv: rv, snapshot: snapshot,
+        path: nil, rv: nil, duration: nil, caller_locations: nil,
+        serialized_locals: nil, args: nil, kwargs: nil, target_self: nil,
+        serialized_entry_args: nil)
+        build_snapshot(probe, rv: rv, serialized_locals: serialized_locals,
           # Actual path of the instrumented file.
-          path: trace_point&.path,
-          duration: duration, caller_locations: caller_locations, args: args, kwargs: kwargs,
+          path: path,
+          duration: duration,
+          # TODO check how many stack frames we should be keeping/sending,
+          # this should be all frames for enriched probes and no frames for
+          # non-enriched probes?
+          caller_locations: caller_locations,
+          args: args, kwargs: kwargs,
+          target_self: target_self,
           serialized_entry_args: serialized_entry_args)
       end
 
-      def build_snapshot(probe, rv: nil, snapshot: nil, path: nil,
-        duration: nil, caller_locations: nil, args: nil, kwargs: nil,
+      def build_snapshot(probe, rv: nil, serialized_locals: nil, path: nil,
+        # In Ruby everything is a method, therefore we should always have
+        # a target self. However, if we are not capturing a snapshot,
+        # there is no need to pass in the target self.
+        target_self: nil,
+        duration: nil, caller_locations: nil,
+        args: nil, kwargs: nil,
         serialized_entry_args: nil)
+        if probe.capture_snapshot? && !target_self
+          raise ArgumentError, "Asked to build snapshot with snapshot capture but target_self is nil"
+        end
+
         # TODO also verify that non-capturing probe does not pass
         # snapshot or vars/args into this method
         captures = if probe.capture_snapshot?
           if probe.method?
+            return_arguments = {
+              "@return": serializer.serialize_value(rv,
+                depth: probe.max_capture_depth || settings.dynamic_instrumentation.max_capture_depth,
+                attribute_count: probe.max_capture_attribute_count || settings.dynamic_instrumentation.max_capture_attribute_count),
+              self: serializer.serialize_value(target_self),
+            }
             {
               entry: {
                 # standard:disable all
                 arguments: if serialized_entry_args
                   serialized_entry_args
                 else
-                  (args || kwargs) && serializer.serialize_args(args, kwargs,
+                  (args || kwargs) && serializer.serialize_args(args, kwargs, target_self,
                     depth: probe.max_capture_depth || settings.dynamic_instrumentation.max_capture_depth,
                     attribute_count: probe.max_capture_attribute_count || settings.dynamic_instrumentation.max_capture_attribute_count)
                 end,
-                throwable: nil,
                 # standard:enable all
               },
               return: {
-                arguments: {
-                  "@return": serializer.serialize_value(rv,
-                    depth: probe.max_capture_depth || settings.dynamic_instrumentation.max_capture_depth,
-                    attribute_count: probe.max_capture_attribute_count || settings.dynamic_instrumentation.max_capture_attribute_count),
-                },
+                arguments: return_arguments,
                 throwable: nil,
               },
             }
           elsif probe.line?
             {
-              lines: snapshot && {
-                probe.line_no => {locals: serializer.serialize_vars(snapshot,
-                  depth: probe.max_capture_depth || settings.dynamic_instrumentation.max_capture_depth,
-                  attribute_count: probe.max_capture_attribute_count || settings.dynamic_instrumentation.max_capture_attribute_count,)},
+              lines: serialized_locals && {
+                probe.line_no => {
+                  locals: serialized_locals,
+                  arguments: {self: serializer.serialize_value(target_self)},
+                },
               },
             }
           end
@@ -191,22 +201,7 @@ module Datadog
       end
 
       def timestamp_now
-        (Time.now.to_f * 1000).to_i
-      end
-
-      def get_local_variables(trace_point)
-        # binding appears to be constructed on access, therefore
-        # 1) we should attempt to cache it and
-        # 2) we should not call +binding+ until we actually need variable values.
-        binding = trace_point.binding
-
-        # steep hack - should never happen
-        return {} unless binding
-
-        binding.local_variables.each_with_object({}) do |name, map|
-          value = binding.local_variable_get(name)
-          map[name] = value
-        end
+        (Core::Utils::Time.now.to_f * 1000).to_i
       end
 
       def active_trace

data/lib/datadog/di/probe_notifier_worker.rb

@@ -171,7 +171,7 @@ module Datadog
       attr_reader :last_sent
 
       def status_transport
-        @status_transport ||= DI::Transport::HTTP.diagnostics(agent_settings: agent_settings)
+        @status_transport ||= DI::Transport::HTTP.diagnostics(agent_settings: agent_settings, logger: logger)
       end
 
       def do_send_status(batch)
@@ -179,11 +179,19 @@ module Datadog
       end
 
       def snapshot_transport
-        @snapshot_transport ||= DI::Transport::HTTP.input(agent_settings: agent_settings)
+        @snapshot_transport ||= DI::Transport::HTTP.input(agent_settings: agent_settings, logger: logger)
       end
 
       def do_send_snapshot(batch)
-        snapshot_transport.send_input(batch)
+        snapshot_transport.send_input(batch, tags)
+      end
+
+      def tags
+        # DEV: The tags could be cached but they need to be recreated
+        # when process forks (since the child receives new runtime IDs).
+        Core::TagBuilder.tags(settings).merge(
+          'debugger_version' => Core::Environment::Identity.gem_datadog_version,
+        )
       end
 
       [
@@ -227,20 +235,6 @@ module Datadog
           start
         end
 
-        # Determine how much longer the worker thread should sleep
-        # so as not to send in less than min send interval since the last send.
-        # Important: this method must be called when @lock is held.
-        #
-        # Returns the time remaining to sleep.
-        def set_sleep_remaining
-          now = Core::Utils::Time.get_time
-          @sleep_remaining = if last_sent
-            [last_sent + min_send_interval - now, 0].max
-          else
-            0
-          end
-        end
-
         public "add_#{event_type}"
 
         # Sends pending probe statuses or snapshots.
@@ -288,6 +282,20 @@ module Datadog
         end
       end
 
+      # Determine how much longer the worker thread should sleep
+      # so as not to send in less than min send interval since the last send.
+      # Important: this method must be called when @lock is held.
+      #
+      # Returns the time remaining to sleep.
+      def set_sleep_remaining
+        now = Core::Utils::Time.get_time
+        @sleep_remaining = if last_sent
+          [last_sent + min_send_interval - now, 0].max
+        else
+          0
+        end
+      end
+
       def maybe_send
         rv = maybe_send_status
         maybe_send_snapshot || rv

data/lib/datadog/di/remote.rb

@@ -18,6 +18,8 @@ module Datadog
         PRODUCT = 'LIVE_DEBUGGING'
 
         def products
+          # TODO: do not send our product on unsupported runtimes
+          # (Ruby 2.5 / JRuby)
           [PRODUCT]
         end
 

data/lib/datadog/di/serializer.rb

@@ -36,6 +36,10 @@ module Datadog
     # efficient but there would be additional overhead from passing this
     # parameter all the time and the API would get more complex.
     #
+    # Note: "self" cannot be used as a parameter name in Ruby, therefore
+    # there should never be a conflict between instance variable
+    # serialization and method parameters.
+    #
     # @api private
     class Serializer
       # Third-party library integration / custom serializers.
@@ -82,7 +86,11 @@ module Datadog
       # between positional and keyword arguments. We convert positional
       # arguments to keyword arguments ("arg1", "arg2", ...) and ensure
       # the positional arguments are listed first.
-      def serialize_args(args, kwargs,
+      #
+      # Instance variables are technically a hash just like kwargs,
+      # we take them as a separate parameter to avoid a hash merge
+      # in upstream code.
+      def serialize_args(args, kwargs, target_self,
         depth: settings.dynamic_instrumentation.max_capture_depth,
         attribute_count: settings.dynamic_instrumentation.max_capture_attribute_count)
         counter = 0
@@ -91,7 +99,7 @@ module Datadog
           # Conversion to symbol is needed here to put args ahead of
           # kwargs when they are merged below.
           c[:"arg#{counter}"] = value
-        end.update(kwargs)
+        end.update(kwargs).update(self: target_self)
         serialize_vars(combined, depth: depth, attribute_count: attribute_count)
       end
 

data/lib/datadog/di/transport/diagnostics.rb

@@ -15,12 +15,13 @@ module Datadog
         end
 
         class Transport
-          attr_reader :client, :apis, :default_api, :current_api_id
+          attr_reader :client, :apis, :default_api, :current_api_id, :logger
 
-          def initialize(apis, default_api)
+          def initialize(apis, default_api, logger:)
             @apis = apis
+            @logger = logger
 
-            @client = HTTP::Client.new(current_api)
+            @client = HTTP::Client.new(current_api, logger: logger)
           end
 
           def current_api

data/lib/datadog/di/transport/http/api.rb

@@ -21,13 +21,13 @@ module Datadog
 
           def defaults
             Datadog::Core::Transport::HTTP::API::Map[
-              DIAGNOSTICS => Spec.new do |s|
+              DIAGNOSTICS => Diagnostics::API::Spec.new do |s|
                 s.diagnostics = Diagnostics::API::Endpoint.new(
                   '/debugger/v1/diagnostics',
                   Core::Encoding::JSONEncoder,
                 )
               end,
-              INPUT => Spec.new do |s|
+              INPUT => Input::API::Spec.new do |s|
                 s.input = Input::API::Endpoint.new(
                   '/debugger/v1/input',
                   Core::Encoding::JSONEncoder,
@@ -35,16 +35,6 @@ module Datadog
               end,
             ]
           end
-
-          class Instance < Core::Transport::HTTP::API::Instance
-            include Diagnostics::API::Instance
-            include Input::API::Instance
-          end
-
-          class Spec < Core::Transport::HTTP::API::Spec
-            include Diagnostics::API::Spec
-            include Input::API::Spec
-          end
         end
       end
     end

data/lib/datadog/di/transport/http/client.rb

@@ -14,10 +14,11 @@ module Datadog
       module HTTP
         # Routes, encodes, and sends DI data to the trace agent via HTTP.
         class Client
-          attr_reader :api
+          attr_reader :api, :logger
 
-          def initialize(api)
+          def initialize(api, logger:)
             @api = api
+            @logger = logger
           end
 
           def send_request(request, &block)
@@ -31,7 +32,7 @@ module Datadog
               "Internal error during #{self.class.name} request. Cause: #{e.class.name} #{e.message} " \
                 "Location: #{Array(e.backtrace).first}"
 
-            Datadog.logger.debug(message)
+            logger.debug(message)
 
             Datadog::Core::Transport::InternalErrorResponse.new(e)
           end

data/lib/datadog/di/transport/http/diagnostics.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require_relative '../../../core/transport/http/api/instance'
+require_relative '../../../core/transport/http/api/spec'
 require_relative 'client'
 
 module Datadog
@@ -16,55 +18,26 @@ module Datadog
           end
 
           module API
-            module Instance
+            class Instance < Core::Transport::HTTP::API::Instance
               def send_diagnostics(env)
-                raise DiagnosticsNotSupportedError, spec unless spec.is_a?(Diagnostics::API::Spec)
+                raise Core::Transport::HTTP::API::Instance::EndpointNotSupportedError.new('diagnostics', self) unless spec.is_a?(Diagnostics::API::Spec)
 
                 spec.send_diagnostics(env) do |request_env|
                   call(request_env)
                 end
               end
-
-              class DiagnosticsNotSupportedError < StandardError
-                attr_reader :spec
-
-                def initialize(spec)
-                  super
-
-                  @spec = spec
-                end
-
-                def message
-                  'Diagnostics not supported for this API!'
-                end
-              end
             end
 
-            module Spec
+            class Spec < Core::Transport::HTTP::API::Spec
               attr_accessor :diagnostics
 
               def send_diagnostics(env, &block)
-                raise NoDiagnosticsEndpointDefinedError, self if diagnostics.nil?
+                raise Core::Transport::HTTP::API::Spec::EndpointNotDefinedError.new('diagnostics', self) if diagnostics.nil?
 
                 diagnostics.call(env, &block)
               end
-
-              class NoDiagnosticsEndpointDefinedError < StandardError
-                attr_reader :spec
-
-                def initialize(spec)
-                  super
-
-                  @spec = spec
-                end
-
-                def message
-                  'No diagnostics endpoint is defined for API specification!'
-                end
-              end
             end
 
-            # Endpoint for negotiation
             class Endpoint < Datadog::Core::Transport::HTTP::API::Endpoint
               attr_reader :encoder
 
@@ -79,7 +52,7 @@ module Datadog
                 )
                 env.form = {'event' => event_payload}
 
-                super(env, &block)
+                super
               end
             end
           end

data/lib/datadog/di/transport/http/input.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require_relative '../../../core/transport/http/api/instance'
+require_relative '../../../core/transport/http/api/spec'
 require_relative 'client'
 
 module Datadog
@@ -16,55 +18,26 @@ module Datadog
           end
 
           module API
-            module Instance
+            class Instance < Core::Transport::HTTP::API::Instance
               def send_input(env)
-                raise InputNotSupportedError, spec unless spec.is_a?(Input::API::Spec)
+                raise Core::Transport::HTTP::API::Instance::EndpointNotSupportedError.new('input', self) unless spec.is_a?(Input::API::Spec)
 
                 spec.send_input(env) do |request_env|
                   call(request_env)
                 end
               end
-
-              class InputNotSupportedError < StandardError
-                attr_reader :spec
-
-                def initialize(spec)
-                  super
-
-                  @spec = spec
-                end
-
-                def message
-                  'Input not supported for this API!'
-                end
-              end
             end
 
-            module Spec
+            class Spec < Core::Transport::HTTP::API::Spec
               attr_accessor :input
 
               def send_input(env, &block)
-                raise NoInputEndpointDefinedError, self if input.nil?
+                raise Core::Transport::HTTP::API::Spec::EndpointNotDefinedError.new('input', self) if input.nil?
 
                 input.call(env, &block)
               end
-
-              class NoInputEndpointDefinedError < StandardError
-                attr_reader :spec
-
-                def initialize(spec)
-                  super
-
-                  @spec = spec
-                end
-
-                def message
-                  'No input endpoint is defined for API specification!'
-                end
-              end
             end
 
-            # Endpoint for negotiation
             class Endpoint < Datadog::Core::Transport::HTTP::API::Endpoint
               HEADER_CONTENT_TYPE = 'Content-Type'
 
@@ -80,8 +53,18 @@ module Datadog
                 # Encode body & type
                 env.headers[HEADER_CONTENT_TYPE] = encoder.content_type
                 env.body = env.request.parcel.data
-
-                super(env, &block)
+                env.query = {
+                  # DEV: In theory we could serialize the tags here
+                  # rather than requiring them to be pre-serialized.
+                  # In practice the tags should be relatively static
+                  # (they would change when process forks, and hostname
+                  # could change at any time but probably we should ignore
+                  # those changes), therefore serializing the tags
+                  # every time would be wasteful.
+                  ddtags: env.request.serialized_tags,
+                }
+
+                super
               end
             end
           end

data/lib/datadog/di/transport/http.rb

@@ -1,15 +1,9 @@
 # frozen_string_literal: true
 
-require 'uri'
-
-require_relative '../../core/environment/container'
-require_relative '../../core/environment/ext'
-require_relative '../../core/transport/ext'
 require_relative 'diagnostics'
 require_relative 'input'
 require_relative 'http/api'
 require_relative '../../core/transport/http'
-require_relative '../../../datadog/version'
 
 module Datadog
   module DI
@@ -18,86 +12,44 @@ module Datadog
       module HTTP
         module_function
 
-        # Builds a new Transport::HTTP::Client
-        def new(klass, &block)
-          Core::Transport::HTTP.build(
-            api_instance_class: API::Instance, &block
-          ).to_transport(klass)
-        end
-
         # Builds a new Transport::HTTP::Client with default settings
         # Pass a block to override any settings.
         def diagnostics(
           agent_settings:,
-          **options
+          logger:,
+          api_version: nil,
+          headers: nil
         )
-          new(DI::Transport::Diagnostics::Transport) do |transport|
-            transport.adapter(agent_settings)
-            transport.headers default_headers
-
+          Core::Transport::HTTP.build(api_instance_class: Diagnostics::API::Instance,
+            logger: logger,
+            agent_settings: agent_settings, api_version: api_version, headers: headers) do |transport|
             apis = API.defaults
 
             transport.api API::DIAGNOSTICS, apis[API::DIAGNOSTICS]
 
-            # Apply any settings given by options
-            unless options.empty?
-              transport.default_api = options[:api_version] if options.key?(:api_version)
-              transport.headers options[:headers] if options.key?(:headers)
-            end
-
             # Call block to apply any customization, if provided
             yield(transport) if block_given?
-          end
+          end.to_transport(DI::Transport::Diagnostics::Transport)
         end
 
         # Builds a new Transport::HTTP::Client with default settings
         # Pass a block to override any settings.
         def input(
           agent_settings:,
-          **options
+          logger:,
+          api_version: nil,
+          headers: nil
         )
-          new(DI::Transport::Input::Transport) do |transport|
-            transport.adapter(agent_settings)
-            transport.headers default_headers
-
+          Core::Transport::HTTP.build(api_instance_class: Input::API::Instance,
+            logger: logger,
+            agent_settings: agent_settings, api_version: api_version, headers: headers) do |transport|
             apis = API.defaults
 
             transport.api API::INPUT, apis[API::INPUT]
 
-            # Apply any settings given by options
-            unless options.empty?
-              transport.default_api = options[:api_version] if options.key?(:api_version)
-              transport.headers options[:headers] if options.key?(:headers)
-            end
-
             # Call block to apply any customization, if provided
             yield(transport) if block_given?
-          end
-        end
-
-        def default_headers
-          {
-            Datadog::Core::Transport::Ext::HTTP::HEADER_CLIENT_COMPUTED_TOP_LEVEL => '1',
-            Datadog::Core::Transport::Ext::HTTP::HEADER_META_LANG => Datadog::Core::Environment::Ext::LANG,
-            Datadog::Core::Transport::Ext::HTTP::HEADER_META_LANG_VERSION => Datadog::Core::Environment::Ext::LANG_VERSION,
-            Datadog::Core::Transport::Ext::HTTP::HEADER_META_LANG_INTERPRETER =>
-              Datadog::Core::Environment::Ext::LANG_INTERPRETER,
-            Datadog::Core::Transport::Ext::HTTP::HEADER_META_LANG_INTERPRETER_VENDOR => Core::Environment::Ext::LANG_ENGINE,
-            Datadog::Core::Transport::Ext::HTTP::HEADER_META_TRACER_VERSION =>
-              Datadog::Core::Environment::Ext::GEM_DATADOG_VERSION
-          }.tap do |headers|
-            # Add container ID, if present.
-            container_id = Datadog::Core::Environment::Container.container_id
-            headers[Datadog::Core::Transport::Ext::HTTP::HEADER_CONTAINER_ID] = container_id unless container_id.nil?
-            # Pretend that stats computation are already done by the client
-            if Datadog.configuration.appsec.standalone.enabled
-              headers[Datadog::Core::Transport::Ext::HTTP::HEADER_CLIENT_COMPUTED_STATS] = 'yes'
-            end
-          end
-        end
-
-        def default_adapter
-          Datadog::Core::Configuration::Ext::Agent::HTTP::ADAPTER
+          end.to_transport(DI::Transport::Input::Transport)
         end
       end
     end