datadog 2.7.1 → 2.18.0

data/lib/datadog/di/probe_notifier_worker.rb

@@ -10,7 +10,7 @@ module Datadog
     # The loop inside the worker rescues all exceptions to prevent termination
     # due to unhandled exceptions raised by any downstream code.
     # This includes communication and protocol errors when sending the
-    # payloads to the agent.
+    # events to the agent.
     #
     # The worker groups the data to send into batches. The goal is to perform
     # no more than one network operation per event type per second.
@@ -23,12 +23,12 @@ module Datadog
     #
     # @api private
     class ProbeNotifierWorker
-      def initialize(settings, transport, logger, telemetry: nil)
+      def initialize(settings, logger, agent_settings:, telemetry: nil)
         @settings = settings
         @telemetry = telemetry
         @status_queue = []
         @snapshot_queue = []
-        @transport = transport
+        @agent_settings = agent_settings
         @logger = logger
         @lock = Mutex.new
         @wake = Core::Semaphore.new
@@ -36,15 +36,18 @@ module Datadog
         @sleep_remaining = nil
         @wake_scheduled = false
         @thread = nil
+        @pid = nil
         @flush = 0
       end
 
       attr_reader :settings
       attr_reader :logger
       attr_reader :telemetry
+      attr_reader :agent_settings
 
       def start
-        return if @thread
+        return if @thread && @pid == Process.pid
+        logger.trace { "di: starting probe notifier: pid #{$$}" }
         @thread = Thread.new do
           loop do
             # TODO If stop is requested, we stop immediately without
@@ -77,7 +80,7 @@ module Datadog
             rescue => exc
               raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
 
-              logger.warn("Error in probe notifier worker: #{exc.class}: #{exc} (at #{exc.backtrace.first})")
+              logger.debug { "di: error in probe notifier worker: #{exc.class}: #{exc} (at #{exc.backtrace.first})" }
               telemetry&.report(exc, description: "Error in probe notifier worker")
             end
             @lock.synchronize do
@@ -86,6 +89,7 @@ module Datadog
             wake.wait(more ? min_send_interval : nil)
           end
         end
+        @pid = Process.pid
       end
 
       # Stops the background thread.
@@ -94,6 +98,7 @@ module Datadog
       # to killing the thread using Thread#kill.
       def stop(timeout = 1)
         @stop_requested = true
+        logger.trace { "di: stopping probe notifier: pid #{$$}" }
         wake.signal
         if thread
           unless thread.join(timeout)
@@ -150,7 +155,6 @@ module Datadog
 
       private
 
-      attr_reader :transport
       attr_reader :wake
       attr_reader :thread
 
@@ -166,6 +170,30 @@ module Datadog
 
       attr_reader :last_sent
 
+      def status_transport
+        @status_transport ||= DI::Transport::HTTP.diagnostics(agent_settings: agent_settings, logger: logger)
+      end
+
+      def do_send_status(batch)
+        status_transport.send_diagnostics(batch)
+      end
+
+      def snapshot_transport
+        @snapshot_transport ||= DI::Transport::HTTP.input(agent_settings: agent_settings, logger: logger)
+      end
+
+      def do_send_snapshot(batch)
+        snapshot_transport.send_input(batch, tags)
+      end
+
+      def tags
+        # DEV: The tags could be cached but they need to be recreated
+        # when process forks (since the child receives new runtime IDs).
+        Core::TagBuilder.tags(settings).merge(
+          'debugger_version' => Core::Environment::Identity.gem_datadog_version,
+        )
+      end
+
       [
         [:status, 'probe status'],
         [:snapshot, 'snapshot'],
@@ -183,10 +211,10 @@ module Datadog
         define_method("add_#{event_type}") do |event|
           @lock.synchronize do
             queue = send("#{event_type}_queue")
-            # TODO determine a suitable limit via testing/benchmarking
-            if queue.length > 100
-              logger.warn("#{self.class.name}: dropping #{event_type} because queue is full")
+            if queue.length > settings.dynamic_instrumentation.internal.snapshot_queue_capacity
+              logger.debug { "di: #{self.class.name}: dropping #{event_type} event because queue is full" }
             else
+              logger.trace { "di: #{self.class.name}: queueing #{event_type} event" }
               queue << event
             end
           end
@@ -201,20 +229,10 @@ module Datadog
               wake.signal
             end
           end
-        end
 
-        # Determine how much longer the worker thread should sleep
-        # so as not to send in less than min send interval since the last send.
-        # Important: this method must be called when @lock is held.
-        #
-        # Returns the time remaining to sleep.
-        def set_sleep_remaining
-          now = Core::Utils::Time.get_time
-          @sleep_remaining = if last_sent
-            [last_sent + min_send_interval - now, 0].max
-          else
-            0
-          end
+          # Worker could be not running if the process forked - check and
+          # start it again in this case.
+          start
         end
 
         public "add_#{event_type}"
@@ -233,16 +251,18 @@ module Datadog
             instance_variable_set("@#{event_type}_queue", [])
             @io_in_progress = batch.any? # steep:ignore
           end
+          logger.trace { "di: #{self.class.name}: checking #{event_type} queue - #{batch.length} entries" } # steep:ignore
           if batch.any? # steep:ignore
             begin
-              transport.public_send("send_#{event_type}", batch)
+              logger.trace { "di: sending #{batch.length} #{event_type} event(s) to agent" } # steep:ignore
+              send("do_send_#{event_type}", batch)
               time = Core::Utils::Time.get_time
               @lock.synchronize do
                 @last_sent = time
               end
             rescue => exc
               raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
-              logger.warn("failed to send #{event_name}: #{exc.class}: #{exc} (at #{exc.backtrace.first})")
+              logger.debug { "di: failed to send #{event_name}: #{exc.class}: #{exc} (at #{exc.backtrace.first})" }
               # Should we report this error to telemetry? Most likely failure
               # to send is due to a network issue, and trying to send a
               # telemetry message would also fail.
@@ -253,7 +273,7 @@ module Datadog
           # Normally the queue should only be consumed in this method,
           # however if anyone consumes it elsewhere we don't want to block
           # while consuming it here. Rescue ThreadError and return.
-          logger.warn("Unexpected #{event_name} queue underflow - consumed elsewhere?")
+          logger.debug { "di: unexpected #{event_name} queue underflow - consumed elsewhere?" }
           telemetry&.report(exc, description: "Unexpected #{event_name} queue underflow")
         ensure
           @lock.synchronize do
@@ -262,9 +282,23 @@ module Datadog
         end
       end
 
+      # Determine how much longer the worker thread should sleep
+      # so as not to send in less than min send interval since the last send.
+      # Important: this method must be called when @lock is held.
+      #
+      # Returns the time remaining to sleep.
+      def set_sleep_remaining
+        now = Core::Utils::Time.get_time
+        @sleep_remaining = if last_sent
+          [last_sent + min_send_interval - now, 0].max
+        else
+          0
+        end
+      end
+
       def maybe_send
         rv = maybe_send_status
-        rv || maybe_send_snapshot
+        maybe_send_snapshot || rv
       end
     end
   end

data/lib/datadog/di/redactor.rb

@@ -119,7 +119,6 @@ module Datadog
         "dburl",
         "encryptionkey",
         "encryptionkeyid",
-        "env",
         "geolocation",
         "gpgkey",
         "ipaddress",

data/lib/datadog/di/remote.rb

@@ -0,0 +1,147 @@
+# frozen_string_literal: true
+
+module Datadog
+  module DI
+    # Provides an interface expected by the core Remote subsystem to
+    # receive DI-specific remote configuration.
+    #
+    # In order to apply (i.e., act on) the configuration, we need the
+    # state stored under DI Component. Thus, this module forwards actual
+    # configuration application to the ProbeManager associated with the
+    # global DI Component.
+    #
+    # @api private
+    module Remote
+      class ReadError < StandardError; end
+
+      class << self
+        PRODUCT = 'LIVE_DEBUGGING'
+
+        def products
+          # TODO: do not send our product on unsupported runtimes
+          # (Ruby 2.5 / JRuby)
+          [PRODUCT]
+        end
+
+        def capabilities
+          []
+        end
+
+        def receivers(telemetry)
+          receiver do |repository, _changes|
+            # DEV: Filter our by product. Given it will be very common
+            # DEV: we can filter this out before we receive the data in this method.
+            # DEV: Apply this refactor to AppSec as well if implemented.
+
+            component = DI.component
+            # We should always have a non-nil DI component here, because we
+            # only add DI product to remote config request if DI is enabled.
+            # Ideally, we should be injected with the DI component here
+            # rather than having to retrieve it from global state.
+            # If the component is nil for some reason, we also don't have a
+            # logger instance to report the issue.
+            if component
+
+              probe_manager = component.probe_manager
+
+              current_probe_ids = {}
+              repository.contents.each do |content|
+                case content.path.product
+                when PRODUCT
+                  begin
+                    probe_spec = parse_content(content)
+                    probe = ProbeBuilder.build_from_remote_config(probe_spec)
+                    probe_notification_builder = component.probe_notification_builder
+                    payload = probe_notification_builder.build_received(probe)
+                    probe_notifier_worker = component.probe_notifier_worker
+                    probe_notifier_worker.add_status(payload)
+                    component.logger.debug { "di: received #{probe.type} probe at #{probe.location} (#{probe.id}) via RC" }
+
+                    begin
+                      # TODO test exception capture
+                      probe_manager.add_probe(probe)
+                      content.applied
+                    rescue DI::Error::DITargetNotInRegistry => exc
+                      component.telemetry&.report(exc, description: "Line probe is targeting a loaded file that is not in code tracker")
+
+                      payload = probe_notification_builder.build_errored(probe, exc)
+                      probe_notifier_worker.add_status(payload)
+
+                      # If a probe fails to install, we will mark the content
+                      # as errored. On subsequent remote configuration application
+                      # attemps, probe manager will raise the "previously errored"
+                      # exception and we'll rescue it here, again marking the
+                      # content as errored but with a somewhat different exception
+                      # message.
+                      # TODO assert content state (errored for this example)
+                      content.errored("Error applying dynamic instrumentation configuration: #{exc.class.name} #{exc.message}")
+                    rescue => exc
+                      raise if component.settings.dynamic_instrumentation.internal.propagate_all_exceptions
+
+                      component.logger.debug { "di: unhandled exception adding #{probe.type} probe at #{probe.location} (#{probe.id}) in DI remote receiver: #{exc.class}: #{exc}" }
+                      component.telemetry&.report(exc, description: "Unhandled exception adding probe in DI remote receiver")
+
+                      # TODO test this path
+                      payload = probe_notification_builder.build_errored(probe, exc)
+                      probe_notifier_worker.add_status(payload)
+
+                      # If a probe fails to install, we will mark the content
+                      # as errored. On subsequent remote configuration application
+                      # attemps, probe manager will raise the "previously errored"
+                      # exception and we'll rescue it here, again marking the
+                      # content as errored but with a somewhat different exception
+                      # message.
+                      # TODO assert content state (errored for this example)
+                      content.errored("Error applying dynamic instrumentation configuration: #{exc.class.name} #{exc.message}")
+                    end
+
+                    # Important: even if processing fails for this probe config,
+                    # we need to note it as being current so that we do not
+                    # try to remove instrumentation that is still supposed to be
+                    # active.
+                    current_probe_ids[probe_spec.fetch('id')] = true
+                  rescue => exc
+                    raise if component.settings.dynamic_instrumentation.internal.propagate_all_exceptions
+
+                    component.logger.debug { "di: unhandled exception handling a probe in DI remote receiver: #{exc.class}: #{exc}" }
+                    component.telemetry&.report(exc, description: "Unhandled exception handling probe in DI remote receiver")
+
+                    # TODO assert content state (errored for this example)
+                    content.errored("Error applying dynamic instrumentation configuration: #{exc.class.name} #{exc.message}")
+                  end
+                end
+              end
+
+              begin
+                # TODO test exception capture
+                probe_manager.remove_other_probes(current_probe_ids.keys)
+              rescue => exc
+                raise if component.settings.dynamic_instrumentation.internal.propagate_all_exceptions
+
+                component.logger.debug { "di: unhandled exception removing probes in DI remote receiver: #{exc.class}: #{exc}" }
+                component.telemetry&.report(exc, description: "Unhandled exception removing probes in DI remote receiver")
+              end
+            end
+          end
+        end
+
+        def receiver(products = [PRODUCT], &block)
+          matcher = Core::Remote::Dispatcher::Matcher::Product.new(products)
+          [Core::Remote::Dispatcher::Receiver.new(matcher, &block)]
+        end
+
+        private
+
+        def parse_content(content)
+          data = content.data.read
+
+          content.data.rewind
+
+          raise ReadError, 'EOF reached' if data.nil?
+
+          JSON.parse(data)
+        end
+      end
+    end
+  end
+end

data/lib/datadog/di/serializer.rb

@@ -82,24 +82,32 @@ module Datadog
       # between positional and keyword arguments. We convert positional
       # arguments to keyword arguments ("arg1", "arg2", ...) and ensure
       # the positional arguments are listed first.
-      def serialize_args(args, kwargs)
+      #
+      # Instance variables are technically a hash just like kwargs,
+      # we take them as a separate parameter to avoid a hash merge
+      # in upstream code.
+      def serialize_args(args, kwargs, instance_vars,
+        depth: settings.dynamic_instrumentation.max_capture_depth,
+        attribute_count: settings.dynamic_instrumentation.max_capture_attribute_count)
         counter = 0
         combined = args.each_with_object({}) do |value, c|
           counter += 1
           # Conversion to symbol is needed here to put args ahead of
           # kwargs when they are merged below.
           c[:"arg#{counter}"] = value
-        end.update(kwargs)
-        serialize_vars(combined)
+        end.update(kwargs).update(instance_vars)
+        serialize_vars(combined, depth: depth, attribute_count: attribute_count)
       end
 
       # Serializes variables captured by a line probe.
       #
       # These are normally local variables that exist on a particular line
       # of executed code.
-      def serialize_vars(vars)
+      def serialize_vars(vars,
+        depth: settings.dynamic_instrumentation.max_capture_depth,
+        attribute_count: settings.dynamic_instrumentation.max_capture_attribute_count)
         vars.each_with_object({}) do |(k, v), agg|
-          agg[k] = serialize_value(v, name: k)
+          agg[k] = serialize_value(v, name: k, depth: depth, attribute_count: attribute_count)
         end
       end
 
@@ -115,7 +123,11 @@ module Datadog
       # (integers, strings, arrays, hashes).
       #
       # Respects string length, collection size and traversal depth limits.
-      def serialize_value(value, name: nil, depth: settings.dynamic_instrumentation.max_capture_depth, type: nil)
+      def serialize_value(value, name: nil,
+        depth: settings.dynamic_instrumentation.max_capture_depth,
+        attribute_count: nil,
+        type: nil)
+        attribute_count ||= settings.dynamic_instrumentation.max_capture_attribute_count
         cls = type || value.class
         begin
           if redactor.redact_type?(value)
@@ -203,7 +215,6 @@ module Datadog
               serialized.update(notCapturedReason: "depth")
             else
               fields = {}
-              max = settings.dynamic_instrumentation.max_capture_attribute_count
               cur = 0
 
               # MRI and JRuby 9.4.5+ preserve instance variable definition
@@ -229,7 +240,7 @@ module Datadog
               ivars = value.instance_variables
 
               ivars.each do |ivar|
-                if cur >= max
+                if cur >= attribute_count
                   serialized.update(notCapturedReason: "fieldCount", fields: fields)
                   break
                 end

data/lib/datadog/di/transport/diagnostics.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+require_relative '../../core/transport/parcel'
+require_relative 'http/client'
+
+module Datadog
+  module DI
+    module Transport
+      module Diagnostics
+        class EncodedParcel
+          include Datadog::Core::Transport::Parcel
+        end
+
+        class Request < Datadog::Core::Transport::Request
+        end
+
+        class Transport
+          attr_reader :client, :apis, :default_api, :current_api_id, :logger
+
+          def initialize(apis, default_api, logger:)
+            @apis = apis
+            @logger = logger
+
+            @client = HTTP::Client.new(current_api, logger: logger)
+          end
+
+          def current_api
+            @apis[HTTP::API::DIAGNOSTICS]
+          end
+
+          def send_diagnostics(payload)
+            json = JSON.dump(payload)
+            parcel = EncodedParcel.new(json)
+            request = Request.new(parcel)
+
+            response = @client.send_diagnostics_payload(request)
+            unless response.ok?
+              # TODO Datadog::Core::Transport::InternalErrorResponse
+              # does not have +code+ method, what is the actual API of
+              # these response objects?
+              raise Error::AgentCommunicationError, "send_diagnostics failed: #{begin
+                response.code
+              rescue
+                "???"
+              end}: #{response.payload}"
+            end
+          rescue Error::AgentCommunicationError
+            raise
+          # Datadog::Core::Transport does not perform any exception mapping,
+          # therefore we could have any exception here from failure to parse
+          # agent URI for example.
+          # If we ever implement retries for network errors, we should distinguish
+          # actual network errors from non-network errors that are raised by
+          # transport code.
+          rescue => exc
+            raise Error::AgentCommunicationError, "send_diagnostics failed: #{exc.class}: #{exc}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/di/transport/http/api.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require_relative '../../../core/encoding'
+require_relative '../../../core/transport/http/api/map'
+require_relative '../../../core/transport/http/api/instance'
+require_relative '../../../core/transport/http/api/spec'
+require_relative 'diagnostics'
+require_relative 'input'
+
+module Datadog
+  module DI
+    module Transport
+      module HTTP
+        # Namespace for API components
+        module API
+          # Default API versions
+          DIAGNOSTICS = 'diagnostics'
+          INPUT = 'input'
+
+          module_function
+
+          def defaults
+            Datadog::Core::Transport::HTTP::API::Map[
+              DIAGNOSTICS => Diagnostics::API::Spec.new do |s|
+                s.diagnostics = Diagnostics::API::Endpoint.new(
+                  '/debugger/v1/diagnostics',
+                  Core::Encoding::JSONEncoder,
+                )
+              end,
+              INPUT => Input::API::Spec.new do |s|
+                s.input = Input::API::Endpoint.new(
+                  '/debugger/v1/input',
+                  Core::Encoding::JSONEncoder,
+                )
+              end,
+            ]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/di/transport/http/client.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+require_relative '../../../core/transport/http/env'
+require_relative '../../../core/transport/http/response'
+
+# TODO: Decouple transport/http/client
+#
+# The standard one does `include Transport::HTTP::Statistics` and performs
+# stats updates, which may or may not be desirable in general.
+
+module Datadog
+  module DI
+    module Transport
+      module HTTP
+        # Routes, encodes, and sends DI data to the trace agent via HTTP.
+        class Client
+          attr_reader :api, :logger
+
+          def initialize(api, logger:)
+            @api = api
+            @logger = logger
+          end
+
+          def send_request(request, &block)
+            # Build request into env
+            env = build_env(request)
+
+            # Get responses from API
+            yield(api, env)
+          rescue => e
+            message =
+              "Internal error during #{self.class.name} request. Cause: #{e.class.name} #{e.message} " \
+                "Location: #{Array(e.backtrace).first}"
+
+            logger.debug(message)
+
+            Datadog::Core::Transport::InternalErrorResponse.new(e)
+          end
+
+          def build_env(request)
+            Datadog::Core::Transport::HTTP::Env.new(request)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/di/transport/http/diagnostics.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+require_relative '../../../core/transport/http/api/instance'
+require_relative '../../../core/transport/http/api/spec'
+require_relative 'client'
+
+module Datadog
+  module DI
+    module Transport
+      module HTTP
+        module Diagnostics
+          module Client
+            def send_diagnostics_payload(request)
+              send_request(request) do |api, env|
+                api.send_diagnostics(env)
+              end
+            end
+          end
+
+          module API
+            class Instance < Core::Transport::HTTP::API::Instance
+              def send_diagnostics(env)
+                raise Core::Transport::HTTP::API::Instance::EndpointNotSupportedError.new('diagnostics', self) unless spec.is_a?(Diagnostics::API::Spec)
+
+                spec.send_diagnostics(env) do |request_env|
+                  call(request_env)
+                end
+              end
+            end
+
+            class Spec < Core::Transport::HTTP::API::Spec
+              attr_accessor :diagnostics
+
+              def send_diagnostics(env, &block)
+                raise Core::Transport::HTTP::API::Spec::EndpointNotDefinedError.new('diagnostics', self) if diagnostics.nil?
+
+                diagnostics.call(env, &block)
+              end
+            end
+
+            class Endpoint < Datadog::Core::Transport::HTTP::API::Endpoint
+              attr_reader :encoder
+
+              def initialize(path, encoder)
+                super(:post, path)
+                @encoder = encoder
+              end
+
+              def call(env, &block)
+                event_payload = Core::Vendor::Multipart::Post::UploadIO.new(
+                  StringIO.new(env.request.parcel.data), 'application/json', 'event.json'
+                )
+                env.form = {'event' => event_payload}
+
+                super
+              end
+            end
+          end
+        end
+
+        HTTP::Client.include(Diagnostics::Client)
+      end
+    end
+  end
+end