datadog 2.22.0 → 2.24.0

data/lib/datadog/di/probe_manager.rb

@@ -94,6 +94,19 @@ module Datadog
       # matches.
       def add_probe(probe)
         @lock.synchronize do
+          if @installed_probes[probe.id]
+            # Either this probe was already installed, or another probe was
+            # installed with the same id (previous version perhaps?).
+            # Since our state tracking is keyed by probe id, we cannot
+            # install this probe since we won't have a way of removing the
+            # instrumentation for the probe with the same id which is already
+            # installed.
+            #
+            # The exception raised here will be caught below and logged and
+            # reported to telemetry.
+            raise Error::AlreadyInstrumented, "Probe with id #{probe.id} is already in installed probes"
+          end
+
           # Probe failed to install previously, do not try to install it again.
           if msg = @failed_probes[probe.id]
             # TODO test this path
@@ -101,7 +114,7 @@ module Datadog
           end
 
           begin
-            instrumenter.hook(probe, &method(:probe_executed_callback))
+            instrumenter.hook(probe, self)
 
             @installed_probes[probe.id] = probe
             payload = probe_notification_builder.build_installed(probe)
@@ -134,38 +147,30 @@ module Datadog
         end
       end
 
-      # Removes probes with ids other than in the specified list.
-      #
-      # This method is meant to be invoked from remote config processor.
-      # Remote config contains the list of currently defined probes; any
-      # probes not in that list have been removed by user and should be
-      # de-instrumented from the application.
-      def remove_other_probes(probe_ids)
+      # Removes probe with specified id. The probe could be pending or
+      # installed. Does nothing if there is no probe with the specified id.
+      def remove_probe(probe_id)
         @lock.synchronize do
-          @pending_probes.values.each do |probe|
-            unless probe_ids.include?(probe.id)
-              @pending_probes.delete(probe.id)
-            end
-          end
-          @installed_probes.values.each do |probe|
-            unless probe_ids.include?(probe.id)
-              begin
-                instrumenter.unhook(probe)
-                # Only remove the probe from installed list if it was
-                # successfully de-instrumented. Active probes do incur overhead
-                # for the running application, and if the error is ephemeral
-                # we want to try removing the probe again at the next opportunity.
-                #
-                # TODO give up after some time?
-                @installed_probes.delete(probe.id)
-              rescue => exc
-                raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
-                # Silence all exceptions?
-                # TODO should we propagate here and rescue upstream?
-                logger.debug { "di: error removing #{probe.type} probe at #{probe.location} (#{probe.id}): #{exc.class}: #{exc}" }
-                telemetry&.report(exc, description: "Error removing probe")
-              end
-            end
+          @pending_probes.delete(probe_id)
+        end
+
+        # Do not delete the probe from the registry here in case
+        # deinstrumentation fails - though I don't know why deinstrumentation
+        # would fail and how we could recover if it does.
+        # I plan on tracking the number of outstanding (instrumented) probes
+        # in the future, and if deinstrumentation fails I would want to
+        # keep that probe as "installed" for the count, so that we can
+        # investigate the situation.
+        if probe = @installed_probes[probe_id]
+          begin
+            instrumenter.unhook(probe)
+            @installed_probes.delete(probe_id)
+          rescue => exc
+            raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
+            # Silence all exceptions?
+            # TODO should we propagate here and rescue upstream?
+            logger.debug { "di: error removing #{probe.type} probe at #{probe.location} (#{probe.id}): #{exc.class}: #{exc}" }
+            telemetry&.report(exc, description: "Error removing probe")
           end
         end
       end
@@ -184,7 +189,8 @@ module Datadog
                 begin
                   # TODO is it OK to hook from trace point handler?
                   # TODO the class is now defined, but can hooking still fail?
-                  instrumenter.hook(probe, &method(:probe_executed_callback))
+                  instrumenter.hook(probe, self)
+                  @installed_probes[probe.id] = probe
                   @pending_probes.delete(probe.id)
                   break
                 rescue Error::DITargetNotDefined
@@ -242,6 +248,14 @@ module Datadog
         probe_notifier_worker.add_snapshot(payload)
       end
 
+      def probe_condition_evaluation_failed_callback(context, expr, exc)
+        probe = context.probe
+        if probe.condition_evaluation_failed_rate_limiter&.allow?
+          payload = probe_notification_builder.build_condition_evaluation_failed(context, expr, exc)
+          probe_notifier_worker.add_snapshot(payload)
+        end
+      end
+
       # Class/module definition trace point (:end type).
       # Used to install hooks when the target classes/modules aren't yet
       # defined when the hook request is received.

data/lib/datadog/di/probe_notification_builder.rb

@@ -46,7 +46,7 @@ module Datadog
         build_snapshot(context)
       end
 
-      NANOSECONDS = 10**9
+      NANOSECONDS = 1_000_000_000
       MILLISECONDS = 1000
 
       def build_snapshot(context)
@@ -87,10 +87,41 @@ module Datadog
           end
         end
 
+        message = nil
+        evaluation_errors = []
+        if segments = probe.template_segments
+          message, evaluation_errors = evaluate_template(segments, context)
+        end
+        build_snapshot_base(context,
+          evaluation_errors: evaluation_errors, message: message,
+          captures: captures)
+      end
+
+      def build_condition_evaluation_failed(context, expression, exception)
+        error = {
+          message: "#{exception.class}: #{exception}",
+          expr: expression.dsl_expr,
+        }
+        build_snapshot_base(context, evaluation_errors: [error])
+      end
+
+      private
+
+      def build_snapshot_base(context, evaluation_errors: [], captures: nil, message: nil)
+        probe = context.probe
+
+        timestamp = timestamp_now
+        duration = context.duration
+
         location = if probe.line?
           {
             file: context.path,
-            lines: [probe.line_no],
+            # Line numbers are required to be strings by the
+            # system tests schema.
+            # Backend I think accepts them also as integers, but some
+            # other languages send strings and we decided to require
+            # strings for everyone.
+            lines: [probe.line_no.to_s],
           }
         elsif probe.method?
           {
@@ -103,28 +134,38 @@ module Datadog
           format_caller_locations(caller_locations)
         end
 
-        timestamp = timestamp_now
-        message = nil
-        evaluation_errors = []
-        if segments = probe.template_segments
-          message, evaluation_errors = evaluate_template(segments, context)
-        end
-        duration = context.duration
-        {
+        payload = {
           service: settings.service,
-          "debugger.snapshot": {
-            id: SecureRandom.uuid,
-            timestamp: timestamp,
-            evaluationErrors: evaluation_errors,
-            probe: {
-              id: probe.id,
-              version: 0,
-              location: location,
+          debugger: {
+            type: 'snapshot',
+            # Product can have three values: di, ld, er.
+            # We do not currently implement exception replay.
+            # There is currently no specification, and no consensus, for
+            # when product should be di (dynamic instrumentation) and when
+            # it should be ld (live debugger). I thought the backend was
+            # supposed to provide this in probe specification via remote
+            # config, but apparently this is not the case and the expectation
+            # is that the library figures out the product via heuristics,
+            # except there is currently no consensus on said heuristics.
+            # .NET always sends ld, other languages send nothing at the moment.
+            # Don't send anything for the time being.
+            #product: 'di/ld',
+            snapshot: {
+              id: SecureRandom.uuid,
+              timestamp: timestamp,
+              evaluationErrors: evaluation_errors,
+              probe: {
+                id: probe.id,
+                version: 0,
+                location: location,
+              },
+              language: 'ruby',
+              # TODO add test coverage for callers being nil
+              stack: stack,
+              # System tests schema validation requires captures to
+              # always be present
+              captures: captures || {},
             },
-            language: 'ruby',
-            # TODO add test coverage for callers being nil
-            stack: stack,
-            captures: captures,
           },
           # In python tracer duration is under debugger.snapshot,
           # but UI appears to expect it here at top level.
@@ -132,7 +173,7 @@ module Datadog
           host: nil,
           logger: {
             name: probe.file,
-            method: probe.method_name || 'no_method',
+            method: probe.method_name,
             thread_name: Thread.current.name,
             # Dynamic instrumentation currently does not need thread_id for
             # anything. It can be sent if a customer requests it at which point
@@ -148,9 +189,11 @@ module Datadog
           message: message,
           timestamp: timestamp,
         }
-      end
 
-      private
+        tag_process_tags!(payload, settings)
+
+        payload
+      end
 
       def build_status(probe, message:, status:)
         {
@@ -197,6 +240,15 @@ module Datadog
         [message, evaluation_errors]
       end
 
+      def tag_process_tags!(payload, settings)
+        return unless settings.experimental_propagate_process_tags_enabled
+
+        process_tags = Core::Environment::Process.serialized
+        return if process_tags.empty?
+
+        payload[:process_tags] = process_tags
+      end
+
       def timestamp_now
         (Core::Utils::Time.now.to_f * MILLISECONDS).to_i
       end

data/lib/datadog/di/proc_responder.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Datadog
+  module DI
+    # An adapter to convert procs to responders.
+    #
+    # Used in test suite and benchmarks.
+    #
+    # @api private
+    class ProcResponder
+      def initialize(executed_proc, failed_proc = nil)
+        @executed_proc = executed_proc
+        @failed_proc = failed_proc
+      end
+
+      attr_reader :executed_proc
+      attr_reader :failed_proc
+
+      def probe_executed_callback(context)
+        executed_proc.call(context)
+      end
+
+      def probe_condition_evaluation_failed_callback(context, exc)
+        if failed_proc.nil?
+          raise NotImplementedError, "Failed proc not provided"
+        end
+
+        failed_proc.call(context, exc)
+      end
+    end
+  end
+end

data/lib/datadog/di/remote.rb

@@ -12,8 +12,6 @@ module Datadog
     #
     # @api private
     module Remote
-      class ReadError < StandardError; end
-
       class << self
         PRODUCT = 'LIVE_DEBUGGING'
 
@@ -28,7 +26,7 @@ module Datadog
         end
 
         def receivers(telemetry)
-          receiver do |repository, _changes|
+          receiver do |repository, changes|
             # DEV: Filter our by product. Given it will be very common
             # DEV: we can filter this out before we receive the data in this method.
             # DEV: Apply this refactor to AppSec as well if implemented.
@@ -41,84 +39,23 @@ module Datadog
             # If the component is nil for some reason, we also don't have a
             # logger instance to report the issue.
             if component
-
-              probe_manager = component.probe_manager
-              probe_notification_builder = component.probe_notification_builder
-              probe_notifier_worker = component.probe_notifier_worker
-
-              current_probe_ids = {}
-              repository.contents.each do |content|
-                case content.path.product
-                when PRODUCT
-                  begin
-                    probe_spec = parse_content(content)
-                    probe = component.parse_probe_spec_and_notify(probe_spec)
-                    component.logger.debug { "di: received #{probe.type} probe at #{probe.location} (#{probe.id}) via RC" }
-
-                    begin
-                      # TODO test exception capture
-                      probe_manager.add_probe(probe)
-                      content.applied
-                    rescue DI::Error::DITargetNotInRegistry => exc
-                      component.telemetry&.report(exc, description: "Line probe is targeting a loaded file that is not in code tracker")
-
-                      payload = probe_notification_builder.build_errored(probe, exc)
-                      probe_notifier_worker.add_status(payload)
-
-                      # If a probe fails to install, we will mark the content
-                      # as errored. On subsequent remote configuration application
-                      # attemps, probe manager will raise the "previously errored"
-                      # exception and we'll rescue it here, again marking the
-                      # content as errored but with a somewhat different exception
-                      # message.
-                      # TODO assert content state (errored for this example)
-                      content.errored("Error applying dynamic instrumentation configuration: #{exc.class.name} #{exc.message}")
-                    rescue => exc
-                      raise if component.settings.dynamic_instrumentation.internal.propagate_all_exceptions
-
-                      component.logger.debug { "di: unhandled exception adding #{probe.type} probe at #{probe.location} (#{probe.id}) in DI remote receiver: #{exc.class}: #{exc}" }
-                      component.telemetry&.report(exc, description: "Unhandled exception adding probe in DI remote receiver")
-
-                      # TODO test this path
-                      payload = probe_notification_builder.build_errored(probe, exc)
-                      probe_notifier_worker.add_status(payload)
-
-                      # If a probe fails to install, we will mark the content
-                      # as errored. On subsequent remote configuration application
-                      # attemps, probe manager will raise the "previously errored"
-                      # exception and we'll rescue it here, again marking the
-                      # content as errored but with a somewhat different exception
-                      # message.
-                      # TODO assert content state (errored for this example)
-                      content.errored("Error applying dynamic instrumentation configuration: #{exc.class.name} #{exc.message}")
-                    end
-
-                    # Important: even if processing fails for this probe config,
-                    # we need to note it as being current so that we do not
-                    # try to remove instrumentation that is still supposed to be
-                    # active.
-                    current_probe_ids[probe_spec.fetch('id')] = true
-                  rescue => exc
-                    raise if component.settings.dynamic_instrumentation.internal.propagate_all_exceptions
-
-                    component.logger.debug { "di: unhandled exception handling a probe in DI remote receiver: #{exc.class}: #{exc}" }
-                    component.telemetry&.report(exc, description: "Unhandled exception handling probe in DI remote receiver")
-
-                    # TODO assert content state (errored for this example)
-                    content.errored("Error applying dynamic instrumentation configuration: #{exc.class.name} #{exc.message}")
-                  end
+              changes.each do |change|
+                case change.type
+                when :insert
+                  add_probe(change.content, component)
+                when :update
+                  # We do not implement updates at the moment, remove the
+                  # probe and reinstall.
+                  remove_probe(change.content, component)
+                  add_probe(change.content, component)
+                when :delete
+                  remove_probe(change.previous, component)
+                else
+                  # This really should never happen since we generate the
+                  # change types in the library.
+                  component.logger.debug { "di: unrecognized change type: #{change.type}" }
                 end
               end
-
-              begin
-                # TODO test exception capture
-                probe_manager.remove_other_probes(current_probe_ids.keys)
-              rescue => exc
-                raise if component.settings.dynamic_instrumentation.internal.propagate_all_exceptions
-
-                component.logger.debug { "di: unhandled exception removing probes in DI remote receiver: #{exc.class}: #{exc}" }
-                component.telemetry&.report(exc, description: "Unhandled exception removing probes in DI remote receiver")
-              end
             end
           end
         end
@@ -130,14 +67,82 @@ module Datadog
 
         private
 
-        def parse_content(content)
-          data = content.data.read
+        def add_probe(content, component)
+          probe_spec = parse_content(content)
+          probe = component.parse_probe_spec_and_notify(probe_spec)
+          component.logger.debug { "di: received #{probe.type} probe at #{probe.location} (#{probe.id}) via RC" }
+
+          begin
+            # TODO test exception capture
+            component.probe_manager.add_probe(probe)
+            content.applied
+          rescue DI::Error::DITargetNotInRegistry => exc
+            component.telemetry&.report(exc, description: "Line probe is targeting a loaded file that is not in code tracker")
+
+            payload = component.probe_notification_builder.build_errored(probe, exc)
+            component.probe_notifier_worker.add_status(payload)
+
+            # If a probe fails to install, we will mark the content
+            # as errored. On subsequent remote configuration application
+            # attemps, probe manager will raise the "previously errored"
+            # exception and we'll rescue it here, again marking the
+            # content as errored but with a somewhat different exception
+            # message.
+            # TODO assert content state (errored for this example)
+            content.errored("Error applying dynamic instrumentation configuration: #{exc.class.name} #{exc.message}")
+          rescue => exc
+            raise if component.settings.dynamic_instrumentation.internal.propagate_all_exceptions
+
+            component.logger.debug { "di: unhandled exception adding #{probe.type} probe at #{probe.location} (#{probe.id}) in DI remote receiver: #{exc.class}: #{exc}" }
+            component.telemetry&.report(exc, description: "Unhandled exception adding probe in DI remote receiver")
+
+            # TODO test this path
+            payload = component.probe_notification_builder.build_errored(probe, exc)
+            component.probe_notifier_worker.add_status(payload)
+
+            # If a probe fails to install, we will mark the content
+            # as errored. On subsequent remote configuration application
+            # attemps, probe manager will raise the "previously errored"
+            # exception and we'll rescue it here, again marking the
+            # content as errored but with a somewhat different exception
+            # message.
+            # TODO assert content state (errored for this example)
+            content.errored("Error applying dynamic instrumentation configuration: #{exc.class.name} #{exc.message}")
+          end
+
+          # Important: even if processing fails for this probe config,
+          # we need to note it as being current so that we do not
+          # try to remove instrumentation that is still supposed to be
+          # active.
+          #current_probe_ids[probe_spec.fetch('id')] = true
+        rescue => exc
+          raise if component.settings.dynamic_instrumentation.internal.propagate_all_exceptions
+
+          component.logger.debug { "di: unhandled exception handling a probe in DI remote receiver: #{exc.class}: #{exc}" }
+          component.telemetry&.report(exc, description: "Unhandled exception handling probe in DI remote receiver")
 
-          content.data.rewind
+          # TODO assert content state (errored for this example)
+          content.errored("Error applying dynamic instrumentation configuration: #{exc.class.name} #{exc.message}")
+        end
 
-          raise ReadError, 'EOF reached' if data.nil?
+        # This method does not mark +previous_content+ as succeeded or errored,
+        # because that content is from a previous RC response and has already
+        # been marked. Removal of probes happens when an RC entry disappears,
+        # as such there is nothing to mark.
+        def remove_probe(previous_content, component)
+          # TODO test exception capture
+          probe_spec = parse_content(previous_content)
+          probe_id = probe_spec.fetch('id')
+          component.probe_manager.remove_probe(probe_id)
+        rescue => exc
+          raise if component.settings.dynamic_instrumentation.internal.propagate_all_exceptions
+
+          component.logger.debug { "di: unhandled exception removing probes in DI remote receiver: #{exc.class}: #{exc}" }
+          component.telemetry&.report(exc, description: "Unhandled exception removing probes in DI remote receiver")
+        end
 
-          JSON.parse(data)
+        def parse_content(content)
+          JSON.parse(content.data)
         end
       end
     end

data/lib/datadog/di/transport/diagnostics.rb

@@ -1,7 +1,11 @@
 # frozen_string_literal: true
 
+require 'json'
+
 require_relative '../../core/transport/parcel'
-require_relative 'http/client'
+require_relative '../../core/transport/request'
+require_relative '../../core/transport/transport'
+require_relative 'http/diagnostics'
 
 module Datadog
   module DI
@@ -14,46 +18,14 @@ module Datadog
         class Request < Datadog::Core::Transport::Request
         end
 
-        class Transport
-          attr_reader :client, :apis, :default_api, :current_api_id, :logger
-
-          def initialize(apis, default_api, logger:)
-            @apis = apis
-            @logger = logger
-
-            @client = HTTP::Client.new(current_api, logger: logger)
-          end
-
-          def current_api
-            @apis[HTTP::API::DIAGNOSTICS]
-          end
-
+        class Transport < Core::Transport::Transport
           def send_diagnostics(payload)
+            # TODO use transport encoder functionality?
             json = JSON.dump(payload)
             parcel = EncodedParcel.new(json)
             request = Request.new(parcel)
 
-            response = @client.send_diagnostics_payload(request)
-            unless response.ok?
-              # TODO Datadog::Core::Transport::InternalErrorResponse
-              # does not have +code+ method, what is the actual API of
-              # these response objects?
-              raise Error::AgentCommunicationError, "send_diagnostics failed: #{begin
-                response.code
-              rescue
-                "???"
-              end}: #{response.payload}"
-            end
-          rescue Error::AgentCommunicationError
-            raise
-          # Datadog::Core::Transport does not perform any exception mapping,
-          # therefore we could have any exception here from failure to parse
-          # agent URI for example.
-          # If we ever implement retries for network errors, we should distinguish
-          # actual network errors from non-network errors that are raised by
-          # transport code.
-          rescue => exc
-            raise Error::AgentCommunicationError, "send_diagnostics failed: #{exc.class}: #{exc}"
+            client.send_request(:diagnostics, request)
           end
         end
       end

data/lib/datadog/di/transport/http/diagnostics.rb

@@ -1,43 +1,13 @@
 # frozen_string_literal: true
 
-require_relative '../../../core/transport/http/api/instance'
-require_relative '../../../core/transport/http/api/spec'
-require_relative 'client'
+require_relative '../../../core/transport/http/api/endpoint'
 
 module Datadog
   module DI
     module Transport
       module HTTP
         module Diagnostics
-          module Client
-            def send_diagnostics_payload(request)
-              send_request(request) do |api, env|
-                api.send_diagnostics(env)
-              end
-            end
-          end
-
           module API
-            class Instance < Core::Transport::HTTP::API::Instance
-              def send_diagnostics(env)
-                raise Core::Transport::HTTP::API::Instance::EndpointNotSupportedError.new('diagnostics', self) unless spec.is_a?(Diagnostics::API::Spec)
-
-                spec.send_diagnostics(env) do |request_env|
-                  call(request_env)
-                end
-              end
-            end
-
-            class Spec < Core::Transport::HTTP::API::Spec
-              attr_accessor :diagnostics
-
-              def send_diagnostics(env, &block)
-                raise Core::Transport::HTTP::API::Spec::EndpointNotDefinedError.new('diagnostics', self) if diagnostics.nil?
-
-                diagnostics.call(env, &block)
-              end
-            end
-
             class Endpoint < Datadog::Core::Transport::HTTP::API::Endpoint
               attr_reader :encoder
 
@@ -57,8 +27,6 @@ module Datadog
             end
           end
         end
-
-        HTTP::Client.include(Diagnostics::Client)
       end
     end
   end

data/lib/datadog/di/transport/http/input.rb

@@ -1,43 +1,13 @@
 # frozen_string_literal: true
 
-require_relative '../../../core/transport/http/api/instance'
-require_relative '../../../core/transport/http/api/spec'
-require_relative 'client'
+require_relative '../../../core/transport/http/api/endpoint'
 
 module Datadog
   module DI
     module Transport
       module HTTP
         module Input
-          module Client
-            def send_input_payload(request)
-              send_request(request) do |api, env|
-                api.send_input(env)
-              end
-            end
-          end
-
           module API
-            class Instance < Core::Transport::HTTP::API::Instance
-              def send_input(env)
-                raise Core::Transport::HTTP::API::Instance::EndpointNotSupportedError.new('input', self) unless spec.is_a?(Input::API::Spec)
-
-                spec.send_input(env) do |request_env|
-                  call(request_env)
-                end
-              end
-            end
-
-            class Spec < Core::Transport::HTTP::API::Spec
-              attr_accessor :input
-
-              def send_input(env, &block)
-                raise Core::Transport::HTTP::API::Spec::EndpointNotDefinedError.new('input', self) if input.nil?
-
-                input.call(env, &block)
-              end
-            end
-
             class Endpoint < Datadog::Core::Transport::HTTP::API::Endpoint
               HEADER_CONTENT_TYPE = 'Content-Type'
 
@@ -69,8 +39,6 @@ module Datadog
             end
           end
         end
-
-        HTTP::Client.include(Input::Client)
       end
     end
   end