datadog 2.8.0 → 2.10.0

data/lib/datadog/core/telemetry/metric.rb

@@ -41,6 +41,18 @@ module Datadog
             }
           end
 
+          def ==(other)
+            other.is_a?(self.class) &&
+              name == other.name &&
+              values == other.values && tags == other.tags && common == other.common && type == other.type
+          end
+
+          alias eql? ==
+
+          def hash
+            [self.class, name, values, tags, common, type].hash
+          end
+
           private
 
           def tags_to_array(tags)
@@ -71,6 +83,16 @@ module Datadog
             res[:interval] = interval
             res
           end
+
+          def ==(other)
+            super && interval == other.interval
+          end
+
+          alias eql? ==
+
+          def hash
+            [super, interval].hash
+          end
         end
 
         # Count metric adds up all the submitted values in a time interval. This would be suitable for a

data/lib/datadog/core/telemetry/worker.rb

@@ -97,6 +97,8 @@ module Datadog
           return if events.empty?
           return if !enabled? || !sent_started_event?
 
+          events = deduplicate_logs(events)
+
           Datadog.logger.debug { "Sending #{events&.count} telemetry events" }
           send_event(Event::MessageBatch.new(events))
         end
@@ -167,6 +169,37 @@ module Datadog
           Datadog.logger.debug('Agent does not support telemetry; disabling future telemetry events.')
           disable!
         end
+
+        # Deduplicate logs by counting the number of repeated occurrences of the same log
+        # entry and replacing them with a single entry with the calculated `count` value.
+        # Non-log events are unchanged.
+        def deduplicate_logs(events)
+          return events if events.empty?
+
+          all_logs = []
+          other_events = events.reject do |event|
+            if event.is_a?(Event::Log)
+              all_logs << event
+              true
+            else
+              false
+            end
+          end
+
+          return events if all_logs.empty?
+
+          uniq_logs = all_logs.group_by(&:itself).map do |_, logs|
+            log = logs.first
+            if logs.size > 1
+              # New log event with a count of repeated occurrences
+              Event::Log.new(message: log.message, level: log.level, stack_trace: log.stack_trace, count: logs.size)
+            else
+              log
+            end
+          end
+
+          other_events + uniq_logs
+        end
       end
     end
   end

data/lib/datadog/di/base.rb

@@ -0,0 +1,115 @@
+# frozen_string_literal: true
+
+# This file is loaded by datadog/di/preload.rb.
+# It contains just the global DI reference to the (normally one and only)
+# code tracker for the current process.
+# This file should not require the rest of DI, specifically none of the
+# contrib code that is meant to be loaded after third-party libraries
+# are loaded, and also none of the rest of datadog library which also
+# has contrib code in other products.
+
+require_relative 'code_tracker'
+
+module Datadog
+  # Namespace for Datadog dynamic instrumentation.
+  #
+  # @api private
+  module DI
+    LOCK = Mutex.new
+
+    class << self
+      attr_reader :code_tracker
+
+      # Activates code tracking. Normally this method should be called
+      # when the application starts. If instrumenting third-party code,
+      # code tracking needs to be enabled before the third-party libraries
+      # are loaded. Any third-party code loaded before code tracking is
+      # activated will NOT be instrumentable using dynamic instrumentation.
+      #
+      # TODO test that activating tracker multiple times preserves
+      # existing mappings in the registry
+      def activate_tracking!
+        (@code_tracker ||= CodeTracker.new).start
+      end
+
+      # Activates code tracking if possible.
+      #
+      # This method does nothing if invoked in an environment that does not
+      # implement required trace points for code tracking (MRI Ruby < 2.6,
+      # JRuby) and rescues any exceptions that may be raised by downstream
+      # DI code.
+      def activate_tracking
+        # :script_compiled trace point was added in Ruby 2.6.
+        return unless RUBY_VERSION >= '2.6'
+
+        begin
+          # Activate code tracking by default because line trace points will not work
+          # without it.
+          Datadog::DI.activate_tracking!
+        rescue => exc
+          if defined?(Datadog.logger)
+            Datadog.logger.warn { "di: Failed to activate code tracking for DI: #{exc.class}: #{exc}" }
+          else
+            # We do not have Datadog logger potentially because DI code tracker is
+            # being loaded early in application boot process and the rest of datadog
+            # wasn't loaded yet. Output to standard error.
+            warn("datadog: di: Failed to activate code tracking for DI: #{exc.class}: #{exc}")
+          end
+        end
+      end
+
+      # Deactivates code tracking. In normal usage of DI this method should
+      # never be called, however it is used by DI's test suite to reset
+      # state for individual tests.
+      #
+      # Note that deactivating tracking clears out the registry, losing
+      # the ability to look up files that have been loaded into the process
+      # already.
+      def deactivate_tracking!
+        code_tracker&.stop
+      end
+
+      # Returns whether code tracking is available.
+      # This method should be used instead of querying #code_tracker
+      # because the latter one may be nil.
+      def code_tracking_active?
+        code_tracker&.active? || false
+      end
+
+      # DI code tracker is instantiated globally before the regular set of
+      # components is created, but the code tracker needs to call out to the
+      # "current" DI component to perform instrumentation when application
+      # code is loaded. Because this call may happen prior to Datadog
+      # components having been initialized, we maintain the "current component"
+      # which contains a reference to the most recently instantiated
+      # DI::Component. This way, if a DI component hasn't been instantiated,
+      # we do not try to reference Datadog.components.
+      # In other words, this method exists so that we never attempt to call
+      # Datadog.components from the code tracker.
+      def current_component
+        LOCK.synchronize do
+          @current_components&.last
+        end
+      end
+
+      # To avoid potential races with DI::Component being added and removed,
+      # we maintain a list of the components. Normally the list should contain
+      # either zero or one component depending on whether DI is enabled in
+      # Datadog configuration. However, if a new instance of DI::Component
+      # is created while the previous instance is still running, we are
+      # guaranteed to not end up with no component when one is running.
+      def add_current_component(component)
+        LOCK.synchronize do
+          @current_components ||= []
+          @current_components << component
+        end
+      end
+
+      def remove_current_component(component)
+        LOCK.synchronize do
+          @current_components&.delete(component)
+        end
+      end
+    end
+  end
+end

data/lib/datadog/di/code_tracker.rb

@@ -2,6 +2,8 @@
 
 # rubocop:disable Lint/AssignmentInCondition
 
+require_relative 'error'
+
 module Datadog
   module DI
     # Tracks loaded Ruby code by source file and maintains a map from
@@ -87,11 +89,12 @@ module Datadog
           # rescue any exceptions that might not be handled to not break said
           # customer applications.
           rescue => exc
-            # TODO we do not have DI.component defined yet, remove steep:ignore
-            # before release.
-            if component = DI.current_component # steep:ignore
+            # Code tracker may be loaded without the rest of DI,
+            # in which case DI.component will not yet be defined,
+            # but we will have DI.current_component (set to nil).
+            if component = DI.current_component
               raise if component.settings.dynamic_instrumentation.internal.propagate_all_exceptions
-              component.logger.warn("Unhandled exception in script_compiled trace point: #{exc.class}: #{exc}")
+              component.logger.debug { "di: unhandled exception in script_compiled trace point: #{exc.class}: #{exc}" }
               component.telemetry&.report(exc, description: "Unhandled exception in script_compiled trace point")
               # TODO test this path
             else

data/lib/datadog/di/component.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require_relative '../core'
+
 module Datadog
   module DI
     # Component for dynamic instrumentation.
@@ -14,22 +16,22 @@ module Datadog
     # resources and installed tracepoints upon shutdown.
     class Component
       class << self
-        def build(settings, agent_settings, telemetry: nil)
+        def build(settings, agent_settings, logger, telemetry: nil)
           return unless settings.respond_to?(:dynamic_instrumentation) && settings.dynamic_instrumentation.enabled
 
           unless settings.respond_to?(:remote) && settings.remote.enabled
-            Datadog.logger.debug("Dynamic Instrumentation could not be enabled because Remote Configuration Management is not available. To enable Remote Configuration, see https://docs.datadoghq.com/agent/remote_config")
+            logger.warn("di: dynamic instrumentation could not be enabled because Remote Configuration Management is not available. To enable Remote Configuration, see https://docs.datadoghq.com/agent/remote_config")
             return
           end
 
-          return unless environment_supported?(settings)
+          return unless environment_supported?(settings, logger)
 
-          new(settings, agent_settings, Datadog.logger, code_tracker: DI.code_tracker, telemetry: telemetry).tap do |component|
+          new(settings, agent_settings, logger, code_tracker: DI.code_tracker, telemetry: telemetry).tap do |component|
             DI.add_current_component(component)
           end
         end
 
-        def build!(settings, agent_settings, telemetry: nil)
+        def build!(settings, agent_settings, logger, telemetry: nil)
           unless settings.respond_to?(:dynamic_instrumentation) && settings.dynamic_instrumentation.enabled
             raise "Requested DI component but DI is not enabled in settings"
           end
@@ -38,27 +40,31 @@ module Datadog
             raise "Requested DI component but remote config is not enabled in settings"
           end
 
-          unless environment_supported?(settings)
+          unless environment_supported?(settings, logger)
             raise "DI does not support the environment (development or Ruby version too low or not MRI)"
           end
 
-          new(settings, agent_settings, Datadog.logger, code_tracker: DI.code_tracker, telemetry: telemetry)
+          new(settings, agent_settings, logger, code_tracker: DI.code_tracker, telemetry: telemetry)
         end
 
         # Checks whether the runtime environment is supported by
         # dynamic instrumentation. Currently we only require that, if Rails
         # is used, that Rails environment is not development because
         # DI does not currently support code unloading and reloading.
-        def environment_supported?(settings)
+        def environment_supported?(settings, logger)
           # TODO add tests?
           unless settings.dynamic_instrumentation.internal.development
             if Datadog::Core::Environment::Execution.development?
-              Datadog.logger.debug("Not enabling dynamic instrumentation because we are in development environment")
+              logger.warn("di: development environment detected; not enabling dynamic instrumentation")
               return false
             end
           end
-          if RUBY_ENGINE != 'ruby' || RUBY_VERSION < '2.6'
-            Datadog.logger.debug("Not enabling dynamic instrumentation because of unsupported Ruby version")
+          if RUBY_ENGINE != 'ruby'
+            logger.warn("di: cannot enable dynamic instrumentation: MRI is required, but running on #{RUBY_ENGINE}")
+            return false
+          end
+          if RUBY_VERSION < '2.6'
+            logger.warn("di: cannot enable dynamic instrumentation: Ruby 2.6+ is required, but running on #{RUBY_VERSION}")
             return false
           end
           true
@@ -70,6 +76,7 @@ module Datadog
         @agent_settings = agent_settings
         @logger = logger
         @telemetry = telemetry
+        @code_tracker = code_tracker
         @redactor = Redactor.new(settings)
         @serializer = Serializer.new(settings, redactor, telemetry: telemetry)
         @instrumenter = Instrumenter.new(settings, serializer, logger, code_tracker: code_tracker, telemetry: telemetry)
@@ -84,6 +91,7 @@ module Datadog
       attr_reader :agent_settings
       attr_reader :logger
       attr_reader :telemetry
+      attr_reader :code_tracker
       attr_reader :instrumenter
       attr_reader :transport
       attr_reader :probe_notifier_worker

data/lib/datadog/di/configuration/settings.rb

@@ -166,10 +166,20 @@ module Datadog
                 # being sent out by the probe notifier worker) and creates a
                 # possibility of dropping payloads if the queue gets too long.
                 option :min_send_interval do |o|
-                  o.type :int
+                  o.type :float
                   o.default 3
                 end
 
+                # Number of snapshots that can be stored in the probe
+                # notifier worker queue. Larger capacity runs the risk of
+                # creating snapshots that exceed the agent's request size
+                # limit. Smaller capacity increases the risk of dropping
+                # snapshots.
+                option :snapshot_queue_capacity do |o|
+                  o.type :int
+                  o.default 100
+                end
+
                 # Enable dynamic instrumentation in development environments.
                 # Currently DI does not fully implement support for code
                 # unloading and reloading, and is not supported in

data/lib/datadog/di/contrib/railtie.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Datadog
+  module DI
+    module Contrib
+      # Railtie class initializes dynamic instrumentation contrib code
+      # in Rails environments.
+      class Railtie < Rails::Railtie
+        initializer 'datadog.dynamic_instrumentation.initialize' do |app|
+          Contrib.load_now
+        end
+      end
+    end
+  end
+end

data/lib/datadog/di/contrib.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require_relative '../core/contrib/rails/utils'
+
+module Datadog
+  module DI
+    module Contrib
+      module_function def load_now_or_later
+        if Datadog::Core::Contrib::Rails::Utils.railtie_supported?
+          require_relative 'contrib/railtie'
+        else
+          load_now
+        end
+      end
+
+      # This method can be called more than once, to attempt to load
+      # DI components that depend on third-party libraries after additional
+      # dependencies are loaded (or potentially loaded).
+      module_function def load_now
+        if defined?(ActiveRecord::Base)
+          require_relative 'contrib/active_record'
+        end
+      end
+    end
+  end
+end

data/lib/datadog/di/error.rb

@@ -27,6 +27,11 @@ module Datadog
       class DITargetNotDefined < Error
       end
 
+      # Attempting to instrument a line and the file containing the line
+      # was loaded prior to code tracking being enabled.
+      class DITargetNotInRegistry < Error
+      end
+
       # Raised when trying to install a probe whose installation failed
       # earlier in the same process. This exception should contain the
       # original exception report from initial installation attempt.

data/lib/datadog/di/instrumenter.rb

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
-# rubocop:disable Lint/AssignmentInCondition
+require_relative '../core/utils/time'
 
-require 'benchmark'
+# rubocop:disable Lint/AssignmentInCondition
 
 module Datadog
   module DI
@@ -115,22 +115,21 @@ module Datadog
                   depth: probe.max_capture_depth || settings.dynamic_instrumentation.max_capture_depth,
                   attribute_count: probe.max_capture_attribute_count || settings.dynamic_instrumentation.max_capture_attribute_count)
               end
-              rv = nil
+              start_time = Core::Utils::Time.get_time
               # Under Ruby 2.6 we cannot just call super(*args, **kwargs)
               # for methods defined via method_missing.
-              duration = Benchmark.realtime do # steep:ignore
-                rv = if args.any?
-                  if kwargs.any?
-                    super(*args, **kwargs, &target_block)
-                  else
-                    super(*args, &target_block)
-                  end
-                elsif kwargs.any?
-                  super(**kwargs, &target_block)
+              rv = if args.any?
+                if kwargs.any?
+                  super(*args, **kwargs, &target_block)
                 else
-                  super(&target_block)
+                  super(*args, &target_block)
                 end
+              elsif kwargs.any?
+                super(**kwargs, &target_block)
+              else
+                super(&target_block)
               end
+              duration = Core::Utils::Time.get_time - start_time
               # The method itself is not part of the stack trace because
               # we are getting the stack trace from outside of the method.
               # Add the method in manually as the top frame.
@@ -246,11 +245,12 @@ module Datadog
               #
               # If the requested file is not in code tracker's registry,
               # or the code tracker does not exist at all,
-              # do not attempt to instrumnet now.
+              # do not attempt to instrument now.
               # The caller should add the line to the list of pending lines
               # to instrument and install the hook when the file in
               # question is loaded (and hopefully, by then code tracking
               # is active, otherwise the line will never be instrumented.)
+              raise_if_probe_in_loaded_features(probe)
               raise Error::DITargetNotDefined, "File not in code tracker registry: #{probe.file}"
             end
           end
@@ -258,6 +258,7 @@ module Datadog
           # Same as previous comment, if untargeted trace points are not
           # explicitly defined, and we do not have code tracking, do not
           # instrument the method.
+          raise_if_probe_in_loaded_features(probe)
           raise Error::DITargetNotDefined, "File not in code tracker registry: #{probe.file}"
         end
 
@@ -303,13 +304,13 @@ module Datadog
             end
           rescue => exc
             raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
-            logger.warn("Unhandled exception in line trace point: #{exc.class}: #{exc}")
+            logger.debug { "di: unhandled exception in line trace point: #{exc.class}: #{exc}" }
             telemetry&.report(exc, description: "Unhandled exception in line trace point")
             # TODO test this path
           end
         rescue => exc
           raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
-          logger.warn("Unhandled exception in line trace point: #{exc.class}: #{exc}")
+          logger.debug { "di: unhandled exception in line trace point: #{exc.class}: #{exc}" }
           telemetry&.report(exc, description: "Unhandled exception in line trace point")
           # TODO test this path
         end
@@ -355,7 +356,7 @@ module Datadog
           hook_line(probe, &block)
         else
           # TODO add test coverage for this path
-          logger.warn("Unknown probe type to hook: #{probe}")
+          logger.debug { "di: unknown probe type to hook: #{probe}" }
         end
       end
 
@@ -366,7 +367,7 @@ module Datadog
           unhook_line(probe)
         else
           # TODO add test coverage for this path
-          logger.warn("Unknown probe type to unhook: #{probe}")
+          logger.debug { "di: unknown probe type to unhook: #{probe}" }
         end
       end
 
@@ -374,6 +375,26 @@ module Datadog
 
       attr_reader :lock
 
+      def raise_if_probe_in_loaded_features(probe)
+        return unless probe.file
+
+        # If the probe file is in the list of loaded files
+        # (as per $LOADED_FEATURES, using either exact or suffix match),
+        # raise an error indicating that
+        # code tracker is missing the loaded file because the file
+        # won't be loaded again (DI only works in production environments
+        # that do not normally reload code).
+        if $LOADED_FEATURES.include?(probe.file)
+          raise Error::DITargetNotInRegistry, "File loaded but is not in code tracker registry: #{probe.file}"
+        end
+        # Ths is an expensive check
+        $LOADED_FEATURES.each do |path|
+          if Utils.path_matches_suffix?(path, probe.file)
+            raise Error::DITargetNotInRegistry, "File matching probe path (#{probe.file}) was loaded and is not in code tracker registry: #{path}"
+          end
+        end
+      end
+
       # TODO test that this resolves qualified names e.g. A::B
       def symbolize_class_name(cls_name)
         Object.const_get(cls_name)

data/lib/datadog/di/{init.rb → preload.rb}

@@ -1,12 +1,10 @@
 # frozen_string_literal: true
 
-# Require 'datadog/di/init' early in the application boot process to
+# Require 'datadog/di/preload' early in the application boot process to
 # enable dynamic instrumentation for third-party libraries used by the
 # application.
 
-require_relative '../tracing'
-require_relative '../tracing/contrib'
-require_relative '../di'
+require_relative 'base'
 
 # Code tracking is required for line probes to work; see the comments
 # on the activate_tracking methods in di.rb for further details.

data/lib/datadog/di/probe_manager.rb

@@ -32,7 +32,7 @@ module Datadog
           install_pending_method_probes(tp.self)
         rescue => exc
           raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
-          logger.warn("Unhandled exception in definition trace point: #{exc.class}: #{exc}")
+          logger.debug { "di: unhandled exception in definition trace point: #{exc.class}: #{exc}" }
           telemetry&.report(exc, description: "Unhandled exception in definition trace point")
           # TODO test this path
         end
@@ -120,7 +120,7 @@ module Datadog
           # In "propagate all exceptions" mode we will try to instrument again.
           raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
 
-          logger.warn("Error processing probe configuration: #{exc.class}: #{exc}")
+          logger.debug { "di: error processing probe configuration: #{exc.class}: #{exc}" }
           telemetry&.report(exc, description: "Error processing probe configuration")
           # TODO report probe as failed to agent since we won't attempt to
           # install it again.
@@ -160,7 +160,7 @@ module Datadog
                 raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
                 # Silence all exceptions?
                 # TODO should we propagate here and rescue upstream?
-                logger.warn("Error removing probe #{probe.id}: #{exc.class}: #{exc}")
+                logger.debug { "di: error removing probe #{probe.id}: #{exc.class}: #{exc}" }
                 telemetry&.report(exc, description: "Error removing probe")
               end
             end
@@ -190,7 +190,7 @@ module Datadog
                 rescue => exc
                   raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
 
-                  logger.warn("Error installing probe after class is defined: #{exc.class}: #{exc}")
+                  logger.debug { "di: error installing probe after class is defined: #{exc.class}: #{exc}" }
                   telemetry&.report(exc, description: "Error installing probe after class is defined")
                 end
               end

data/lib/datadog/di/probe_notification_builder.rb

@@ -32,6 +32,12 @@ module Datadog
           status: 'EMITTING',)
       end
 
+      def build_errored(probe, exc)
+        build_status(probe,
+          message: "Instrumentation for probe #{probe.id} failed: #{exc}",
+          status: 'ERROR',)
+      end
+
       # Duration is in seconds.
       def build_executed(probe,
         trace_point: nil, rv: nil, duration: nil, caller_locations: nil,
@@ -141,8 +147,8 @@ module Datadog
             version: 2,
           },
           # TODO add tests that the trace/span id is correctly propagated
-          "dd.trace_id": Datadog::Tracing.active_trace&.id&.to_s,
-          "dd.span_id": Datadog::Tracing.active_span&.id&.to_s,
+          "dd.trace_id": active_trace&.id&.to_s,
+          "dd.span_id": active_span&.id&.to_s,
           ddsource: 'dd_debugger',
           message: probe.template && evaluate_template(probe.template,
             duration: duration ? duration * 1000 : 0),
@@ -150,6 +156,8 @@ module Datadog
         }
       end
 
+      private
+
       def build_status(probe, message:, status:)
         {
           service: settings.service,
@@ -200,6 +208,18 @@ module Datadog
           map[name] = value
         end
       end
+
+      def active_trace
+        if defined?(Datadog::Tracing)
+          Datadog::Tracing.active_trace
+        end
+      end
+
+      def active_span
+        if defined?(Datadog::Tracing)
+          Datadog::Tracing.active_span
+        end
+      end
     end
   end
 end

data/lib/datadog/di/probe_notifier_worker.rb

@@ -77,7 +77,7 @@ module Datadog
             rescue => exc
               raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
 
-              logger.warn("Error in probe notifier worker: #{exc.class}: #{exc} (at #{exc.backtrace.first})")
+              logger.debug { "di: error in probe notifier worker: #{exc.class}: #{exc} (at #{exc.backtrace.first})" }
               telemetry&.report(exc, description: "Error in probe notifier worker")
             end
             @lock.synchronize do
@@ -183,9 +183,8 @@ module Datadog
         define_method("add_#{event_type}") do |event|
           @lock.synchronize do
             queue = send("#{event_type}_queue")
-            # TODO determine a suitable limit via testing/benchmarking
-            if queue.length > 100
-              logger.warn("#{self.class.name}: dropping #{event_type} because queue is full")
+            if queue.length > settings.dynamic_instrumentation.internal.snapshot_queue_capacity
+              logger.debug { "di: #{self.class.name}: dropping #{event_type} because queue is full" }
             else
               queue << event
             end
@@ -242,7 +241,7 @@ module Datadog
               end
             rescue => exc
               raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
-              logger.warn("failed to send #{event_name}: #{exc.class}: #{exc} (at #{exc.backtrace.first})")
+              logger.debug { "di: failed to send #{event_name}: #{exc.class}: #{exc} (at #{exc.backtrace.first})" }
               # Should we report this error to telemetry? Most likely failure
               # to send is due to a network issue, and trying to send a
               # telemetry message would also fail.
@@ -253,7 +252,7 @@ module Datadog
           # Normally the queue should only be consumed in this method,
           # however if anyone consumes it elsewhere we don't want to block
           # while consuming it here. Rescue ThreadError and return.
-          logger.warn("Unexpected #{event_name} queue underflow - consumed elsewhere?")
+          logger.debug { "di: unexpected #{event_name} queue underflow - consumed elsewhere?" }
           telemetry&.report(exc, description: "Unexpected #{event_name} queue underflow")
         ensure
           @lock.synchronize do