datadog 2.4.0 → 2.6.0

data/lib/datadog/appsec/scope.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require_relative 'processor'
-
 module Datadog
   module AppSec
     # Capture context essential to consistently call processor and report via traces
@@ -22,8 +20,7 @@ module Datadog
         def activate_scope(trace, service_entry_span, processor)
           raise ActiveScopeError, 'another scope is active, nested scopes are not supported' if active_scope
 
-          context = Datadog::AppSec::Processor::Context.new(processor)
-
+          context = processor.new_context
           self.active_scope = new(trace, service_entry_span, context)
         end
 

data/lib/datadog/appsec/utils/trace_operation.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    module Utils
+      # Utility class to to AppSec-specific trace operations
+      class TraceOperation
+        def self.appsec_standalone_reject?(trace)
+          Datadog.configuration.appsec.standalone.enabled &&
+            (trace.nil? || trace.get_tag(Datadog::AppSec::Ext::TAG_DISTRIBUTED_APPSEC_EVENT) != '1')
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/utils.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require_relative 'utils/trace_operation'
+
 module Datadog
   module AppSec
     # Utilities for AppSec

data/lib/datadog/appsec.rb

@@ -4,6 +4,7 @@ require_relative 'appsec/configuration'
 require_relative 'appsec/extensions'
 require_relative 'appsec/scope'
 require_relative 'appsec/ext'
+require_relative 'appsec/utils'
 
 module Datadog
   # Namespace for Datadog AppSec instrumentation

data/lib/datadog/core/configuration/agent_settings_resolver.rb

@@ -232,7 +232,32 @@ module Datadog
         end
 
         def should_use_uds?
-          can_use_uds? && !mixed_http_and_uds?
+          # When we have mixed settings for http/https and uds, we print a warning
+          # and use the uds settings.
+          mixed_http_and_uds
+          can_use_uds?
+        end
+
+        def mixed_http_and_uds
+          return @mixed_http_and_uds if defined?(@mixed_http_and_uds)
+
+          @mixed_http_and_uds = (configured_hostname || configured_port) && can_use_uds?
+          if @mixed_http_and_uds
+            warn_if_configuration_mismatch(
+              [
+                DetectedConfiguration.new(
+                  friendly_name: 'configuration for unix domain socket',
+                  value: parsed_url.to_s,
+                ),
+                DetectedConfiguration.new(
+                  friendly_name: 'configuration of hostname/port for http/https use',
+                  value: "hostname: '#{hostname}', port: '#{port}'",
+                ),
+              ]
+            )
+          end
+
+          @mixed_http_and_uds
         end
 
         def can_use_uds?
@@ -307,30 +332,6 @@ module Datadog
           uri.scheme == 'unix'
         end
 
-        # When we have mixed settings for http/https and uds, we print a warning and ignore the uds settings
-        def mixed_http_and_uds?
-          return @mixed_http_and_uds if defined?(@mixed_http_and_uds)
-
-          @mixed_http_and_uds = (configured_hostname || configured_port) && can_use_uds?
-
-          if @mixed_http_and_uds
-            warn_if_configuration_mismatch(
-              [
-                DetectedConfiguration.new(
-                  friendly_name: 'configuration of hostname/port for http/https use',
-                  value: "hostname: '#{hostname}', port: '#{port}'",
-                ),
-                DetectedConfiguration.new(
-                  friendly_name: 'configuration for unix domain socket',
-                  value: parsed_url.to_s,
-                ),
-              ]
-            )
-          end
-
-          @mixed_http_and_uds
-        end
-
         # Represents a given configuration value and where we got it from
         class DetectedConfiguration
           attr_reader :friendly_name, :value

data/lib/datadog/core/configuration/settings.rb

@@ -514,6 +514,18 @@ module Datadog
                 end
               end
             end
+
+            # Controls if the heap profiler should attempt to clean young objects after GC, rather than just at
+            # serialization time. This lowers memory usage and high percentile latency.
+            #
+            # Only takes effect when used together with `gc_enabled: true` and `experimental_heap_enabled: true`.
+            #
+            # @default false
+            option :heap_clean_after_gc_enabled do |o|
+              o.type :bool
+              o.env 'DD_PROFILING_HEAP_CLEAN_AFTER_GC_ENABLED'
+              o.default false
+            end
           end
 
           # @public_api

data/lib/datadog/core/configuration.rb

@@ -278,9 +278,7 @@ module Datadog
       def handle_interrupt_shutdown!
         logger = Datadog.logger
         shutdown_thread = Thread.new { shutdown! }
-        unless Gem::Version.new(RUBY_VERSION) < Gem::Version.new('2.3')
-          shutdown_thread.name = Datadog::Core::Configuration.name
-        end
+        shutdown_thread.name = Datadog::Core::Configuration.name
 
         print_message_treshold_seconds = 0.2
 

data/lib/datadog/core/crashtracking/component.rb

@@ -66,7 +66,8 @@ module Datadog
         def start
           Utils::AtForkMonkeyPatch.apply!
 
-          start_or_update_on_fork(action: :start)
+          start_or_update_on_fork(action: :start, tags: tags)
+
           ONLY_ONCE.run do
             Utils::AtForkMonkeyPatch.at_fork(:child) do
               # Must NOT reference `self` here, as only the first instance will
@@ -77,8 +78,10 @@ module Datadog
           end
         end
 
-        def update_on_fork
-          start_or_update_on_fork(action: :update_on_fork)
+        def update_on_fork(settings: Datadog.configuration)
+          # Here we pick up the latest settings, so that we pick up any tags that change after forking
+          # such as the pid or runtime-id
+          start_or_update_on_fork(action: :update_on_fork, tags: TagBuilder.call(settings))
         end
 
         def stop
@@ -92,7 +95,7 @@ module Datadog
 
         attr_reader :tags, :agent_base_url, :ld_library_path, :path_to_crashtracking_receiver_binary, :logger
 
-        def start_or_update_on_fork(action:)
+        def start_or_update_on_fork(action:, tags:)
           self.class._native_start_or_update_on_fork(
             action: action,
             agent_base_url: agent_base_url,
@@ -101,7 +104,7 @@ module Datadog
             tags_as_array: tags.to_a,
             upload_timeout_seconds: 1
           )
-          logger.debug("Crash tracking #{action} successfully")
+          logger.debug("Crash tracking action: #{action} successful")
         rescue => e
           logger.error("Failed to #{action} crash tracking: #{e.message}")
         end

data/lib/datadog/core/environment/yjit.rb

@@ -52,6 +52,11 @@ module Datadog
           ::RubyVM::YJIT.runtime_stats[:yjit_alloc_size]
         end
 
+        # Ratio of YJIT-executed instructions
+        def ratio_in_yjit
+          ::RubyVM::YJIT.runtime_stats[:ratio_in_yjit]
+        end
+
         def available?
           defined?(::RubyVM::YJIT) \
             && ::RubyVM::YJIT.enabled? \

data/lib/datadog/core/remote/transport/http.rb

@@ -120,6 +120,11 @@ module Datadog
               # Add container ID, if present.
               container_id = Datadog::Core::Environment::Container.container_id
               headers[Datadog::Core::Transport::Ext::HTTP::HEADER_CONTAINER_ID] = container_id unless container_id.nil?
+              # Sending this header to the agent will disable metrics computation (and billing) on the agent side
+              # by pretending it has already been done on the library side.
+              if Datadog.configuration.appsec.standalone.enabled
+                headers[Datadog::Core::Transport::Ext::HTTP::HEADER_CLIENT_COMPUTED_STATS] = 'yes'
+              end
             end
           end
 

data/lib/datadog/core/remote/worker.rb

@@ -34,7 +34,7 @@ module Datadog
           @starting = true
 
           thread = Thread.new { poll(@interval) }
-          thread.name = self.class.name unless Gem::Version.new(RUBY_VERSION) < Gem::Version.new('2.3')
+          thread.name = self.class.name
           thread.thread_variable_set(:fork_safe, true)
           @thr = thread
 

data/lib/datadog/core/runtime/ext.rb

@@ -31,6 +31,7 @@ module Datadog
           METRIC_YJIT_OBJECT_SHAPE_COUNT = 'runtime.ruby.yjit.object_shape_count'
           METRIC_YJIT_OUTLINED_CODE_SIZE = 'runtime.ruby.yjit.outlined_code_size'
           METRIC_YJIT_YJIT_ALLOC_SIZE = 'runtime.ruby.yjit.yjit_alloc_size'
+          METRIC_YJIT_RATIO_IN_YJIT = 'runtime.ruby.yjit.ratio_in_yjit'
 
           TAG_SERVICE = 'service'
         end

data/lib/datadog/core/runtime/metrics.rb

@@ -181,6 +181,10 @@ module Datadog
                 Core::Runtime::Ext::Metrics::METRIC_YJIT_YJIT_ALLOC_SIZE,
                 Core::Environment::YJIT.yjit_alloc_size
               )
+              gauge_if_not_nil(
+                Core::Runtime::Ext::Metrics::METRIC_YJIT_RATIO_IN_YJIT,
+                Core::Environment::YJIT.ratio_in_yjit
+              )
             end
           end
         end

data/lib/datadog/core/semaphore.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Core
+    # Semaphore pattern implementation, as described in documentation for
+    # ConditionVariable.
+    #
+    # @api private
+    class Semaphore
+      def initialize
+        @wake_lock = Mutex.new
+        @wake = ConditionVariable.new
+      end
+
+      def signal
+        wake_lock.synchronize do
+          wake.signal
+        end
+      end
+
+      def wait(timeout = nil)
+        wake_lock.synchronize do
+          # steep specifies that the second argument to wait is of type
+          # ::Time::_Timeout which for some reason is not Numeric and is not
+          # castable from Numeric.
+          wake.wait(wake_lock, timeout) # steep:ignore
+        end
+      end
+
+      private
+
+      attr_reader :wake_lock, :wake
+    end
+  end
+end

data/lib/datadog/core/telemetry/logging.rb

@@ -31,17 +31,17 @@ module Datadog
             return unless backtrace
             return if backtrace.empty?
 
-            stack_trace = +''
-            backtrace.each do |line|
-              stack_trace << if line.start_with?(GEM_ROOT)
-                               line[GEM_ROOT.length..-1] || ''
-                             else
-                               'REDACTED'
-                             end
-              stack_trace << ','
-            end
+            # vendored deps
+            vendored_deps = Gem.path.any? { |p| p.start_with?(GEM_ROOT) }
 
-            stack_trace.chomp(',')
+            backtrace.map do |line|
+              if !vendored_deps && line.start_with?(GEM_ROOT) ||
+                  vendored_deps && line.start_with?(GEM_ROOT) && Gem.path.none? { |p| line.start_with?(p) }
+                line[GEM_ROOT.length..-1] || ''
+              else
+                'REDACTED'
+              end
+            end.join(',')
           end
         end
 

data/lib/datadog/core/transport/ext.rb

@@ -16,6 +16,7 @@ module Datadog
           #
           # Setting this header to any non-empty value enables this feature.
           HEADER_CLIENT_COMPUTED_TOP_LEVEL = 'Datadog-Client-Computed-Top-Level'
+          HEADER_CLIENT_COMPUTED_STATS = 'Datadog-Client-Computed-Stats'
           HEADER_META_LANG = 'Datadog-Meta-Lang'
           HEADER_META_LANG_VERSION = 'Datadog-Meta-Lang-Version'
           HEADER_META_LANG_INTERPRETER = 'Datadog-Meta-Lang-Interpreter'

data/lib/datadog/core/workers/async.rb

@@ -148,7 +148,7 @@ module Datadog
               end
               # rubocop:enable Lint/RescueException
             end
-            @worker.name = self.class.name unless Gem::Version.new(RUBY_VERSION) < Gem::Version.new('2.3')
+            @worker.name = self.class.name
             @worker.thread_variable_set(:fork_safe, true)
 
             nil

data/lib/datadog/di/code_tracker.rb

@@ -109,25 +109,15 @@ module Datadog
       # to be an absolute path), only the exactly matching path is returned.
       # Otherwise all known paths that end in the suffix are returned.
       # If no paths match, an empty array is returned.
-      def iseqs_for_path(suffix)
+      def iseqs_for_path_suffix(suffix)
         registry_lock.synchronize do
           exact = registry[suffix]
           return [exact] if exact
 
           inexact = []
           registry.each do |path, iseq|
-            # Exact match is not possible here, meaning any matching path
-            # has to be longer than the suffix. Require full component matches,
-            # meaning either the first character of the suffix is a slash
-            # or the previous character in the path is a slash.
-            # For now only check for forward slashes for Unix-like OSes;
-            # backslash is a legitimate character of a file name in Unix
-            # therefore simply permitting forward or back slash is not
-            # sufficient, we need to perform an OS check to know which
-            # path separator to use.
-            if path.length > suffix.length && path.end_with?(suffix)
-              previous_char = path[path.length - suffix.length - 1]
-              inexact << iseq if previous_char == "/" || suffix[0] == "/"
+            if Utils.path_matches_suffix?(path, suffix)
+              inexact << iseq
             end
           end
           inexact
@@ -150,6 +140,14 @@ module Datadog
           # reinstated in the future.
           @compiled_trace_point = nil
         end
+        clear
+      end
+
+      # Clears the stored mapping from paths to compiled code.
+      #
+      # This method should normally never be called. It is meant to be
+      # used only by the test suite.
+      def clear
         registry_lock.synchronize do
           registry.clear
         end

data/lib/datadog/di/instrumenter.rb

@@ -0,0 +1,301 @@
+# frozen_string_literal: true
+
+# rubocop:disable Lint/AssignmentInCondition
+
+require 'benchmark'
+
+module Datadog
+  module DI
+    # Arranges to invoke a callback when a particular Ruby method or
+    # line of code is executed.
+    #
+    # Method instrumentation is accomplished via module prepending.
+    # Unlike the alias_method_chain pattern, module prepending permits
+    # removing instrumentation with no virtually performance side-effects
+    # (the target class retains an empty included module, but no additional
+    # code is executed as part of target method).
+    #
+    # Method hooking works with explicitly defined methods and "virtual"
+    # methods defined via method_missing.
+    #
+    # Line instrumentation is normally accomplished with a targeted line
+    # trace point. This requires MRI and at least Ruby 2.6.
+    # For testing purposes, it is also possible to use untargeted trace
+    # points, but they have a huge performance penalty and should generally
+    # not be used in production.
+    #
+    # Targeted line trace points require tracking of loaded code; see
+    # the CodeTracker class for more details.
+    #
+    # Instrumentation state (i.e., the module or trace point used for
+    # instrumentation) is stored in the Probe instance. Thus, Instrumenter
+    # mutates attributes of Probes it is asked to install or remove.
+    # A previous version of the code attempted to maintain the instrumentation
+    # state within Instrumenter but this was very messy and hard to
+    # guarantee correctness of. With the state stored in Probes, it is
+    # straightforward to determine if a Probe has been successfully instrumented,
+    # and thus requires cleanup, and to properly clean it up.
+    #
+    # Note that the upstream code is responsible for generally storing Probes.
+    # This is normally accomplished by ProbeManager. ProbeManager stores all
+    # known probes, instrumented or not, and is responsible for calling
+    # +unhook+ of Instrumenter to clean up instrumentation when a user
+    # deletes a probe in UI or when DI is shut down.
+    #
+    # Given the need to store state, and also that there are several Probe
+    # attributes that affect how instrumentation is set up and that must be
+    # consulted very early in the callback invocation (e.g., to perform
+    # rate limiting correctly), Instrumenter takes Probe instances as
+    # arguments rather than e.g. file + line number or class + method name.
+    # As a result, Instrumenter is rather coupled to DI the product and is
+    # not trivially usable as a general-purpose Ruby instrumentation tool
+    # (however, Probe instances can be replaced by OpenStruct instances
+    # providing the same interface with not much effort).
+    #
+    # @api private
+    class Instrumenter
+      def initialize(settings, serializer, logger, code_tracker: nil)
+        @settings = settings
+        @serializer = serializer
+        @logger = logger
+        @code_tracker = code_tracker
+
+        @lock = Mutex.new
+      end
+
+      attr_reader :settings
+      attr_reader :serializer
+      attr_reader :logger
+      attr_reader :code_tracker
+
+      # This is a substitute for Thread::Backtrace::Location
+      # which does not have a public constructor.
+      # Used for the fabricated stack frame for the method itself
+      # for method probes (which use Module#prepend and thus aren't called
+      # from the method but from outside of the method).
+      Location = Struct.new(:path, :lineno, :label)
+
+      def hook_method(probe, &block)
+        unless block
+          raise ArgumentError, 'block is required'
+        end
+
+        lock.synchronize do
+          if probe.instrumentation_module
+            # Already instrumented, warn?
+            return
+          end
+        end
+
+        cls = symbolize_class_name(probe.type_name)
+        serializer = self.serializer
+        method_name = probe.method_name
+        target_method = cls.instance_method(method_name)
+        loc = target_method.source_location
+        rate_limiter = probe.rate_limiter
+
+        mod = Module.new do
+          define_method(method_name) do |*args, **kwargs| # steep:ignore
+            if rate_limiter.nil? || rate_limiter.allow?
+              # Arguments may be mutated by the method, therefore
+              # they need to be serialized prior to method invocation.
+              entry_args = if probe.capture_snapshot?
+                serializer.serialize_args(args, kwargs)
+              end
+              rv = nil
+              duration = Benchmark.realtime do # steep:ignore
+                rv = super(*args, **kwargs)
+              end
+              # The method itself is not part of the stack trace because
+              # we are getting the stack trace from outside of the method.
+              # Add the method in manually as the top frame.
+              method_frame = Location.new(loc.first, loc.last, method_name)
+              caller_locs = [method_frame] + caller_locations # steep:ignore
+              # TODO capture arguments at exit
+              # & is to stop steep complaints, block is always present here.
+              block&.call(probe: probe, rv: rv, duration: duration, caller_locations: caller_locs,
+                serialized_entry_args: entry_args)
+              rv
+            else
+              super(*args, **kwargs)
+            end
+          end
+        end
+
+        lock.synchronize do
+          if probe.instrumentation_module
+            # Already instrumented from another thread
+            return
+          end
+
+          probe.instrumentation_module = mod
+          cls.send(:prepend, mod)
+        end
+      end
+
+      def unhook_method(probe)
+        # Ruby does not permit removing modules from classes.
+        # We can, however, remove method definitions from modules.
+        # After this the modules remain in memory and stay included
+        # in the classes but are empty (have no methods).
+        lock.synchronize do
+          if mod = probe.instrumentation_module
+            mod.send(:remove_method, probe.method_name)
+            probe.instrumentation_module = nil
+          end
+        end
+      end
+
+      # Instruments a particluar line in a source file.
+      # Note that this method only works for physical files,
+      # not for eval'd code, unless the eval'd code is associated with
+      # a file name and client invokes this method with the correct
+      # file name for the eval'd code.
+      def hook_line(probe, &block)
+        unless block
+          raise ArgumentError, 'No block given to hook_line'
+        end
+
+        lock.synchronize do
+          if probe.instrumentation_trace_point
+            # Already instrumented, warn?
+            return
+          end
+        end
+
+        line_no = probe.line_no!
+        rate_limiter = probe.rate_limiter
+
+        # Memoize the value to ensure this method always uses the same
+        # value for the setting.
+        # Normally none of the settings should change, but in the test suite
+        # we use mock objects and the methods may be mocked with
+        # individual invocations, yielding different return values on
+        # different calls to the same method.
+        permit_untargeted_trace_points = settings.dynamic_instrumentation.untargeted_trace_points
+
+        iseq = nil
+        if code_tracker
+          iseq = code_tracker.iseqs_for_path_suffix(probe.file).first # steep:ignore
+          unless iseq
+            if permit_untargeted_trace_points
+              # Continue withoout targeting the trace point.
+              # This is going to cause a serious performance penalty for
+              # the entire file containing the line to be instrumented.
+            else
+              # Do not use untargeted trace points unless they have been
+              # explicitly requested by the user, since they cause a
+              # serious performance penalty.
+              #
+              # If the requested file is not in code tracker's registry,
+              # or the code tracker does not exist at all,
+              # do not attempt to instrumnet now.
+              # The caller should add the line to the list of pending lines
+              # to instrument and install the hook when the file in
+              # question is loaded (and hopefully, by then code tracking
+              # is active, otherwise the line will never be instrumented.)
+              raise Error::DITargetNotDefined, "File not in code tracker registry: #{probe.file}"
+            end
+          end
+        elsif !permit_untargeted_trace_points
+          # Same as previous comment, if untargeted trace points are not
+          # explicitly defined, and we do not have code tracking, do not
+          # instrument the method.
+          raise Error::DITargetNotDefined, "File not in code tracker registry: #{probe.file}"
+        end
+
+        # If trace point is not targeted, we only need one trace point per file.
+        # Creating a trace point for each probe does work but the performance
+        # penalty will be taken for each trace point defined in the file.
+        # Since untargeted trace points are only (currently) used internally
+        # for benchmarking, and shouldn't be used in customer applications,
+        # we always create a trace point here to reduce complexity.
+        #
+        # For targeted trace points, if multiple probes target the same
+        # file and line, we also only need one trace point, but since the
+        # overhead of targeted trace points is minimal, don't worry about
+        # this optimization just yet and create a trace point for each probe.
+
+        tp = TracePoint.new(:line) do |tp|
+          # If trace point is not targeted, we must verify that the invocation
+          # is the file & line that we want, because untargeted trace points
+          # are invoked for *each* line of Ruby executed.
+          if iseq || tp.lineno == probe.line_no && probe.file_matches?(tp.path)
+            if rate_limiter.nil? || rate_limiter.allow?
+              # & is to stop steep complaints, block is always present here.
+              block&.call(probe: probe, trace_point: tp, caller_locations: caller_locations)
+            end
+          end
+        rescue => exc
+          raise if settings.dynamic_instrumentation.propagate_all_exceptions
+          logger.warn("Unhandled exception in line trace point: #{exc.class}: #{exc}")
+          # TODO test this path
+        end
+
+        # TODO internal check - remove or use a proper exception
+        if !iseq && !permit_untargeted_trace_points
+          raise "Trying to use an untargeted trace point when user did not permit it"
+        end
+
+        lock.synchronize do
+          if probe.instrumentation_trace_point
+            # Already instrumented in another thread, warn?
+            return
+          end
+
+          probe.instrumentation_trace_point = tp
+
+          if iseq
+            tp.enable(target: iseq, target_line: line_no)
+          else
+            tp.enable
+          end
+        end
+      end
+
+      def unhook_line(probe)
+        lock.synchronize do
+          if tp = probe.instrumentation_trace_point
+            tp.disable
+            probe.instrumentation_trace_point = nil
+          end
+        end
+      end
+
+      def hook(probe, &block)
+        if probe.method?
+          hook_method(probe, &block)
+        elsif probe.line?
+          hook_line(probe, &block)
+        else
+          # TODO add test coverage for this path
+          logger.warn("Unknown probe type to hook: #{probe}")
+        end
+      end
+
+      def unhook(probe)
+        if probe.method?
+          unhook_method(probe)
+        elsif probe.line?
+          unhook_line(probe)
+        else
+          # TODO add test coverage for this path
+          logger.warn("Unknown probe type to unhook: #{probe}")
+        end
+      end
+
+      private
+
+      attr_reader :lock
+
+      # TODO test that this resolves qualified names e.g. A::B
+      def symbolize_class_name(cls_name)
+        Object.const_get(cls_name)
+      rescue NameError => exc
+        raise Error::DITargetNotDefined, "Class not defined: #{cls_name}: #{exc.class}: #{exc}"
+      end
+    end
+  end
+end
+
+# rubocop:enable Lint/AssignmentInCondition