datadog 2.3.0 → 2.4.0

data/lib/datadog/core/telemetry/component.rb

@@ -5,6 +5,7 @@ require_relative 'event'
 require_relative 'http/transport'
 require_relative 'metrics_manager'
 require_relative 'worker'
+require_relative 'logging'
 
 require_relative '../configuration/ext'
 require_relative '../utils/forking'
@@ -17,6 +18,7 @@ module Datadog
         attr_reader :enabled
 
         include Core::Utils::Forking
+        include Telemetry::Logging
 
         def self.build(settings, agent_settings, logger)
           enabled = settings.telemetry.enabled

data/lib/datadog/core/telemetry/event.rb

@@ -1,5 +1,8 @@
 # frozen_string_literal: true
 
+require_relative '../utils/forking'
+require_relative '../utils/sequence'
+
 module Datadog
   module Core
     module Telemetry
@@ -338,7 +341,6 @@ module Datadog
         class Log < Base
           LEVELS = {
             error: 'ERROR',
-            debug: 'DEBUG',
             warn: 'WARN',
           }.freeze
 
@@ -346,19 +348,22 @@ module Datadog
             'logs'
           end
 
-          def initialize(message:, level:)
+          def initialize(message:, level:, stack_trace: nil)
             super()
             @message = message
+            @stack_trace = stack_trace
             @level = LEVELS.fetch(level) { |k| raise ArgumentError, "Invalid log level :#{k}" }
           end
 
           def payload
             {
-              logs: [{
-                message: @message,
-                level: @level,
-                # More optional fields to be added here...
-              }]
+              logs: [
+                {
+                  message: @message,
+                  level: @level,
+                  stack_trace: @stack_trace,
+                }.compact
+              ]
             }
           end
         end

data/lib/datadog/core/telemetry/logger.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Core
+    module Telemetry
+      # === INTERNAL USAGE ONLY ===
+      #
+      # Report telemetry logs via delegating to the telemetry component instance via mutex.
+      #
+      # IMPORTANT: Invoking this method during the lifecycle of component initialization will
+      # be no-op instead.
+      #
+      # For developer using this module:
+      #   read: lib/datadog/core/telemetry/logging.rb
+      module Logger
+        class << self
+          def report(exception, level: :error, description: nil)
+            instance&.report(exception, level: level, description: description)
+          end
+
+          def error(description)
+            instance&.error(description)
+          end
+
+          private
+
+          def instance
+            # Component initialization uses a mutex to avoid having concurrent initialization.
+            # Trying to access the telemetry component during initialization (specifically:
+            # from the thread that's actually doing the initialization) would cause a deadlock,
+            # since accessing the components would try to recursively lock the mutex.
+            #
+            # To work around this, we use allow_initialization: false to avoid triggering this issue.
+            #
+            # The downside is: this leaves us unable to report telemetry during component initialization.
+            components = Datadog.send(:components, allow_initialization: false)
+
+            if components && components.telemetry
+              components.telemetry
+            else
+              Datadog.logger.warn(
+                'Failed to send telemetry before components initialization or within components lifecycle'
+              )
+              nil
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/telemetry/logging.rb

@@ -2,32 +2,68 @@
 
 require_relative 'event'
 
+require 'pathname'
+
 module Datadog
   module Core
     module Telemetry
-      # Logging interface for sending telemetry logs.
+      # === INTERNAL USAGE ONLY ===
+      #
+      # Logging interface for sending telemetry logs... so we can fix them.
+      #
+      # For developer using this module:
+      # - MUST NOT provide any sensitive information (PII)
+      # - SHOULD reduce the data cardinality for batching/aggregation
+      #
+      # Before using it, ask yourself:
+      # - Do we need to know about this (ie. internal error or client error)?
+      # - How severe/critical is this error? (ie. error, warning, fatal)
+      # - What information needed to make it actionable?
       #
-      # Reporting internal error so that we can fix them.
-      # IMPORTANT: Make sure to not log any sensitive information.
       module Logging
-        module_function
+        # Extract datadog stack trace from the exception
+        module DatadogStackTrace
+          GEM_ROOT = Pathname.new("#{__dir__}/../../../..").cleanpath.to_s
+
+          def self.from(exception)
+            backtrace = exception.backtrace
+
+            return unless backtrace
+            return if backtrace.empty?
+
+            stack_trace = +''
+            backtrace.each do |line|
+              stack_trace << if line.start_with?(GEM_ROOT)
+                               line[GEM_ROOT.length..-1] || ''
+                             else
+                               'REDACTED'
+                             end
+              stack_trace << ','
+            end
 
-        def report(exception, level:)
+            stack_trace.chomp(',')
+          end
+        end
+
+        def report(exception, level: :error, description: nil)
           # Annoymous exceptions to be logged as <Class:0x00007f8b1c0b3b40>
-          message = exception.class.name || exception.class.inspect
+          message = +''
+          message << (exception.class.name || exception.class.inspect)
+          message << ':' << description if description
 
           event = Event::Log.new(
             message: message,
-            level: level
+            level: level,
+            stack_trace: DatadogStackTrace.from(exception)
           )
 
-          if (telemetry = Datadog.send(:components).telemetry)
-            telemetry.log!(event)
-          else
-            Datadog.logger.debug do
-              "Attempting to send telemetry log when telemetry component is not ready: #{message}"
-            end
-          end
+          log!(event)
+        end
+
+        def error(description)
+          event = Event::Log.new(message: description, level: :error)
+
+          log!(event)
         end
       end
     end

data/lib/datadog/core/telemetry/request.rb

@@ -31,6 +31,18 @@ module Datadog
 
           def application
             config = Datadog.configuration
+
+            tracer_version = Core::Environment::Identity.gem_datadog_version_semver2
+
+            # We need some to distinguish datadog-ci gem versions
+            # when examining telemetry metrics emitted from the datadog-ci gem.
+            #
+            # This code checks that Datadog::CI is loaded and ci mode is enabled and adds
+            # "-ci-X.Y.Z" suffix to the tracer version.
+            if defined?(::Datadog::CI::VERSION) && config.respond_to?(:ci) && config.ci.enabled
+              tracer_version = "#{tracer_version}-ci-#{::Datadog::CI::VERSION::STRING}"
+            end
+
             {
               env: config.env,
               language_name: Core::Environment::Ext::LANG,
@@ -39,7 +51,7 @@ module Datadog
               runtime_version: Core::Environment::Ext::ENGINE_VERSION,
               service_name: config.service,
               service_version: config.version,
-              tracer_version: Core::Environment::Identity.gem_datadog_version_semver2
+              tracer_version: tracer_version
             }
           end
 

data/lib/datadog/core/utils/time.rb

@@ -34,6 +34,18 @@ module Datadog
           define_singleton_method(:now, &block)
         end
 
+        # Overrides the implementation of `#get_time
+        # with the provided callable.
+        #
+        # Overriding the method `#get_time` instead of
+        # indirectly calling `block` removes
+        # one level of method call overhead.
+        #
+        # @param block [Proc] block that accepts unit and returns timestamp in the requested unit
+        def get_time_provider=(block)
+          define_singleton_method(:get_time, &block)
+        end
+
         def measure(unit = :float_second)
           before = get_time(unit)
           yield

data/lib/datadog/di/code_tracker.rb

@@ -0,0 +1,168 @@
+# frozen_string_literal: true
+
+module Datadog
+  module DI
+    # Tracks loaded Ruby code by source file and maintains a map from
+    # source file to the loaded code (instruction sequences).
+    # Also arranges for code in the loaded files to be instrumented by
+    # line probes that have already been received by the library.
+    #
+    # The loaded code is used to target line trace points when installing
+    # line probes which dramatically improves efficiency of line trace points.
+    #
+    # Note that, since most files will only be loaded one time (via the
+    # "require" mechanism), the code tracker needs to be global and not be
+    # recreated when the DI component is created.
+    #
+    # @api private
+    class CodeTracker
+      def initialize
+        @registry = {}
+        @trace_point_lock = Mutex.new
+        @registry_lock = Mutex.new
+        @compiled_trace_point = nil
+      end
+
+      # Starts tracking loaded code.
+      #
+      # This method should generally be called early in application boot
+      # process, because any code loaded before code tracking is enabled
+      # will not be instrumentable via line probes.
+      #
+      # Normally tracking should remain active for the lifetime of the
+      # process and would not be ever stopped.
+      def start
+        trace_point_lock.synchronize do
+          # If this code tracker is already running, we can do nothing or
+          # restart it (by disabling the trace point and recreating it).
+          # It is likely that some applications will attempt to activate
+          # DI more than once where the intention is to just activate DI;
+          # do not break such applications by clearing out the registry.
+          # For now, until there is a use case for recreating the trace point,
+          # do nothing if the code tracker has already started.
+          return if @compiled_trace_point
+
+          # Note: .trace enables the trace point.
+          @compiled_trace_point = TracePoint.trace(:script_compiled) do |tp|
+            # Useful attributes of the trace point object here:
+            # .instruction_sequence
+            # .instruction_sequence.path (either absolute file path for
+            #   loaded or required code, or for eval'd code, if filename
+            #   is specified as argument to eval, then this is the provided
+            #   filename, otherwise this is a synthesized
+            #   "(eval at <definition-file>:<line>)" string)
+            # .instruction_sequence.absolute_path (absolute file path when
+            #   load or require are used to load code, nil for eval'd code
+            #   regardless of whether filename was specified as an argument
+            #   to eval on ruby 3.1+, same as path for eval'd code on ruby 3.0
+            #   and lower)
+            # .method_id
+            # .path (refers to the code location that called the require/eval/etc.,
+            #   not where the loaded code is; use .path on the instruction sequence
+            #   to obtain the location of the compiled code)
+            # .eval_script
+            #
+            # For now just map the path to the instruction sequence.
+            path = tp.instruction_sequence.absolute_path
+            # Do not store mapping for eval'd code, since there is no way
+            # to target such code from dynamic instrumentation UI.
+            # eval'd code always sets tp.eval_script.
+            # When tp.eval_script is nil, code is either 'load'ed or 'require'd.
+            # steep, of course, complains about indexing with +path+
+            # without checking that it is not nil, so here, maybe there is
+            # some situation where path would in fact be nil and
+            # steep would end up saving the day.
+            if path && !tp.eval_script
+              registry_lock.synchronize do
+                registry[path] = tp.instruction_sequence
+              end
+            end
+          end
+        end
+      end
+
+      # Returns whether this code tracker has been activated and is
+      # tracking.
+      def active?
+        trace_point_lock.synchronize do
+          !!@compiled_trace_point
+        end
+      end
+
+      # Returns an array of RubVM::InstructionSequence (i.e. the compiled code)
+      # for the provided path.
+      #
+      # The argument can be a full path to a Ruby source code file or a
+      # suffix (basename + one or more directories preceding the basename).
+      # The idea with suffix matches is that file paths are likely to
+      # be different between development and production environments and
+      # the source control system uses relative paths and doesn't have
+      # absolute paths at all.
+      #
+      # Suffix matches are not guaranteed to be correct, meaning there may
+      # be multiple files with the same basename and they may all match a
+      # given suffix. In such cases, this method will return all matching
+      # paths (and all of these paths will be attempted to be instrumented
+      # by upstream code).
+      #
+      # If the suffix matches one of the paths completely (which requires it
+      # to be an absolute path), only the exactly matching path is returned.
+      # Otherwise all known paths that end in the suffix are returned.
+      # If no paths match, an empty array is returned.
+      def iseqs_for_path(suffix)
+        registry_lock.synchronize do
+          exact = registry[suffix]
+          return [exact] if exact
+
+          inexact = []
+          registry.each do |path, iseq|
+            # Exact match is not possible here, meaning any matching path
+            # has to be longer than the suffix. Require full component matches,
+            # meaning either the first character of the suffix is a slash
+            # or the previous character in the path is a slash.
+            # For now only check for forward slashes for Unix-like OSes;
+            # backslash is a legitimate character of a file name in Unix
+            # therefore simply permitting forward or back slash is not
+            # sufficient, we need to perform an OS check to know which
+            # path separator to use.
+            if path.length > suffix.length && path.end_with?(suffix)
+              previous_char = path[path.length - suffix.length - 1]
+              inexact << iseq if previous_char == "/" || suffix[0] == "/"
+            end
+          end
+          inexact
+        end
+      end
+
+      # Stops tracking code that is being loaded.
+      #
+      # This method should ordinarily never be called - if a file is loaded
+      # when code tracking is not active, this file will not be instrumentable
+      # by line probes.
+      #
+      # This method is intended for test suite use only, where multiple
+      # code tracker instances are created, to fully clean up the old instances.
+      def stop
+        # Permit multiple stop calls.
+        trace_point_lock.synchronize do
+          @compiled_trace_point&.disable
+          # Clear the instance variable so that the trace point may be
+          # reinstated in the future.
+          @compiled_trace_point = nil
+        end
+        registry_lock.synchronize do
+          registry.clear
+        end
+      end
+
+      private
+
+      # Mapping from paths of loaded files to RubyVM::InstructionSequence
+      # objects representing compiled code of those files.
+      attr_reader :registry
+
+      attr_reader :trace_point_lock
+      attr_reader :registry_lock
+    end
+  end
+end

data/lib/datadog/di/configuration/settings.rb

@@ -0,0 +1,163 @@
+# frozen_string_literal: true
+
+module Datadog
+  module DI
+    module Configuration
+      # Settings
+      module Settings
+        def self.extended(base)
+          base = base.singleton_class unless base.is_a?(Class)
+          add_settings!(base)
+        end
+
+        def self.add_settings!(base)
+          base.class_eval do
+            # The setting has "internal" prefix to prevent it from being
+            # prematurely turned on by customers.
+            settings :dynamic_instrumentation do
+              option :enabled do |o|
+                o.type :bool
+                # The environment variable has an "internal" prefix so that
+                # any customers that have the "proper" environment variable
+                # turned on (i.e. DD_DYNAMIC_INSTRUMENTATION_ENABLED)
+                # do not enable Ruby DI until the latter is ready for
+                # customer testing.
+                o.env "DD_DYNAMIC_INSTRUMENTATION_ENABLED"
+                o.default false
+              end
+
+              # This option instructs dynamic instrumentation to use
+              # untargeted trace points when installing line probes and
+              # code tracking is not active.
+              # WARNING: untargeted trace points carry a massive performance
+              # penalty for the entire file in which a line probe is placed.
+              #
+              # If this option is set to false, which is the default,
+              # dynamic instrumentation will add probes that reference
+              # unknown files to the list of pending probes, and when
+              # the respective files are loaded, the line probes will be
+              # installed using targeted trace points. If the file in
+              # question is already loaded when the probe is received
+              # (for example, it is in a third-party library loaded during
+              # application boot), and code tracking was not active when
+              # the file was loaded, such files will not be instrumentable
+              # via line probes.
+              #
+              # If this option is set to true
+              #
+              # activated, DI will in
+              # activated or because the files being targeted have beenIf true and code tracking is not enabled, dynamic instrumentation
+              # will use untargeted trace points.
+              # If false and code tracking is not enabled, dynamic
+              # instrumentation will not instrument any files loaded
+              # WARNING: these trace points will greatly degrade performance
+              # of all code in the instrumented files.
+              option :untargeted_trace_points do |o|
+                o.type :bool
+                o.default false
+              end
+
+              # If true, all of the catch-all rescue blocks in DI
+              # will propagate the exceptions onward.
+              # WARNING: for internal Datadog use only - this will break
+              # the DI product and potentially the library in general in
+              # a multitude of ways, cause resource leakage, permanent
+              # performance decreases, etc.
+              option :propagate_all_exceptions do |o|
+                o.type :bool
+                o.default false
+              end
+
+              # An array of variable and key names to redact in addition to
+              # the built-in list of identifiers.
+              #
+              # The names will be normalized by removing the following
+              # symbols: _, -, @, $, and then matched to the complete
+              # variable or key name while ignoring the case.
+              # For example, specifying pass_word will match password and
+              # PASSWORD, and specifying PASSWORD will match pass_word.
+              # Note that, while the at sign (@) is used in Ruby to refer
+              # to instance variables, it does not have any significance
+              # for this setting (and is removed before matching identifiers).
+              option :redacted_identifiers do |o|
+                o.env "DD_DYNAMIC_INSTRUMENTATION_REDACTED_IDENTIFIERS"
+                o.env_parser do |value|
+                  value&.split(",")&.map(&:strip)
+                end
+
+                o.type :array
+                o.default []
+              end
+
+              # An array of class names, values of which will be redacted from
+              # dynamic instrumentation snapshots. Example: FooClass.
+              # If a name is suffixed by '*', it becomes a wildcard and
+              # instances of any class whose name begins with the specified
+              # prefix will be redacted (example: Foo*).
+              #
+              # The names must all be fully-qualified, if any prefix of a
+              # class name is configured to be redacted, the value will be
+              # subject to redaction. For example, if Foo* is in the
+              # redacted class name list, instances of Foo, FooBar,
+              # Foo::Bar are all subject to redaction, but Bar::Foo will
+              # not be subject to redaction.
+              #
+              # Leading double-colon is permitted but has no effect,
+              # because the names are always considered to be fully-qualified.
+              # For example, adding ::Foo to the list will redact instances
+              # of Foo.
+              #
+              # Trailing colons should not be used because they will trigger
+              # exact match behavior but Ruby class names do not have
+              # trailing colons. For example, Foo:: will not cause anything
+              # to be redacted. Use Foo::* to redact all classes under
+              # the Foo module.
+              option :redacted_type_names do |o|
+                o.env "DD_DYNAMIC_INSTRUMENTATION_REDACTED_TYPES"
+                o.env_parser do |value|
+                  value&.split(",")&.map(&:strip)
+                end
+
+                o.type :array
+                o.default []
+              end
+
+              # Maximum number of object or collection traversals that
+              # will be permitted when serializing captured values.
+              option :max_capture_depth do |o|
+                o.type :int
+                o.default 3
+              end
+
+              # Maximum number of collection (Array and Hash) elements
+              # that will be captured. Arrays and hashes that have more
+              # elements will be truncated to this many elements.
+              option :max_capture_collection_size do |o|
+                o.type :int
+                o.default 100
+              end
+
+              # Strings longer than this length will be truncated to this
+              # length in dynamic instrumentation snapshots.
+              #
+              # Note that while all values are stringified during
+              # serialization, only values which are originally instances
+              # of the String class are subject to this length limit.
+              option :max_capture_string_length do |o|
+                o.type :int
+                o.default 255
+              end
+
+              # Maximim number of attributes that will be captured for
+              # a single non-primitive value.
+              option :max_capture_attribute_count do |o|
+                o.type :int
+                o.default 20
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/di/configuration.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Datadog
+  module DI
+    # Configuration for DI
+    module Configuration
+    end
+  end
+end
+
+require_relative "configuration/settings"

data/lib/datadog/di/error.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Datadog
+  module DI
+    # Base class for Dynamic Instrumentation exceptions.
+    #
+    # None of these exceptions should be propagated out of DI to user
+    # applications, therefore these exceptions are not considered to be
+    # part of the public API of the library.
+    #
+    # @api private
+    class Error < StandardError
+      # Probe does not contain a line number (i.e., is not a line probe).
+      class MissingLineNumber < Error
+      end
+
+      # Failed to communicate to the local Datadog agent (e.g. to send
+      # probe status or a snapshot).
+      class AgentCommunicationError < Error
+      end
+
+      # Attempting to instrument a method or file which does not exist.
+      #
+      # This could be due to the code that is referenced in the probe
+      # having not been loaded yet, or due to the probe referencing code
+      # that does not in fact exist anywhere (e.g. due to a misspelling).
+      class DITargetNotDefined < Error
+      end
+    end
+  end
+end

data/lib/datadog/di/extensions.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+require_relative "../core/configuration"
+require_relative "configuration"
+
+module Datadog
+  module DI
+    # Extends Datadog tracing with DI features
+    module Extensions
+      # Inject DI into global objects.
+      def self.activate!
+        Core::Configuration::Settings.extend(Configuration::Settings)
+      end
+    end
+  end
+end