datadog 2.28.0 → 2.41.0

data/lib/datadog/di/configuration.rb

@@ -4,8 +4,247 @@ module Datadog
   module DI
     # Configuration for DI
     module Configuration
+      # Settings
+      module Settings
+        def self.extended(base)
+          base = base.singleton_class unless base.is_a?(Class)
+          add_settings!(base)
+        end
+
+        def self.add_settings!(base)
+          base.class_eval do
+            settings :dynamic_instrumentation do
+              option :enabled do |o|
+                o.type :bool
+                # The environment variable has an "internal" prefix so that
+                # any customers that have the "proper" environment variable
+                # turned on (i.e. DD_DYNAMIC_INSTRUMENTATION_ENABLED)
+                # do not enable Ruby DI until the latter is ready for
+                # customer testing.
+                o.env "DD_DYNAMIC_INSTRUMENTATION_ENABLED"
+                o.default false
+              end
+
+              # An array of variable and key names to redact in addition to
+              # the built-in list of identifiers.
+              #
+              # The names will be normalized by removing the following
+              # symbols: _, -, @, $, and then matched to the complete
+              # variable or key name while ignoring the case.
+              # For example, specifying pass_word will match password and
+              # PASSWORD, and specifying PASSWORD will match pass_word.
+              # Note that, while the at sign (@) is used in Ruby to refer
+              # to instance variables, it does not have any significance
+              # for this setting (and is removed before matching identifiers).
+              option :redacted_identifiers do |o|
+                o.env "DD_DYNAMIC_INSTRUMENTATION_REDACTED_IDENTIFIERS"
+                o.env_parser do |value|
+                  value&.split(",")&.map(&:strip)
+                end
+
+                o.type :array
+                o.default []
+              end
+
+              # An array of variable and key names to exclude from the
+              # built-in redaction list.
+              #
+              # This allows users to capture values of variables that would
+              # otherwise be redacted by the default identifier list.
+              # For example, if an application has a "session" variable
+              # that does not contain sensitive data, "session" can be added
+              # to this list to exclude it from redaction.
+              #
+              # The names will be normalized the same way as redacted_identifiers,
+              # by removing the following symbols: _, -, @, $, and then matched
+              # against the complete variable or key name while ignoring the case.
+              option :redaction_excluded_identifiers do |o|
+                o.env "DD_DYNAMIC_INSTRUMENTATION_REDACTION_EXCLUDED_IDENTIFIERS"
+                o.env_parser do |value|
+                  value&.split(",")&.map(&:strip)
+                end
+
+                o.type :array
+                o.default []
+              end
+
+              # An array of class names, values of which will be redacted from
+              # dynamic instrumentation snapshots. Example: FooClass.
+              # If a name is suffixed by '*', it becomes a wildcard and
+              # instances of any class whose name begins with the specified
+              # prefix will be redacted (example: Foo*).
+              #
+              # The names must all be fully-qualified, if any prefix of a
+              # class name is configured to be redacted, the value will be
+              # subject to redaction. For example, if Foo* is in the
+              # redacted class name list, instances of Foo, FooBar,
+              # Foo::Bar are all subject to redaction, but Bar::Foo will
+              # not be subject to redaction.
+              #
+              # Leading double-colon is permitted but has no effect,
+              # because the names are always considered to be fully-qualified.
+              # For example, adding ::Foo to the list will redact instances
+              # of Foo.
+              #
+              # Trailing colons should not be used because they will trigger
+              # exact match behavior but Ruby class names do not have
+              # trailing colons. For example, Foo:: will not cause anything
+              # to be redacted. Use Foo::* to redact all classes under
+              # the Foo module.
+              option :redacted_type_names do |o|
+                o.env "DD_DYNAMIC_INSTRUMENTATION_REDACTED_TYPES"
+                o.env_parser do |value|
+                  value&.split(",")&.map(&:strip)
+                end
+
+                o.type :array
+                o.default []
+              end
+
+              # Maximum number of object or collection traversals that
+              # will be permitted when serializing captured values.
+              option :max_capture_depth do |o|
+                o.type :int
+                o.default 3
+              end
+
+              # Maximum number of collection (Array and Hash) elements
+              # that will be captured. Arrays and hashes that have more
+              # elements will be truncated to this many elements.
+              option :max_capture_collection_size do |o|
+                o.type :int
+                o.default 100
+              end
+
+              # Strings longer than this length will be truncated to this
+              # length in dynamic instrumentation snapshots.
+              #
+              # Note that while all values are stringified during
+              # serialization, only values which are originally instances
+              # of the String class are subject to this length limit.
+              option :max_capture_string_length do |o|
+                o.type :int
+                o.default 255
+              end
+
+              # Maximim number of attributes that will be captured for
+              # a single non-primitive value.
+              option :max_capture_attribute_count do |o|
+                o.type :int
+                o.default 20
+              end
+
+              option :max_time_to_serialize_ms do |o|
+                o.type :int
+                o.default 200
+                o.env "DD_DYNAMIC_INSTRUMENTATION_MAX_TIME_TO_SERIALIZE"
+              end
+
+              # Settings in the 'internal' group are for internal Datadog
+              # use only, and are needed to test dynamic instrumentation or
+              # experiment with features not released to customers.
+              settings :internal do
+                # This option instructs dynamic instrumentation to use
+                # untargeted trace points when installing line probes and
+                # code tracking is not active.
+                # WARNING: untargeted trace points carry a massive performance
+                # penalty for the entire file in which a line probe is placed.
+                #
+                # If this option is set to false, which is the default,
+                # dynamic instrumentation will add probes that reference
+                # unknown files to the list of pending probes, and when
+                # the respective files are loaded, the line probes will be
+                # installed using targeted trace points. If the file in
+                # question is already loaded when the probe is received
+                # (for example, it is in a third-party library loaded during
+                # application boot), and code tracking was not active when
+                # the file was loaded, such files will not be instrumentable
+                # via line probes.
+                #
+                # If this option is set to true, dynamic instrumentation will
+                # install untargeted trace points for all line probes,
+                # regardless of whether the referenced file is loaded.
+                # This permits instrumenting code which was loaded prior to
+                # code tracking being activated and instrumenting lines when
+                # code tracking is not activated at all. However, untargeted
+                # trace points are extremely slow and will greatly degrade
+                # performance of *all* code executed while they are installed,
+                # not just the instrumentation target.
+                option :untargeted_trace_points do |o|
+                  o.type :bool
+                  o.default false
+                end
+
+                # If true, all of the catch-all rescue blocks in DI
+                # will propagate the exceptions onward.
+                # WARNING: for internal Datadog use only - this will break
+                # the DI product and potentially the library in general in
+                # a multitude of ways, cause resource leakage, permanent
+                # performance decreases, etc.
+                option :propagate_all_exceptions do |o|
+                  o.type :bool
+                  o.default false
+                end
+
+                # Minimum interval, in seconds, between probe status and
+                # snapshot submissions to the agent. Probe notifier worker will
+                # batch together payloads submitted during each interval.
+                # A longer interval reduces the overhead imposed by dynamic
+                # instrumentation on the application, but also increases the
+                # time when application code cannot run (when the batches are
+                # being sent out by the probe notifier worker) and creates a
+                # possibility of dropping payloads if the queue gets too long.
+                option :min_send_interval do |o|
+                  o.type :float
+                  o.default 3
+                end
+
+                # Number of snapshots that can be stored in the probe
+                # notifier worker queue. Larger capacity runs the risk of
+                # creating snapshots that exceed the agent's request size
+                # limit. Smaller capacity increases the risk of dropping
+                # snapshots.
+                option :snapshot_queue_capacity do |o|
+                  o.type :int
+                  o.default 100
+                end
+
+                # Enable dynamic instrumentation in development environments.
+                # Currently DI does not fully implement support for code
+                # unloading and reloading, and is not supported in
+                # non-production environments.
+                option :development do |o|
+                  o.type :bool
+                  o.default false
+                end
+
+                # Enable logging of dynamic instrumentation activity.
+                # This is quite verbose.
+                option :trace_logging do |o|
+                  o.type :bool
+                  o.default false
+
+                  # Use the same environment variable as the rest of
+                  # dd-trace-rb logging for now. Could change to a
+                  # dedicated environment variable in the future but
+                  # will likely need a way to turn on remote config
+                  # debugging (since DI uses RC for configuration).
+                  o.env "DD_TRACE_DEBUG"
+                end
+
+                # If the CPU time consumed by the thread performing instrumentation
+                # exceeds this amount, the offending probe will be automatically disabled.
+                # Set to nil to disable the circuit breaker.
+                # Set to zero to disable every probe after it executes once.
+                option :max_processing_time do |o|
+                  o.type :float
+                  o.default 0.5
+                end
+              end
+            end
+          end
+        end
+      end
     end
   end
 end
-
-require_relative "configuration/settings"

data/lib/datadog/di/context.rb

@@ -14,6 +14,8 @@ module Datadog
         target_self: nil,
         path: nil, caller_locations: nil,
         serialized_entry_args: nil,
+        entry_capture_expressions: nil,
+        entry_capture_evaluation_errors: nil,
         return_value: nil, duration: nil, exception: nil)
         @probe = probe
         @settings = settings
@@ -23,6 +25,8 @@ module Datadog
         @path = path
         @caller_locations = caller_locations
         @serialized_entry_args = serialized_entry_args
+        @entry_capture_expressions = entry_capture_expressions
+        @entry_capture_evaluation_errors = entry_capture_evaluation_errors
         @return_value = return_value
         @duration = duration
         @exception = exception
@@ -33,25 +37,33 @@ module Datadog
       attr_reader :serializer
       attr_reader :locals
       attr_reader :target_self
+
       # Actual path of the instrumented file.
       attr_reader :path
+
       # TODO check how many stack frames we should be keeping/sending,
       # this should be all frames for enriched probes and no frames for
       # non-enriched probes?
       attr_reader :caller_locations
+
       attr_reader :serialized_entry_args
+
+      attr_reader :entry_capture_expressions
+
+      attr_reader :entry_capture_evaluation_errors
+
       # Return value for the method, for a method probe
       attr_reader :return_value
+
       # How long the method took to execute, for a method probe
       attr_reader :duration
+
       # Exception raised by the method, if any, for a method probe
       attr_reader :exception
 
       def serialized_locals
         # TODO cache?
-        locals && serializer.serialize_vars(locals,
-          depth: probe.max_capture_depth || settings.dynamic_instrumentation.max_capture_depth,
-          attribute_count: probe.max_capture_attribute_count || settings.dynamic_instrumentation.max_capture_attribute_count,)
+        locals && serializer.serialize_vars(locals, **probe.snapshot_serializer_limits(settings))
       end
 
       def fetch(var_name)

data/lib/datadog/di/contrib/active_record.rb

@@ -23,7 +23,11 @@ Datadog::DI::Serializer.register(
   #
   # +depth+ is the remaining depth for serializing collections and objects.
   # It should always be an integer.
-  # Reduce it by 1 when invoking +serialize_value+ on the contents of +value+.
+  # Pass it through to +serialize_value+ when the structure you produce
+  # represents +value+ directly (a transparent wrapper); +serialize_value+
+  # decrements depth itself when it recurses into Array/Hash/object entries.
+  # Decrement +depth+ only if your wrapper introduces real additional
+  # nesting levels in the output.
   # This serializer could also potentially do its own depth limiting.
   #
   # Steep: steep thinks all of the arguments are nil here
@@ -33,5 +37,5 @@ Datadog::DI::Serializer.register(
     attributes: value.attributes,
     new_record: value.new_record?,
   }
-  serializer.serialize_value(value_to_serialize, depth: depth - 1, type: value.class)
+  serializer.serialize_value(value_to_serialize, depth: depth, type: value.class)
 end

data/lib/datadog/di/contrib/railtie.rb

@@ -6,7 +6,7 @@ module Datadog
       # Railtie class initializes dynamic instrumentation contrib code
       # in Rails environments.
       class Railtie < Rails::Railtie
-        initializer 'datadog.dynamic_instrumentation.initialize' do |app|
+        initializer "datadog.dynamic_instrumentation.initialize" do |app|
           Contrib.load_now
         end
       end

data/lib/datadog/di/contrib.rb

@@ -1,14 +1,14 @@
 # frozen_string_literal: true
 
-require_relative '../core/contrib/rails/utils'
+require_relative "../core/contrib/rails/utils"
 
 module Datadog
   module DI
     module Contrib
       module_function def load_now_or_later
         if Datadog::Core::Contrib::Rails::Utils.railtie_supported?
-          Datadog.logger.debug('di: loading contrib/railtie')
-          require_relative 'contrib/railtie'
+          Datadog.logger.debug("di: loading contrib/railtie")
+          require_relative "contrib/railtie"
         else
           load_now
         end
@@ -19,8 +19,8 @@ module Datadog
       # dependencies are loaded (or potentially loaded).
       module_function def load_now
         if defined?(ActiveRecord::Base)
-          Datadog.logger.debug('di: loading contrib/active_record')
-          require_relative 'contrib/active_record'
+          Datadog.logger.debug("di: loading contrib/active_record")
+          require_relative "contrib/active_record"
         end
       end
     end

data/lib/datadog/di/el/compiler.rb

@@ -16,20 +16,54 @@ module Datadog
       #
       # @api private
       class Compiler
+        # Compiles +ast+ into eval'able Ruby source.
+        #
+        # Returns the compiled source and companion compiled Regexp
+        # objects.
+        #
+        # @param ast [untyped] expression AST from the probe definition.
+        # @return [Array(String, Array<Regexp>)] the compiled Ruby source and
+        #   the precompiled Regexps, indexed in the order the compiled code
+        #   references them.
         def compile(ast)
-          compile_partial(ast)
+          regexps = []
+          code = compile_partial(ast, regexps)
+          [code, regexps]
+        end
+
+        # Returns the identifier that +ast+ directly references at its top
+        # level, so the message path can redact direct references the same
+        # way snapshots redact by name.
+        #
+        # @param ast [untyped] top-level expression AST from a probe definition.
+        # @return [String, nil] the variable name for a `ref`/`iref`, the field
+        #   name for a `getmember`, or the string key for an `index`; nil when
+        #   the top-level expression is not a direct reference.
+        def redaction_identifier(ast)
+          return nil unless Hash === ast && ast.length == 1
+
+          entry = ast.first
+          return nil if entry.nil?
+
+          op, target = entry
+          case op
+          when "ref"
+            target if String === target
+          when "getmember", "index"
+            target[1] if Array === target && target.length == 2 && String === target[1]
+          end
         end
 
         private
 
         # Steep: https://github.com/soutaro/steep/issues/363
         OPERATORS = { # steep:ignore IncompatibleAssignment
-          'eq' => '==',
-          'ne' => '!=',
-          'ge' => '>=',
-          'gt' => '>',
-          'le' => '<=',
-          'lt' => '<',
+          "eq" => "==",
+          "ne" => "!=",
+          "ge" => ">=",
+          "gt" => ">",
+          "le" => "<=",
+          "lt" => "<",
         }.freeze
 
         # Steep: https://github.com/soutaro/steep/issues/363
@@ -37,49 +71,58 @@ module Datadog
           len isEmpty isUndefined
         ].freeze # steep:ignore IncompatibleAssignment
 
+        # `matches` is also a two-argument method but is special-cased in
+        # #compile_partial so that regular expressions can be precompiled
+        # once, so it is not listed here.
+        #
         # Steep: https://github.com/soutaro/steep/issues/363
         TWO_ARG_METHODS = %w[
-          startsWith endsWith contains matches
+          startsWith endsWith contains
           getmember index instanceof
         ].freeze # steep:ignore IncompatibleAssignment
 
         # Steep: https://github.com/soutaro/steep/issues/363
         MULTI_ARG_METHODS = { # steep:ignore IncompatibleAssignment
-          'and' => '&&',
-          'or' => '||',
+          "and" => "&&",
+          "or" => "||",
         }.freeze
 
-        def compile_partial(ast)
+        # @param ast [untyped] AST node to compile.
+        # @param regexps [Array<Regexp>] output array that collects the
+        #   companion precompiled Regexp objects.
+        # @return [String] compiled Ruby source for +ast+.
+        def compile_partial(ast, regexps)
           case ast
           when Hash
-            if ast.length != 1
+            entry = ast.first
+            if ast.length != 1 || entry.nil?
               raise DI::Error::InvalidExpression, "Expected hash of length 1: #{ast}"
             end
-            op, target = ast.first
+            op, target = entry
             case op
-            when 'ref'
+            when "ref"
               unless String === target
                 raise DI::Error::InvalidExpression, "Bad ref value type: #{target.class}: #{target}"
               end
               case target
-              when '@it'
-                'current_item'
-              when '@key'
-                'current_key'
-              when '@value'
-                'current_value'
-              when '@return'
+              when "@it"
+                "current_item"
+              when "@key"
+                "current_key"
+              when "@value"
+                "current_value"
+              when "@return"
                 # For @return, @duration and @exception we shadow
                 # instance variables.
                 "context.return_value"
-              when '@duration'
+              when "@duration"
                 # There is no way to explicitly format the duration.
                 # TODO come up with better formatting?
                 # We could format to a string here but what if customer
                 # has @duration as part of an expression and wants
                 # to retain it as a number?
-                "(context.duration * 1000)"
-              when '@exception'
+                "(context.duration && context.duration * 1000)"
+              when "@exception"
                 "context.exception"
               else
                 # Ruby technically allows all kinds of symbols in variable
@@ -89,44 +132,60 @@ module Datadog
                 unless target =~ %r{\A(@?)([a-zA-Z0-9_]+)\z}
                   raise DI::Error::BadVariableName, "Bad variable name: #{target}"
                 end
-                method_name = (($1 == '@') ? 'iref' : 'ref')
+                method_name = (($1 == "@") ? "iref" : "ref")
                 "#{method_name}('#{target}')"
               end
             when *SINGLE_ARG_METHODS
               method_name = op.gsub(/[A-Z]/) { |m| "_#{m.downcase}" }
-              "#{method_name}(#{compile_partial(target)}, '#{var_name_maybe(target)}')"
+              "#{method_name}(#{compile_partial(target, regexps)}, '#{var_name_maybe(target)}')"
+            when "matches"
+              unless Array === target && target.length == 2
+                raise DI::Error::InvalidExpression, "Improper matches syntax"
+              end
+              first, second = target
+              if String === second
+                # Match against a literal regular expression (string).
+                index = precompile_regexp(second, regexps)
+                "matches_compiled(#{compile_partial(first, regexps)}, #{index})"
+              else
+                # Match against a (complex) expression.
+                "matches(#{compile_partial(first, regexps)}, (#{compile_partial(second, regexps)}))"
+              end
             when *TWO_ARG_METHODS
               method_name = op.gsub(/[A-Z]/) { |m| "_#{m.downcase}" }
               unless Array === target && target.length == 2
                 raise DI::Error::InvalidExpression, "Improper #{op} syntax"
               end
               first, second = target
-              "#{method_name}(#{compile_partial(first)}, (#{compile_partial(second)}))"
+              "#{method_name}(#{compile_partial(first, regexps)}, (#{compile_partial(second, regexps)}))"
             when *MULTI_ARG_METHODS.keys
               unless Array === target && target.length >= 1
                 raise DI::Error::InvalidExpression, "Improper #{op} syntax"
               end
               compiled_targets = target.map do |item|
-                "(#{compile_partial(item)})"
+                "(#{compile_partial(item, regexps)})"
               end
               compiled_op = MULTI_ARG_METHODS[op]
               "(#{compiled_targets.join(" #{compiled_op} ")})"
-            when 'substring'
+            when "substring"
               unless Array === target && target.length == 3
                 raise DI::Error::InvalidExpression, "Improper #{op} syntax"
               end
-              "#{op}(#{target.map { |arg| "(#{compile_partial(arg)})" }.join(", ")})"
-            when 'not'
-              "!(#{compile_partial(target)})"
+              "#{op}(#{target.map { |arg| "(#{compile_partial(arg, regexps)})" }.join(", ")})"
+            when "not"
+              "!(#{compile_partial(target, regexps)})"
             when *OPERATORS.keys
               unless Array === target && target.length == 2
                 raise DI::Error::InvalidExpression, "Improper #{op} syntax"
               end
               first, second = target
               operator = OPERATORS.fetch(op)
-              "(#{compile_partial(first)}) #{operator} (#{compile_partial(second)})"
-            when 'any', 'all', 'filter'
-              "#{op}(#{compile_partial(target.first)}) { |current_item, current_key, current_value| #{compile_partial(target.last)} }"
+              "(#{compile_partial(first, regexps)}) #{operator} (#{compile_partial(second, regexps)})"
+            when "any", "all", "filter"
+              unless Array === target && target.length == 2
+                raise DI::Error::InvalidExpression, "Improper #{op} syntax"
+              end
+              "#{op}(#{compile_partial(target.first, regexps)}) { |current_item, current_key, current_value| #{compile_partial(target.last, regexps)} }"
             else
               raise DI::Error::InvalidExpression, "Unknown operation: #{op}"
             end
@@ -151,16 +210,33 @@ module Datadog
         # WARNING: the result of this method is included in eval'd code,
         # it must be sanitized to avoid injection.
         def var_name_maybe(target)
-          if Hash === target && target.length == 1 && target.keys.first == 'ref' &&
+          if Hash === target && target.length == 1 && target.keys.first == "ref" &&
               String === (value = target.values.first)
             escape(value)
           else
-            '(expression)'
+            "(expression)"
           end
         end
 
         def escape(needle)
-          needle.gsub("\\") { "\\\\" }.gsub('"') { "\\\"" }.gsub('#') { "\\#" }
+          needle.gsub("\\") { "\\\\" }.gsub('"') { "\\\"" }.gsub("#") { "\\#" }
+        end
+
+        # Compile a literal regular expression +regexp_str+ at
+        # instrumentation time. Append it to +regexps+, returning its index for
+        # Evaluator#matches_compiled to look up.
+        #
+        # @param regexp_str [String] regular expression source.
+        # @param regexps [Array<Regexp>] output array to append the compiled
+        #   regexp to.
+        # @return [Integer] index into +regexps+.
+        # @raise [DI::Error::InvalidExpression] if +needle+ is not a valid regexp.
+        def precompile_regexp(regexp_str, regexps)
+          index = regexps.length
+          regexps << Evaluator.compile_regexp(regexp_str)
+          index
+        rescue RegexpError => exc
+          raise DI::Error::InvalidExpression, "Invalid regular expression in matches: #{exc.class}: #{exc.message}"
         end
       end
     end