datadog 2.4.0 → 2.44.0

data/lib/datadog/di/configuration.rb

@@ -4,8 +4,247 @@ module Datadog
   module DI
     # Configuration for DI
     module Configuration
+      # Settings
+      module Settings
+        def self.extended(base)
+          base = base.singleton_class unless base.is_a?(Class)
+          add_settings!(base)
+        end
+
+        def self.add_settings!(base)
+          base.class_eval do
+            settings :dynamic_instrumentation do
+              option :enabled do |o|
+                o.type :bool
+                # The environment variable has an "internal" prefix so that
+                # any customers that have the "proper" environment variable
+                # turned on (i.e. DD_DYNAMIC_INSTRUMENTATION_ENABLED)
+                # do not enable Ruby DI until the latter is ready for
+                # customer testing.
+                o.env "DD_DYNAMIC_INSTRUMENTATION_ENABLED"
+                o.default false
+              end
+
+              # An array of variable and key names to redact in addition to
+              # the built-in list of identifiers.
+              #
+              # The names will be normalized by removing the following
+              # symbols: _, -, @, $, and then matched to the complete
+              # variable or key name while ignoring the case.
+              # For example, specifying pass_word will match password and
+              # PASSWORD, and specifying PASSWORD will match pass_word.
+              # Note that, while the at sign (@) is used in Ruby to refer
+              # to instance variables, it does not have any significance
+              # for this setting (and is removed before matching identifiers).
+              option :redacted_identifiers do |o|
+                o.env "DD_DYNAMIC_INSTRUMENTATION_REDACTED_IDENTIFIERS"
+                o.env_parser do |value|
+                  value&.split(",")&.map(&:strip)
+                end
+
+                o.type :array
+                o.default []
+              end
+
+              # An array of variable and key names to exclude from the
+              # built-in redaction list.
+              #
+              # This allows users to capture values of variables that would
+              # otherwise be redacted by the default identifier list.
+              # For example, if an application has a "session" variable
+              # that does not contain sensitive data, "session" can be added
+              # to this list to exclude it from redaction.
+              #
+              # The names will be normalized the same way as redacted_identifiers,
+              # by removing the following symbols: _, -, @, $, and then matched
+              # against the complete variable or key name while ignoring the case.
+              option :redaction_excluded_identifiers do |o|
+                o.env "DD_DYNAMIC_INSTRUMENTATION_REDACTION_EXCLUDED_IDENTIFIERS"
+                o.env_parser do |value|
+                  value&.split(",")&.map(&:strip)
+                end
+
+                o.type :array
+                o.default []
+              end
+
+              # An array of class names, values of which will be redacted from
+              # dynamic instrumentation snapshots. Example: FooClass.
+              # If a name is suffixed by '*', it becomes a wildcard and
+              # instances of any class whose name begins with the specified
+              # prefix will be redacted (example: Foo*).
+              #
+              # The names must all be fully-qualified, if any prefix of a
+              # class name is configured to be redacted, the value will be
+              # subject to redaction. For example, if Foo* is in the
+              # redacted class name list, instances of Foo, FooBar,
+              # Foo::Bar are all subject to redaction, but Bar::Foo will
+              # not be subject to redaction.
+              #
+              # Leading double-colon is permitted but has no effect,
+              # because the names are always considered to be fully-qualified.
+              # For example, adding ::Foo to the list will redact instances
+              # of Foo.
+              #
+              # Trailing colons should not be used because they will trigger
+              # exact match behavior but Ruby class names do not have
+              # trailing colons. For example, Foo:: will not cause anything
+              # to be redacted. Use Foo::* to redact all classes under
+              # the Foo module.
+              option :redacted_type_names do |o|
+                o.env "DD_DYNAMIC_INSTRUMENTATION_REDACTED_TYPES"
+                o.env_parser do |value|
+                  value&.split(",")&.map(&:strip)
+                end
+
+                o.type :array
+                o.default []
+              end
+
+              # Maximum number of object or collection traversals that
+              # will be permitted when serializing captured values.
+              option :max_capture_depth do |o|
+                o.type :int
+                o.default 3
+              end
+
+              # Maximum number of collection (Array and Hash) elements
+              # that will be captured. Arrays and hashes that have more
+              # elements will be truncated to this many elements.
+              option :max_capture_collection_size do |o|
+                o.type :int
+                o.default 100
+              end
+
+              # Strings longer than this length will be truncated to this
+              # length in dynamic instrumentation snapshots.
+              #
+              # Note that while all values are stringified during
+              # serialization, only values which are originally instances
+              # of the String class are subject to this length limit.
+              option :max_capture_string_length do |o|
+                o.type :int
+                o.default 255
+              end
+
+              # Maximim number of attributes that will be captured for
+              # a single non-primitive value.
+              option :max_capture_attribute_count do |o|
+                o.type :int
+                o.default 20
+              end
+
+              option :max_time_to_serialize_ms do |o|
+                o.type :int
+                o.default 200
+                o.env "DD_DYNAMIC_INSTRUMENTATION_MAX_TIME_TO_SERIALIZE"
+              end
+
+              # Settings in the 'internal' group are for internal Datadog
+              # use only, and are needed to test dynamic instrumentation or
+              # experiment with features not released to customers.
+              settings :internal do
+                # This option instructs dynamic instrumentation to use
+                # untargeted trace points when installing line probes and
+                # code tracking is not active.
+                # WARNING: untargeted trace points carry a massive performance
+                # penalty for the entire file in which a line probe is placed.
+                #
+                # If this option is set to false, which is the default,
+                # dynamic instrumentation will add probes that reference
+                # unknown files to the list of pending probes, and when
+                # the respective files are loaded, the line probes will be
+                # installed using targeted trace points. If the file in
+                # question is already loaded when the probe is received
+                # (for example, it is in a third-party library loaded during
+                # application boot), and code tracking was not active when
+                # the file was loaded, such files will not be instrumentable
+                # via line probes.
+                #
+                # If this option is set to true, dynamic instrumentation will
+                # install untargeted trace points for all line probes,
+                # regardless of whether the referenced file is loaded.
+                # This permits instrumenting code which was loaded prior to
+                # code tracking being activated and instrumenting lines when
+                # code tracking is not activated at all. However, untargeted
+                # trace points are extremely slow and will greatly degrade
+                # performance of *all* code executed while they are installed,
+                # not just the instrumentation target.
+                option :untargeted_trace_points do |o|
+                  o.type :bool
+                  o.default false
+                end
+
+                # If true, all of the catch-all rescue blocks in DI
+                # will propagate the exceptions onward.
+                # WARNING: for internal Datadog use only - this will break
+                # the DI product and potentially the library in general in
+                # a multitude of ways, cause resource leakage, permanent
+                # performance decreases, etc.
+                option :propagate_all_exceptions do |o|
+                  o.type :bool
+                  o.default false
+                end
+
+                # Minimum interval, in seconds, between probe status and
+                # snapshot submissions to the agent. Probe notifier worker will
+                # batch together payloads submitted during each interval.
+                # A longer interval reduces the overhead imposed by dynamic
+                # instrumentation on the application, but also increases the
+                # time when application code cannot run (when the batches are
+                # being sent out by the probe notifier worker) and creates a
+                # possibility of dropping payloads if the queue gets too long.
+                option :min_send_interval do |o|
+                  o.type :float
+                  o.default 3
+                end
+
+                # Number of snapshots that can be stored in the probe
+                # notifier worker queue. Larger capacity runs the risk of
+                # creating snapshots that exceed the agent's request size
+                # limit. Smaller capacity increases the risk of dropping
+                # snapshots.
+                option :snapshot_queue_capacity do |o|
+                  o.type :int
+                  o.default 100
+                end
+
+                # Enable dynamic instrumentation in development environments.
+                # Currently DI does not fully implement support for code
+                # unloading and reloading, and is not supported in
+                # non-production environments.
+                option :development do |o|
+                  o.type :bool
+                  o.default false
+                end
+
+                # Enable logging of dynamic instrumentation activity.
+                # This is quite verbose.
+                option :trace_logging do |o|
+                  o.type :bool
+                  o.default false
+
+                  # Use the same environment variable as the rest of
+                  # dd-trace-rb logging for now. Could change to a
+                  # dedicated environment variable in the future but
+                  # will likely need a way to turn on remote config
+                  # debugging (since DI uses RC for configuration).
+                  o.env "DD_TRACE_DEBUG"
+                end
+
+                # If the CPU time consumed by the thread performing instrumentation
+                # exceeds this amount, the offending probe will be automatically disabled.
+                # Set to nil to disable the circuit breaker.
+                # Set to zero to disable every probe after it executes once.
+                option :max_processing_time do |o|
+                  o.type :float
+                  o.default 0.5
+                end
+              end
+            end
+          end
+        end
+      end
     end
   end
 end
-
-require_relative "configuration/settings"

data/lib/datadog/di/context.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+module Datadog
+  module DI
+    # Contains local and instance variables used when evaluating
+    # expressions in DI Expression Language.
+    #
+    # @api private
+    class Context
+      def initialize(probe:, settings:, serializer:, locals: nil,
+        # In Ruby everything is a method, therefore we should always have
+        # a target self. However, if we are not capturing a snapshot,
+        # there is no need to pass in the target self.
+        target_self: nil,
+        path: nil, caller_locations: nil,
+        serialized_entry_args: nil,
+        entry_capture_expressions: nil,
+        entry_capture_evaluation_errors: nil,
+        return_value: nil, duration: nil, exception: nil)
+        @probe = probe
+        @settings = settings
+        @serializer = serializer
+        @locals = locals
+        @target_self = target_self
+        @path = path
+        @caller_locations = caller_locations
+        @serialized_entry_args = serialized_entry_args
+        @entry_capture_expressions = entry_capture_expressions
+        @entry_capture_evaluation_errors = entry_capture_evaluation_errors
+        @return_value = return_value
+        @duration = duration
+        @exception = exception
+      end
+
+      attr_reader :probe
+      attr_reader :settings
+      attr_reader :serializer
+      attr_reader :locals
+      attr_reader :target_self
+
+      # Actual path of the instrumented file.
+      attr_reader :path
+
+      # TODO check how many stack frames we should be keeping/sending,
+      # this should be all frames for enriched probes and no frames for
+      # non-enriched probes?
+      attr_reader :caller_locations
+
+      attr_reader :serialized_entry_args
+
+      attr_reader :entry_capture_expressions
+
+      attr_reader :entry_capture_evaluation_errors
+
+      # Return value for the method, for a method probe
+      attr_reader :return_value
+
+      # How long the method took to execute, for a method probe
+      attr_reader :duration
+
+      # Exception raised by the method, if any, for a method probe
+      attr_reader :exception
+
+      def serialized_locals
+        # TODO cache?
+        locals && serializer.serialize_vars(locals, **probe.snapshot_serializer_limits(settings))
+      end
+
+      def fetch(var_name)
+        unless locals
+          # TODO return "undefined" instead?
+          return nil
+        end
+        locals[var_name.to_sym]
+      end
+
+      def fetch_ivar(var_name)
+        target_self.instance_variable_get(var_name)
+      end
+    end
+  end
+end

data/lib/datadog/di/contrib/active_record.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+Datadog::DI::Serializer.register(
+  # This serializer uses a dynamic condition to determine its applicability
+  # to a particular value. A simpler case could have been a serializer for
+  # a particular class, but in this case any ActiveRecord model is covered
+  # and they all have different classes.
+  #
+  # An alternative could have been to make DI specifically provide lookup
+  # logic for "instances of classes derived from X", but a condition Proc
+  # is more universal.
+  condition: lambda { |value| ActiveRecord::Base === value }
+) do |serializer, value, name:, depth:|
+  # +serializer+ is an instance of DI::Serializer.
+  # Use it to perform the serialization to primitive values.
+  #
+  # +value+ is the value to serialize. It should match the condition
+  # provided above, meaning it would be an ActiveRecord::Base instance.
+  #
+  # +name+ is the name of the (local/instance) variable being serialized.
+  # The name is used by DI for redaction (upstream of serialization logic),
+  # and could potentially be used for redaction here also.
+  #
+  # +depth+ is the remaining depth for serializing collections and objects.
+  # It should always be an integer.
+  # Pass it through to +serialize_value+ when the structure you produce
+  # represents +value+ directly (a transparent wrapper); +serialize_value+
+  # decrements depth itself when it recurses into Array/Hash/object entries.
+  # Decrement +depth+ only if your wrapper introduces real additional
+  # nesting levels in the output.
+  # This serializer could also potentially do its own depth limiting.
+  #
+  # Steep: steep thinks all of the arguments are nil here
+  # Looks like it cannot handle kwargs in lambdas
+  # @type var depth: Integer
+  value_to_serialize = {
+    attributes: value.attributes,
+    new_record: value.new_record?,
+  }
+  serializer.serialize_value(value_to_serialize, depth: depth, type: value.class)
+end

data/lib/datadog/di/contrib/railtie.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Datadog
+  module DI
+    module Contrib
+      # Railtie class initializes dynamic instrumentation contrib code
+      # in Rails environments.
+      class Railtie < Rails::Railtie
+        initializer "datadog.dynamic_instrumentation.initialize" do |app|
+          Contrib.load_now
+        end
+      end
+    end
+  end
+end

data/lib/datadog/di/contrib.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require_relative "../core/contrib/rails/utils"
+
+module Datadog
+  module DI
+    module Contrib
+      module_function def load_now_or_later
+        if Datadog::Core::Contrib::Rails::Utils.railtie_supported?
+          Datadog.logger.debug("di: loading contrib/railtie")
+          require_relative "contrib/railtie"
+        else
+          load_now
+        end
+      end
+
+      # This method can be called more than once, to attempt to load
+      # DI components that depend on third-party libraries after additional
+      # dependencies are loaded (or potentially loaded).
+      module_function def load_now
+        if defined?(ActiveRecord::Base)
+          Datadog.logger.debug("di: loading contrib/active_record")
+          require_relative "contrib/active_record"
+        end
+      end
+    end
+  end
+end

data/lib/datadog/di/el/compiler.rb

@@ -0,0 +1,244 @@
+# frozen_string_literal: true
+
+module Datadog
+  module DI
+    module EL
+      # DI Expression Language compiler.
+      #
+      # Converts AST in probe definitions into Expression objects.
+      #
+      # WARNING: this class produces strings that are then eval'd as
+      # Ruby code. Input ASTs are user-controlled. As such the compiler
+      # must sanitize and escape all input to avoid injection.
+      #
+      # Besides quotes and backslashes we must also escape # which is
+      # starting string interpolation (#{...}).
+      #
+      # @api private
+      class Compiler
+        # Compiles +ast+ into eval'able Ruby source.
+        #
+        # Returns the compiled source and companion compiled Regexp
+        # objects.
+        #
+        # @param ast [untyped] expression AST from the probe definition.
+        # @return [Array(String, Array<Regexp>)] the compiled Ruby source and
+        #   the precompiled Regexps, indexed in the order the compiled code
+        #   references them.
+        def compile(ast)
+          regexps = []
+          code = compile_partial(ast, regexps)
+          [code, regexps]
+        end
+
+        # Returns the identifier that +ast+ directly references at its top
+        # level, so the message path can redact direct references the same
+        # way snapshots redact by name.
+        #
+        # @param ast [untyped] top-level expression AST from a probe definition.
+        # @return [String, nil] the variable name for a `ref`/`iref`, the field
+        #   name for a `getmember`, or the string key for an `index`; nil when
+        #   the top-level expression is not a direct reference.
+        def redaction_identifier(ast)
+          return nil unless Hash === ast && ast.length == 1
+
+          entry = ast.first
+          return nil if entry.nil?
+
+          op, target = entry
+          case op
+          when "ref"
+            target if String === target
+          when "getmember", "index"
+            target[1] if Array === target && target.length == 2 && String === target[1]
+          end
+        end
+
+        private
+
+        # Steep: https://github.com/soutaro/steep/issues/363
+        OPERATORS = { # steep:ignore IncompatibleAssignment
+          "eq" => "==",
+          "ne" => "!=",
+          "ge" => ">=",
+          "gt" => ">",
+          "le" => "<=",
+          "lt" => "<",
+        }.freeze
+
+        # Steep: https://github.com/soutaro/steep/issues/363
+        SINGLE_ARG_METHODS = %w[
+          len isEmpty isUndefined
+        ].freeze # steep:ignore IncompatibleAssignment
+
+        # `matches` is also a two-argument method but is special-cased in
+        # #compile_partial so that regular expressions can be precompiled
+        # once, so it is not listed here.
+        #
+        # Steep: https://github.com/soutaro/steep/issues/363
+        TWO_ARG_METHODS = %w[
+          startsWith endsWith contains
+          getmember index instanceof
+        ].freeze # steep:ignore IncompatibleAssignment
+
+        # Steep: https://github.com/soutaro/steep/issues/363
+        MULTI_ARG_METHODS = { # steep:ignore IncompatibleAssignment
+          "and" => "&&",
+          "or" => "||",
+        }.freeze
+
+        # @param ast [untyped] AST node to compile.
+        # @param regexps [Array<Regexp>] output array that collects the
+        #   companion precompiled Regexp objects.
+        # @return [String] compiled Ruby source for +ast+.
+        def compile_partial(ast, regexps)
+          case ast
+          when Hash
+            entry = ast.first
+            if ast.length != 1 || entry.nil?
+              raise DI::Error::InvalidExpression, "Expected hash of length 1: #{ast}"
+            end
+            op, target = entry
+            case op
+            when "ref"
+              unless String === target
+                raise DI::Error::InvalidExpression, "Bad ref value type: #{target.class}: #{target}"
+              end
+              case target
+              when "@it"
+                "current_item"
+              when "@key"
+                "current_key"
+              when "@value"
+                "current_value"
+              when "@return"
+                # For @return, @duration and @exception we shadow
+                # instance variables.
+                "context.return_value"
+              when "@duration"
+                # There is no way to explicitly format the duration.
+                # TODO come up with better formatting?
+                # We could format to a string here but what if customer
+                # has @duration as part of an expression and wants
+                # to retain it as a number?
+                "(context.duration && context.duration * 1000)"
+              when "@exception"
+                "context.exception"
+              else
+                # Ruby technically allows all kinds of symbols in variable
+                # names, for example spaces and many characters.
+                # Start out with strict validation to avoid possible
+                # surprises and need to escape.
+                unless target =~ %r{\A(@?)([a-zA-Z0-9_]+)\z}
+                  raise DI::Error::BadVariableName, "Bad variable name: #{target}"
+                end
+                method_name = (($1 == "@") ? "iref" : "ref")
+                "#{method_name}('#{target}')"
+              end
+            when *SINGLE_ARG_METHODS
+              method_name = op.gsub(/[A-Z]/) { |m| "_#{m.downcase}" }
+              "#{method_name}(#{compile_partial(target, regexps)}, '#{var_name_maybe(target)}')"
+            when "matches"
+              unless Array === target && target.length == 2
+                raise DI::Error::InvalidExpression, "Improper matches syntax"
+              end
+              first, second = target
+              if String === second
+                # Match against a literal regular expression (string).
+                index = precompile_regexp(second, regexps)
+                "matches_compiled(#{compile_partial(first, regexps)}, #{index})"
+              else
+                # Match against a (complex) expression.
+                "matches(#{compile_partial(first, regexps)}, (#{compile_partial(second, regexps)}))"
+              end
+            when *TWO_ARG_METHODS
+              method_name = op.gsub(/[A-Z]/) { |m| "_#{m.downcase}" }
+              unless Array === target && target.length == 2
+                raise DI::Error::InvalidExpression, "Improper #{op} syntax"
+              end
+              first, second = target
+              "#{method_name}(#{compile_partial(first, regexps)}, (#{compile_partial(second, regexps)}))"
+            when *MULTI_ARG_METHODS.keys
+              unless Array === target && target.length >= 1
+                raise DI::Error::InvalidExpression, "Improper #{op} syntax"
+              end
+              compiled_targets = target.map do |item|
+                "(#{compile_partial(item, regexps)})"
+              end
+              compiled_op = MULTI_ARG_METHODS[op]
+              "(#{compiled_targets.join(" #{compiled_op} ")})"
+            when "substring"
+              unless Array === target && target.length == 3
+                raise DI::Error::InvalidExpression, "Improper #{op} syntax"
+              end
+              "#{op}(#{target.map { |arg| "(#{compile_partial(arg, regexps)})" }.join(", ")})"
+            when "not"
+              "!(#{compile_partial(target, regexps)})"
+            when *OPERATORS.keys
+              unless Array === target && target.length == 2
+                raise DI::Error::InvalidExpression, "Improper #{op} syntax"
+              end
+              first, second = target
+              operator = OPERATORS.fetch(op)
+              "(#{compile_partial(first, regexps)}) #{operator} (#{compile_partial(second, regexps)})"
+            when "any", "all", "filter"
+              unless Array === target && target.length == 2
+                raise DI::Error::InvalidExpression, "Improper #{op} syntax"
+              end
+              "#{op}(#{compile_partial(target.first, regexps)}) { |current_item, current_key, current_value| #{compile_partial(target.last, regexps)} }"
+            else
+              raise DI::Error::InvalidExpression, "Unknown operation: #{op}"
+            end
+          when Numeric, true, false, nil
+            # No escaping is needed for the values here.
+            ast.inspect
+          when String
+            "\"#{escape(ast)}\""
+          when Array
+            # Arrays are commonly used as arguments of operators/methods,
+            # but there are no arrays at the top level in the syntax that
+            # we currently understand. Provide a helpful error message in case
+            # syntax is expanded in the future.
+            raise DI::Error::InvalidExpression, "Array is not valid at its location, do you need to upgrade dd-trace-rb? #{ast}"
+          else
+            raise DI::Error::InvalidExpression, "Unknown type in AST: #{ast}"
+          end
+        end
+
+        # Returns a textual description of +target+ for use in exception
+        # messages. +target+ could be any expression language expression.
+        # WARNING: the result of this method is included in eval'd code,
+        # it must be sanitized to avoid injection.
+        def var_name_maybe(target)
+          if Hash === target && target.length == 1 && target.keys.first == "ref" &&
+              String === (value = target.values.first)
+            escape(value)
+          else
+            "(expression)"
+          end
+        end
+
+        def escape(needle)
+          needle.gsub("\\") { "\\\\" }.gsub('"') { "\\\"" }.gsub("#") { "\\#" }
+        end
+
+        # Compile a literal regular expression +regexp_str+ at
+        # instrumentation time. Append it to +regexps+, returning its index for
+        # Evaluator#matches_compiled to look up.
+        #
+        # @param regexp_str [String] regular expression source.
+        # @param regexps [Array<Regexp>] output array to append the compiled
+        #   regexp to.
+        # @return [Integer] index into +regexps+.
+        # @raise [DI::Error::InvalidExpression] if +needle+ is not a valid regexp.
+        def precompile_regexp(regexp_str, regexps)
+          index = regexps.length
+          regexps << Evaluator.compile_regexp(regexp_str)
+          index
+        rescue RegexpError => exc
+          raise DI::Error::InvalidExpression, "Invalid regular expression in matches: #{exc.class}: #{exc.message}"
+        end
+      end
+    end
+  end
+end