datadog 2.2.0 → 2.4.0

data/lib/datadog/di/configuration/settings.rb

@@ -0,0 +1,163 @@
+# frozen_string_literal: true
+
+module Datadog
+  module DI
+    module Configuration
+      # Settings
+      module Settings
+        def self.extended(base)
+          base = base.singleton_class unless base.is_a?(Class)
+          add_settings!(base)
+        end
+
+        def self.add_settings!(base)
+          base.class_eval do
+            # The setting has "internal" prefix to prevent it from being
+            # prematurely turned on by customers.
+            settings :dynamic_instrumentation do
+              option :enabled do |o|
+                o.type :bool
+                # The environment variable has an "internal" prefix so that
+                # any customers that have the "proper" environment variable
+                # turned on (i.e. DD_DYNAMIC_INSTRUMENTATION_ENABLED)
+                # do not enable Ruby DI until the latter is ready for
+                # customer testing.
+                o.env "DD_DYNAMIC_INSTRUMENTATION_ENABLED"
+                o.default false
+              end
+
+              # This option instructs dynamic instrumentation to use
+              # untargeted trace points when installing line probes and
+              # code tracking is not active.
+              # WARNING: untargeted trace points carry a massive performance
+              # penalty for the entire file in which a line probe is placed.
+              #
+              # If this option is set to false, which is the default,
+              # dynamic instrumentation will add probes that reference
+              # unknown files to the list of pending probes, and when
+              # the respective files are loaded, the line probes will be
+              # installed using targeted trace points. If the file in
+              # question is already loaded when the probe is received
+              # (for example, it is in a third-party library loaded during
+              # application boot), and code tracking was not active when
+              # the file was loaded, such files will not be instrumentable
+              # via line probes.
+              #
+              # If this option is set to true
+              #
+              # activated, DI will in
+              # activated or because the files being targeted have beenIf true and code tracking is not enabled, dynamic instrumentation
+              # will use untargeted trace points.
+              # If false and code tracking is not enabled, dynamic
+              # instrumentation will not instrument any files loaded
+              # WARNING: these trace points will greatly degrade performance
+              # of all code in the instrumented files.
+              option :untargeted_trace_points do |o|
+                o.type :bool
+                o.default false
+              end
+
+              # If true, all of the catch-all rescue blocks in DI
+              # will propagate the exceptions onward.
+              # WARNING: for internal Datadog use only - this will break
+              # the DI product and potentially the library in general in
+              # a multitude of ways, cause resource leakage, permanent
+              # performance decreases, etc.
+              option :propagate_all_exceptions do |o|
+                o.type :bool
+                o.default false
+              end
+
+              # An array of variable and key names to redact in addition to
+              # the built-in list of identifiers.
+              #
+              # The names will be normalized by removing the following
+              # symbols: _, -, @, $, and then matched to the complete
+              # variable or key name while ignoring the case.
+              # For example, specifying pass_word will match password and
+              # PASSWORD, and specifying PASSWORD will match pass_word.
+              # Note that, while the at sign (@) is used in Ruby to refer
+              # to instance variables, it does not have any significance
+              # for this setting (and is removed before matching identifiers).
+              option :redacted_identifiers do |o|
+                o.env "DD_DYNAMIC_INSTRUMENTATION_REDACTED_IDENTIFIERS"
+                o.env_parser do |value|
+                  value&.split(",")&.map(&:strip)
+                end
+
+                o.type :array
+                o.default []
+              end
+
+              # An array of class names, values of which will be redacted from
+              # dynamic instrumentation snapshots. Example: FooClass.
+              # If a name is suffixed by '*', it becomes a wildcard and
+              # instances of any class whose name begins with the specified
+              # prefix will be redacted (example: Foo*).
+              #
+              # The names must all be fully-qualified, if any prefix of a
+              # class name is configured to be redacted, the value will be
+              # subject to redaction. For example, if Foo* is in the
+              # redacted class name list, instances of Foo, FooBar,
+              # Foo::Bar are all subject to redaction, but Bar::Foo will
+              # not be subject to redaction.
+              #
+              # Leading double-colon is permitted but has no effect,
+              # because the names are always considered to be fully-qualified.
+              # For example, adding ::Foo to the list will redact instances
+              # of Foo.
+              #
+              # Trailing colons should not be used because they will trigger
+              # exact match behavior but Ruby class names do not have
+              # trailing colons. For example, Foo:: will not cause anything
+              # to be redacted. Use Foo::* to redact all classes under
+              # the Foo module.
+              option :redacted_type_names do |o|
+                o.env "DD_DYNAMIC_INSTRUMENTATION_REDACTED_TYPES"
+                o.env_parser do |value|
+                  value&.split(",")&.map(&:strip)
+                end
+
+                o.type :array
+                o.default []
+              end
+
+              # Maximum number of object or collection traversals that
+              # will be permitted when serializing captured values.
+              option :max_capture_depth do |o|
+                o.type :int
+                o.default 3
+              end
+
+              # Maximum number of collection (Array and Hash) elements
+              # that will be captured. Arrays and hashes that have more
+              # elements will be truncated to this many elements.
+              option :max_capture_collection_size do |o|
+                o.type :int
+                o.default 100
+              end
+
+              # Strings longer than this length will be truncated to this
+              # length in dynamic instrumentation snapshots.
+              #
+              # Note that while all values are stringified during
+              # serialization, only values which are originally instances
+              # of the String class are subject to this length limit.
+              option :max_capture_string_length do |o|
+                o.type :int
+                o.default 255
+              end
+
+              # Maximim number of attributes that will be captured for
+              # a single non-primitive value.
+              option :max_capture_attribute_count do |o|
+                o.type :int
+                o.default 20
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/di/configuration.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Datadog
+  module DI
+    # Configuration for DI
+    module Configuration
+    end
+  end
+end
+
+require_relative "configuration/settings"

data/lib/datadog/di/error.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Datadog
+  module DI
+    # Base class for Dynamic Instrumentation exceptions.
+    #
+    # None of these exceptions should be propagated out of DI to user
+    # applications, therefore these exceptions are not considered to be
+    # part of the public API of the library.
+    #
+    # @api private
+    class Error < StandardError
+      # Probe does not contain a line number (i.e., is not a line probe).
+      class MissingLineNumber < Error
+      end
+
+      # Failed to communicate to the local Datadog agent (e.g. to send
+      # probe status or a snapshot).
+      class AgentCommunicationError < Error
+      end
+
+      # Attempting to instrument a method or file which does not exist.
+      #
+      # This could be due to the code that is referenced in the probe
+      # having not been loaded yet, or due to the probe referencing code
+      # that does not in fact exist anywhere (e.g. due to a misspelling).
+      class DITargetNotDefined < Error
+      end
+    end
+  end
+end

data/lib/datadog/di/extensions.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+require_relative "../core/configuration"
+require_relative "configuration"
+
+module Datadog
+  module DI
+    # Extends Datadog tracing with DI features
+    module Extensions
+      # Inject DI into global objects.
+      def self.activate!
+        Core::Configuration::Settings.extend(Configuration::Settings)
+      end
+    end
+  end
+end

data/lib/datadog/di/probe.rb

@@ -0,0 +1,133 @@
+# frozen_string_literal: true
+
+require_relative "error"
+require_relative "../core/rate_limiter"
+
+module Datadog
+  module DI
+    # Encapsulates probe information (as received via remote config)
+    # and state (e.g. whether the probe was installed, or executed).
+    #
+    # It is possible that remote configuration will specify an unsupported
+    # probe type or attribute, due to new DI functionality being added
+    # over time. We want to have predictable behavior in such cases, and
+    # since we can't guarantee that there will be enough information in
+    # a remote config payload to construct a functional probe, ProbeBuilder
+    # and remote config code must be prepared to deal with exceptions
+    # raised by Probe constructor in particular. Therefore, Probe constructor
+    # will raise an exception if it determines that there is not enough
+    # information (or confilcting information) in the arguments to create a
+    # functional probe, and upstream code is tasked with not spamming logs
+    # with notifications of such errors (and potentially limiting the
+    # attempts to construct probe from a given payload).
+    #
+    # Note that, while remote configuration provides line numbers as an
+    # array, the only supported line number configuration is a single line
+    # (this is the case for all languages currently). Therefore Probe
+    # only supports one line number, and ProbeBuilder is responsible for
+    # extracting that one line number out of the array received from RC.
+    #
+    # Note: only some of the parameter/attribute values are currently validated.
+    #
+    # @api private
+    class Probe
+      def initialize(id:, type:,
+        file: nil, line_no: nil, type_name: nil, method_name: nil,
+        template: nil, capture_snapshot: false, max_capture_depth: nil, rate_limit: nil)
+        # Perform some sanity checks here to detect unexpected attribute
+        # combinations, in order to not do them in subsequent code.
+        if line_no && method_name
+          raise ArgumentError, "Probe contains both line number and method name: #{id}"
+        end
+
+        if type_name && !method_name || method_name && !type_name
+          raise ArgumentError, "Partial method probe definition: #{id}"
+        end
+
+        @id = id
+        @type = type
+        @file = file
+        @line_no = line_no
+        @type_name = type_name
+        @method_name = method_name
+        @template = template
+        @capture_snapshot = !!capture_snapshot
+        @max_capture_depth = max_capture_depth
+
+        # These checks use instance methods that have more complex logic
+        # than checking a single argument value. To avoid duplicating
+        # the logic here, use the methods and perform these checks after
+        # instance variable assignment.
+        unless method? || line?
+          raise ArgumentError, "Unhandled probe type: neither method nor line probe: #{id}"
+        end
+
+        @rate_limit = rate_limit || (@capture_snapshot ? 1 : 5000)
+        @rate_limiter = Datadog::Core::TokenBucket.new(@rate_limit)
+      end
+
+      attr_reader :id
+      attr_reader :type
+      attr_reader :file
+      attr_reader :line_no
+      attr_reader :type_name
+      attr_reader :method_name
+      attr_reader :template
+
+      # Configured maximum capture depth. Can be nil in which case
+      # the global default will be used.
+      attr_reader :max_capture_depth
+
+      # Rate limit in effect, in invocations per second. Always present.
+      attr_reader :rate_limit
+
+      # Rate limiter object. For internal DI use only.
+      attr_reader :rate_limiter
+
+      def capture_snapshot?
+        @capture_snapshot
+      end
+
+      # Returns whether the probe is a line probe.
+      #
+      # Method probes may still specify a file name (to aid in locating the
+      # method or for stack traversal purposes?), therefore we do not check
+      # for file name/path presence here and just consider the line number.
+      def line?
+        !line_no.nil?
+      end
+
+      # Returns whether the probe is a method probe.
+      def method?
+        !!(type_name && method_name)
+      end
+
+      # Returns the line number associated with the probe, raising
+      # Error::MissingLineNumber if the probe does not have a line number
+      # associated with it.
+      #
+      # This method is used by instrumentation driver to ensure a line number
+      # that is passed into the instrumentation logic is actually a line number
+      # and not nil.
+      def line_no!
+        if line_no.nil?
+          raise Error::MissingLineNumber, "Probe #{id} does not have a line number associated with it"
+        end
+        line_no
+      end
+
+      # Source code location of the probe, for diagnostic reporting.
+      def location
+        if method?
+          "#{type_name}.#{method_name}"
+        elsif line?
+          "#{file}:#{line_no}"
+        else
+          # This case should not be possible because constructor verifies that
+          # the probe is a method or a line probe.
+          raise NotImplementedError
+        end
+      end
+    end
+  end
+end

data/lib/datadog/di/probe_builder.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+require_relative "probe"
+
+module Datadog
+  module DI
+    # Creates Probe instances from remote configuration payloads.
+    #
+    # Due to the dynamic instrumentation product evolving over time,
+    # it is possible that the payload corresponds to a type of probe that the
+    # current version of the library does not handle.
+    # For now ArgumentError is raised in such cases (by ProbeBuilder or
+    # Probe constructor), since generally DI is meant to rescue all exceptions
+    # internally and not propagate any exceptions to applications.
+    # A dedicated exception could be added in the future if there is a use case
+    # for it.
+    #
+    # @api private
+    module ProbeBuilder
+      module_function def build_from_remote_config(config)
+        # The validations here are not yet comprehensive.
+        Probe.new(
+          id: config.fetch("id"),
+          type: config.fetch("type"),
+          file: config["where"]&.[]("sourceFile"),
+          # Sometimes lines are sometimes received as an array of nil
+          # for some reason.
+          line_no: config["where"]&.[]("lines")&.compact&.map(&:to_i)&.first,
+          type_name: config["where"]&.[]("typeName"),
+          method_name: config["where"]&.[]("methodName"),
+          template: config["template"],
+          capture_snapshot: !!config["captureSnapshot"],
+          max_capture_depth: config["capture"]&.[]("maxReferenceDepth"),
+          rate_limit: config["sampling"]&.[]("snapshotsPerSecond"),
+        )
+      rescue KeyError => exc
+        raise ArgumentError, "Malformed remote configuration entry for probe: #{exc.class}: #{exc}: #{config}"
+      end
+    end
+  end
+end

data/lib/datadog/di/redactor.rb

@@ -0,0 +1,188 @@
+# frozen_string_literal: true
+
+module Datadog
+  module DI
+    # Provides logic to identify sensitive information in snapshots captured
+    # by dynamic instrumentation.
+    #
+    # Redaction can be performed based on identifier or attribute name,
+    # or class name of said identifier or attribute. Redaction does not take
+    # into account variable values.
+    #
+    # There is a built-in list of identifier names which will be subject to
+    # redaction. Additional names can be provided by the user via the
+    # settings.dynamic_instrumentation.redacted_identifiers setting or
+    # the DD_DYNAMIC_INSTRUMENTATION_REDACTED_IDENTIFIERS environment
+    # variable. Currently no class names are subject to redaction by default;
+    # class names can be provided via the
+    # settings.dynamic_instrumentation.redacted_type_names setting or
+    # DD_DYNAMIC_INSTRUMENTATION_REDACTED_TYPES environment variable.
+    #
+    # Redacted identifiers must match exactly to an attribute name, a key
+    # in a hash or a variable name. Redacted types can either be matched
+    # exactly or, if the name is suffixed with an asterisk (*), any class
+    # whose name contains the specified prefix will be subject to redaction.
+    #
+    # When specifying class (type) names to be redacted, user must specify
+    # fully-qualified names. For example, if `Token` or `Token*` are
+    # specified to be redacted, instances of ::Token will be redacted
+    # but instances of ::Foo::Token will not be. To redact the latter,
+    # specify `Foo::Token` or `::Foo::Token` as redacted types.
+    #
+    # This class does not perform redaction itself (i.e., value replacement
+    # with a placeholder). This replacement is performed by Serializer.
+    #
+    # @api private
+    class Redactor
+      def initialize(settings)
+        @settings = settings
+      end
+
+      attr_reader :settings
+
+      def redact_identifier?(name)
+        redacted_identifiers.include?(normalize(name))
+      end
+
+      def redact_type?(value)
+        # Classses can be nameless, do not attempt to redact in that case.
+        if (cls_name = value.class.name)
+          redacted_type_names_regexp.match?(cls_name)
+        else
+          false
+        end
+      end
+
+      private
+
+      def redacted_identifiers
+        @redacted_identifiers ||= begin
+          names = DEFAULT_REDACTED_IDENTIFIERS + settings.dynamic_instrumentation.redacted_identifiers
+          names.map! do |name|
+            normalize(name)
+          end
+          Set.new(names)
+        end
+      end
+
+      def redacted_type_names_regexp
+        @redacted_type_names_regexp ||= begin
+          names = settings.dynamic_instrumentation.redacted_type_names
+          names = names.map do |name|
+            if name.start_with?("::")
+              # :: prefix is redundant, all names are expected to be
+              # fully-qualified.
+              #
+              # Defaulting to empty string is for steep.
+              name = name[2...name.length] || ""
+            end
+            if name.end_with?("*")
+              # Defaulting to empty string is for steep.
+              name = name[0..-2] || ""
+              suffix = ".*"
+            else
+              suffix = ""
+            end
+            Regexp.escape(name) + suffix
+          end.join("|")
+          Regexp.new("\\A(?:#{names})\\z")
+        end
+      end
+
+      # Copied from dd-trace-py
+      DEFAULT_REDACTED_IDENTIFIERS = [
+        "2fa",
+        "accesstoken",
+        "aiohttpsession",
+        "apikey",
+        "apisecret",
+        "apisignature",
+        "appkey",
+        "applicationkey",
+        "auth",
+        "authorization",
+        "authtoken",
+        "ccnumber",
+        "certificatepin",
+        "cipher",
+        "clientid",
+        "clientsecret",
+        "connectionstring",
+        "connectsid",
+        "cookie",
+        "credentials",
+        "creditcard",
+        "csrf",
+        "csrftoken",
+        "cvv",
+        "databaseurl",
+        "dburl",
+        "encryptionkey",
+        "encryptionkeyid",
+        "env",
+        "geolocation",
+        "gpgkey",
+        "ipaddress",
+        "jti",
+        "jwt",
+        "licensekey",
+        "masterkey",
+        "mysqlpwd",
+        "nonce",
+        "oauth",
+        "oauthtoken",
+        "otp",
+        "passhash",
+        "passwd",
+        "password",
+        "passwordb",
+        "pemfile",
+        "pgpkey",
+        "phpsessid",
+        "pin",
+        "pincode",
+        "pkcs8",
+        "privatekey",
+        "publickey",
+        "pwd",
+        "recaptchakey",
+        "refreshtoken",
+        "routingnumber",
+        "salt",
+        "secret",
+        "secretkey",
+        "secrettoken",
+        "securityanswer",
+        "securitycode",
+        "securityquestion",
+        "serviceaccountcredentials",
+        "session",
+        "sessionid",
+        "sessionkey",
+        "setcookie",
+        "signature",
+        "signaturekey",
+        "sshkey",
+        "ssn",
+        "symfony",
+        "token",
+        "transactionid",
+        "twiliotoken",
+        "usersession",
+        "voterid",
+        "xapikey",
+        "xauthtoken",
+        "xcsrftoken",
+        "xforwardedfor",
+        "xrealip",
+        "xsrf",
+        "xsrftoken",
+      ]
+
+      # Input can be a string or a symbol.
+      def normalize(str)
+        str.to_s.strip.downcase.gsub(/[-_$@]/, "")
+      end
+    end
+  end
+end