datadog 2.3.0 → 2.4.0

data/lib/datadog/di/probe.rb

@@ -0,0 +1,133 @@
+# frozen_string_literal: true
+
+require_relative "error"
+require_relative "../core/rate_limiter"
+
+module Datadog
+  module DI
+    # Encapsulates probe information (as received via remote config)
+    # and state (e.g. whether the probe was installed, or executed).
+    #
+    # It is possible that remote configuration will specify an unsupported
+    # probe type or attribute, due to new DI functionality being added
+    # over time. We want to have predictable behavior in such cases, and
+    # since we can't guarantee that there will be enough information in
+    # a remote config payload to construct a functional probe, ProbeBuilder
+    # and remote config code must be prepared to deal with exceptions
+    # raised by Probe constructor in particular. Therefore, Probe constructor
+    # will raise an exception if it determines that there is not enough
+    # information (or confilcting information) in the arguments to create a
+    # functional probe, and upstream code is tasked with not spamming logs
+    # with notifications of such errors (and potentially limiting the
+    # attempts to construct probe from a given payload).
+    #
+    # Note that, while remote configuration provides line numbers as an
+    # array, the only supported line number configuration is a single line
+    # (this is the case for all languages currently). Therefore Probe
+    # only supports one line number, and ProbeBuilder is responsible for
+    # extracting that one line number out of the array received from RC.
+    #
+    # Note: only some of the parameter/attribute values are currently validated.
+    #
+    # @api private
+    class Probe
+      def initialize(id:, type:,
+        file: nil, line_no: nil, type_name: nil, method_name: nil,
+        template: nil, capture_snapshot: false, max_capture_depth: nil, rate_limit: nil)
+        # Perform some sanity checks here to detect unexpected attribute
+        # combinations, in order to not do them in subsequent code.
+        if line_no && method_name
+          raise ArgumentError, "Probe contains both line number and method name: #{id}"
+        end
+
+        if type_name && !method_name || method_name && !type_name
+          raise ArgumentError, "Partial method probe definition: #{id}"
+        end
+
+        @id = id
+        @type = type
+        @file = file
+        @line_no = line_no
+        @type_name = type_name
+        @method_name = method_name
+        @template = template
+        @capture_snapshot = !!capture_snapshot
+        @max_capture_depth = max_capture_depth
+
+        # These checks use instance methods that have more complex logic
+        # than checking a single argument value. To avoid duplicating
+        # the logic here, use the methods and perform these checks after
+        # instance variable assignment.
+        unless method? || line?
+          raise ArgumentError, "Unhandled probe type: neither method nor line probe: #{id}"
+        end
+
+        @rate_limit = rate_limit || (@capture_snapshot ? 1 : 5000)
+        @rate_limiter = Datadog::Core::TokenBucket.new(@rate_limit)
+      end
+
+      attr_reader :id
+      attr_reader :type
+      attr_reader :file
+      attr_reader :line_no
+      attr_reader :type_name
+      attr_reader :method_name
+      attr_reader :template
+
+      # Configured maximum capture depth. Can be nil in which case
+      # the global default will be used.
+      attr_reader :max_capture_depth
+
+      # Rate limit in effect, in invocations per second. Always present.
+      attr_reader :rate_limit
+
+      # Rate limiter object. For internal DI use only.
+      attr_reader :rate_limiter
+
+      def capture_snapshot?
+        @capture_snapshot
+      end
+
+      # Returns whether the probe is a line probe.
+      #
+      # Method probes may still specify a file name (to aid in locating the
+      # method or for stack traversal purposes?), therefore we do not check
+      # for file name/path presence here and just consider the line number.
+      def line?
+        !line_no.nil?
+      end
+
+      # Returns whether the probe is a method probe.
+      def method?
+        !!(type_name && method_name)
+      end
+
+      # Returns the line number associated with the probe, raising
+      # Error::MissingLineNumber if the probe does not have a line number
+      # associated with it.
+      #
+      # This method is used by instrumentation driver to ensure a line number
+      # that is passed into the instrumentation logic is actually a line number
+      # and not nil.
+      def line_no!
+        if line_no.nil?
+          raise Error::MissingLineNumber, "Probe #{id} does not have a line number associated with it"
+        end
+        line_no
+      end
+
+      # Source code location of the probe, for diagnostic reporting.
+      def location
+        if method?
+          "#{type_name}.#{method_name}"
+        elsif line?
+          "#{file}:#{line_no}"
+        else
+          # This case should not be possible because constructor verifies that
+          # the probe is a method or a line probe.
+          raise NotImplementedError
+        end
+      end
+    end
+  end
+end

data/lib/datadog/di/probe_builder.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+require_relative "probe"
+
+module Datadog
+  module DI
+    # Creates Probe instances from remote configuration payloads.
+    #
+    # Due to the dynamic instrumentation product evolving over time,
+    # it is possible that the payload corresponds to a type of probe that the
+    # current version of the library does not handle.
+    # For now ArgumentError is raised in such cases (by ProbeBuilder or
+    # Probe constructor), since generally DI is meant to rescue all exceptions
+    # internally and not propagate any exceptions to applications.
+    # A dedicated exception could be added in the future if there is a use case
+    # for it.
+    #
+    # @api private
+    module ProbeBuilder
+      module_function def build_from_remote_config(config)
+        # The validations here are not yet comprehensive.
+        Probe.new(
+          id: config.fetch("id"),
+          type: config.fetch("type"),
+          file: config["where"]&.[]("sourceFile"),
+          # Sometimes lines are sometimes received as an array of nil
+          # for some reason.
+          line_no: config["where"]&.[]("lines")&.compact&.map(&:to_i)&.first,
+          type_name: config["where"]&.[]("typeName"),
+          method_name: config["where"]&.[]("methodName"),
+          template: config["template"],
+          capture_snapshot: !!config["captureSnapshot"],
+          max_capture_depth: config["capture"]&.[]("maxReferenceDepth"),
+          rate_limit: config["sampling"]&.[]("snapshotsPerSecond"),
+        )
+      rescue KeyError => exc
+        raise ArgumentError, "Malformed remote configuration entry for probe: #{exc.class}: #{exc}: #{config}"
+      end
+    end
+  end
+end

data/lib/datadog/di/redactor.rb

@@ -0,0 +1,188 @@
+# frozen_string_literal: true
+
+module Datadog
+  module DI
+    # Provides logic to identify sensitive information in snapshots captured
+    # by dynamic instrumentation.
+    #
+    # Redaction can be performed based on identifier or attribute name,
+    # or class name of said identifier or attribute. Redaction does not take
+    # into account variable values.
+    #
+    # There is a built-in list of identifier names which will be subject to
+    # redaction. Additional names can be provided by the user via the
+    # settings.dynamic_instrumentation.redacted_identifiers setting or
+    # the DD_DYNAMIC_INSTRUMENTATION_REDACTED_IDENTIFIERS environment
+    # variable. Currently no class names are subject to redaction by default;
+    # class names can be provided via the
+    # settings.dynamic_instrumentation.redacted_type_names setting or
+    # DD_DYNAMIC_INSTRUMENTATION_REDACTED_TYPES environment variable.
+    #
+    # Redacted identifiers must match exactly to an attribute name, a key
+    # in a hash or a variable name. Redacted types can either be matched
+    # exactly or, if the name is suffixed with an asterisk (*), any class
+    # whose name contains the specified prefix will be subject to redaction.
+    #
+    # When specifying class (type) names to be redacted, user must specify
+    # fully-qualified names. For example, if `Token` or `Token*` are
+    # specified to be redacted, instances of ::Token will be redacted
+    # but instances of ::Foo::Token will not be. To redact the latter,
+    # specify `Foo::Token` or `::Foo::Token` as redacted types.
+    #
+    # This class does not perform redaction itself (i.e., value replacement
+    # with a placeholder). This replacement is performed by Serializer.
+    #
+    # @api private
+    class Redactor
+      def initialize(settings)
+        @settings = settings
+      end
+
+      attr_reader :settings
+
+      def redact_identifier?(name)
+        redacted_identifiers.include?(normalize(name))
+      end
+
+      def redact_type?(value)
+        # Classses can be nameless, do not attempt to redact in that case.
+        if (cls_name = value.class.name)
+          redacted_type_names_regexp.match?(cls_name)
+        else
+          false
+        end
+      end
+
+      private
+
+      def redacted_identifiers
+        @redacted_identifiers ||= begin
+          names = DEFAULT_REDACTED_IDENTIFIERS + settings.dynamic_instrumentation.redacted_identifiers
+          names.map! do |name|
+            normalize(name)
+          end
+          Set.new(names)
+        end
+      end
+
+      def redacted_type_names_regexp
+        @redacted_type_names_regexp ||= begin
+          names = settings.dynamic_instrumentation.redacted_type_names
+          names = names.map do |name|
+            if name.start_with?("::")
+              # :: prefix is redundant, all names are expected to be
+              # fully-qualified.
+              #
+              # Defaulting to empty string is for steep.
+              name = name[2...name.length] || ""
+            end
+            if name.end_with?("*")
+              # Defaulting to empty string is for steep.
+              name = name[0..-2] || ""
+              suffix = ".*"
+            else
+              suffix = ""
+            end
+            Regexp.escape(name) + suffix
+          end.join("|")
+          Regexp.new("\\A(?:#{names})\\z")
+        end
+      end
+
+      # Copied from dd-trace-py
+      DEFAULT_REDACTED_IDENTIFIERS = [
+        "2fa",
+        "accesstoken",
+        "aiohttpsession",
+        "apikey",
+        "apisecret",
+        "apisignature",
+        "appkey",
+        "applicationkey",
+        "auth",
+        "authorization",
+        "authtoken",
+        "ccnumber",
+        "certificatepin",
+        "cipher",
+        "clientid",
+        "clientsecret",
+        "connectionstring",
+        "connectsid",
+        "cookie",
+        "credentials",
+        "creditcard",
+        "csrf",
+        "csrftoken",
+        "cvv",
+        "databaseurl",
+        "dburl",
+        "encryptionkey",
+        "encryptionkeyid",
+        "env",
+        "geolocation",
+        "gpgkey",
+        "ipaddress",
+        "jti",
+        "jwt",
+        "licensekey",
+        "masterkey",
+        "mysqlpwd",
+        "nonce",
+        "oauth",
+        "oauthtoken",
+        "otp",
+        "passhash",
+        "passwd",
+        "password",
+        "passwordb",
+        "pemfile",
+        "pgpkey",
+        "phpsessid",
+        "pin",
+        "pincode",
+        "pkcs8",
+        "privatekey",
+        "publickey",
+        "pwd",
+        "recaptchakey",
+        "refreshtoken",
+        "routingnumber",
+        "salt",
+        "secret",
+        "secretkey",
+        "secrettoken",
+        "securityanswer",
+        "securitycode",
+        "securityquestion",
+        "serviceaccountcredentials",
+        "session",
+        "sessionid",
+        "sessionkey",
+        "setcookie",
+        "signature",
+        "signaturekey",
+        "sshkey",
+        "ssn",
+        "symfony",
+        "token",
+        "transactionid",
+        "twiliotoken",
+        "usersession",
+        "voterid",
+        "xapikey",
+        "xauthtoken",
+        "xcsrftoken",
+        "xforwardedfor",
+        "xrealip",
+        "xsrf",
+        "xsrftoken",
+      ]
+
+      # Input can be a string or a symbol.
+      def normalize(str)
+        str.to_s.strip.downcase.gsub(/[-_$@]/, "")
+      end
+    end
+  end
+end

data/lib/datadog/di/serializer.rb

@@ -0,0 +1,193 @@
+# frozen_string_literal: true
+
+require_relative "redactor"
+
+module Datadog
+  module DI
+    # Serializes captured snapshot to primitive types, which are subsequently
+    # serialized to JSON and sent to the backend.
+    #
+    # This class performs actual removal of sensitive values from the
+    # snapshots. It uses Redactor to determine which values are sensitive
+    # and need to be removed.
+    #
+    # Serializer normally ought not to invoke user (application) code,
+    # to guarantee predictable performance. However, objects like ActiveRecord
+    # models cannot be usefully serialized into primitive types without
+    # custom logic (for example, the attributes are more than 3 levels
+    # down from the top-level object which is the default capture depth,
+    # thus they won't be captured at all). To accommodate complex objects,
+    # there is an extension mechanism implemented permitting registration
+    # of serializer callbacks for arbitrary types. Applications and libraries
+    # definining such serializer callbacks should be very careful to
+    # have predictable performance and avoid exceptions and infinite loops
+    # and other such issues.
+    #
+    # All serialization methods take the names of the variables being
+    # serialized in order to be able to redact values.
+    #
+    # @api private
+    class Serializer
+      def initialize(settings, redactor)
+        @settings = settings
+        @redactor = redactor
+      end
+
+      attr_reader :settings
+      attr_reader :redactor
+
+      # Serializes positional and keyword arguments to a method,
+      # as obtained by a method probe.
+      #
+      # UI supports a single argument list only and does not distinguish
+      # between positional and keyword arguments. We convert positional
+      # arguments to keyword arguments ("arg1", "arg2", ...) and ensure
+      # the positional arguments are listed first.
+      def serialize_args(args, kwargs)
+        counter = 0
+        combined = args.each_with_object({}) do |value, c|
+          counter += 1
+          # Conversion to symbol is needed here to put args ahead of
+          # kwargs when they are merged below.
+          c[:"arg#{counter}"] = value
+        end.update(kwargs)
+        serialize_vars(combined)
+      end
+
+      # Serializes variables captured by a line probe.
+      #
+      # These are normally local variables that exist on a particular line
+      # of executed code.
+      def serialize_vars(vars)
+        vars.each_with_object({}) do |(k, v), agg|
+          agg[k] = serialize_value(v, name: k)
+        end
+      end
+
+      # Serializes a single named value.
+      #
+      # The name is needed to perform sensitive data redaction.
+      #
+      # In some cases, the value being serialized does not have a name
+      # (for example, it is the return value of a method).
+      # In this case +name+ can be nil.
+      #
+      # Returns a data structure comprised of only values of basic types
+      # (integers, strings, arrays, hashes).
+      #
+      # Respects string length, collection size and traversal depth limits.
+      def serialize_value(value, name: nil, depth: settings.dynamic_instrumentation.max_capture_depth)
+        if redactor.redact_type?(value)
+          return {type: class_name(value.class), notCapturedReason: "redactedType"}
+        end
+
+        if name && redactor.redact_identifier?(name)
+          return {type: class_name(value.class), notCapturedReason: "redactedIdent"}
+        end
+
+        serialized = {type: class_name(value.class)}
+        case value
+        when NilClass
+          serialized.update(isNull: true)
+        when Integer, Float, TrueClass, FalseClass
+          serialized.update(value: value.to_s)
+        when String, Symbol
+          value = value.to_s
+          max = settings.dynamic_instrumentation.max_capture_string_length
+          if value.length > max
+            serialized.update(truncated: true, size: value.length)
+            value = value[0...max]
+          end
+          serialized.update(value: value)
+        when Array
+          if depth < 0
+            serialized.update(notCapturedReason: "depth")
+          else
+            max = settings.dynamic_instrumentation.max_capture_collection_size
+            if max != 0 && value.length > max
+              serialized.update(notCapturedReason: "collectionSize", size: value.length)
+              # same steep failure with array slices.
+              # https://github.com/soutaro/steep/issues/1219
+              value = value[0...max] || []
+            end
+            entries = value.map do |elt|
+              serialize_value(elt, depth: depth - 1)
+            end
+            serialized.update(elements: entries)
+          end
+        when Hash
+          if depth < 0
+            serialized.update(notCapturedReason: "depth")
+          else
+            max = settings.dynamic_instrumentation.max_capture_collection_size
+            cur = 0
+            entries = []
+            value.each do |k, v|
+              if max != 0 && cur >= max
+                serialized.update(notCapturedReason: "collectionSize", size: value.length)
+                break
+              end
+              cur += 1
+              entries << [serialize_value(k, depth: depth - 1), serialize_value(v, name: k, depth: depth - 1)]
+            end
+            serialized.update(entries: entries)
+          end
+        else
+          if depth < 0
+            serialized.update(notCapturedReason: "depth")
+          else
+            fields = {}
+            max = settings.dynamic_instrumentation.max_capture_attribute_count
+            cur = 0
+
+            # MRI and JRuby 9.4.5+ preserve instance variable definition
+            # order when calling #instance_variables. Previous JRuby versions
+            # did not preserve order and returned the variables in arbitrary
+            # order.
+            #
+            # The arbitrary order is problematic because 1) when there are
+            # fewer instance variables than capture limit, the order in which
+            # the variables are shown in UI will change from one capture to
+            # the next and generally will be arbitrary to the user, and
+            # 2) when there are more instance variables than capture limit,
+            # *which* variables are captured will also change meaning user
+            # looking at the UI may have "new" instance variables appear and
+            # existing ones disappear as they are looking at multiple captures.
+            #
+            # For consistency, we should have some kind of stable order of
+            # instance variables on all supported Ruby runtimes, so that the UI
+            # stays consistent. Given that initial implementation of Ruby DI
+            # does not support JRuby, we don't handle JRuby's lack of ordering
+            # of #instance_variables here, but if JRuby is supported in the
+            # future this may need to be addressed.
+            ivars = value.instance_variables
+
+            ivars.each do |ivar|
+              if cur >= max
+                serialized.update(notCapturedReason: "fieldCount", fields: fields)
+                break
+              end
+              cur += 1
+              fields[ivar] = serialize_value(value.instance_variable_get(ivar), name: ivar, depth: depth - 1)
+            end
+            serialized.update(fields: fields)
+          end
+        end
+        serialized
+      end
+
+      private
+
+      # Returns the name for the specified class object.
+      #
+      # Ruby can have nameless classes, e.g. Class.new is a class object
+      # with no name. We return a placeholder for such nameless classes.
+      def class_name(cls)
+        # We could call `cls.to_s` to get the "standard" Ruby inspection of
+        # the class, but it is likely that user code can override #to_s
+        # and we don't want to invoke user code.
+        cls.name || "[Unnamed class]"
+      end
+    end
+  end
+end

data/lib/datadog/di.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+require_relative 'di/configuration'
+require_relative 'di/extensions'
+
+module Datadog
+  # Namespace for Datadog dynamic instrumentation.
+  #
+  # @api private
+  module DI
+    # Expose DI to global shared objects
+    Extensions.activate!
+  end
+end

data/lib/datadog/opentelemetry/sdk/propagator.rb

@@ -15,6 +15,7 @@ module Datadog
           setter: ::OpenTelemetry::Context::Propagation.text_map_setter
         )
           unless setter == ::OpenTelemetry::Context::Propagation.text_map_setter
+            # PENDING: Not to report telemetry logs for now
             Datadog.logger.error(
               'Custom setter is not supported. Please inform the `datadog` team at ' \
             ' https://github.com/DataDog/dd-trace-rb of your use case so we can best support you. Using the default ' \
@@ -31,6 +32,7 @@ module Datadog
         )
           if getter != ::OpenTelemetry::Context::Propagation.text_map_getter &&
               getter != ::OpenTelemetry::Common::Propagation.rack_env_getter
+            # PENDING: Not to report telemetry logs for now
             Datadog.logger.error(
               "Custom getter #{getter} is not supported. Please inform the `datadog` team at " \
             ' https://github.com/DataDog/dd-trace-rb of your use case so we can best support you. Using the default ' \

data/lib/datadog/profiling/collectors/cpu_and_wall_time_worker.rb

@@ -22,6 +22,7 @@ module Datadog
           dynamic_sampling_rate_overhead_target_percentage:,
           allocation_profiling_enabled:,
           allocation_counting_enabled:,
+          gvl_profiling_enabled:,
           # **NOTE**: This should only be used for testing; disabling the dynamic sampling rate will increase the
           # profiler overhead!
           dynamic_sampling_rate_enabled: true,
@@ -35,16 +36,17 @@ module Datadog
           end
 
           self.class._native_initialize(
-            self,
-            thread_context_collector,
-            gc_profiling_enabled,
-            idle_sampling_helper,
-            no_signals_workaround_enabled,
-            dynamic_sampling_rate_enabled,
-            dynamic_sampling_rate_overhead_target_percentage,
-            allocation_profiling_enabled,
-            allocation_counting_enabled,
-            skip_idle_samples_for_testing,
+            self_instance: self,
+            thread_context_collector: thread_context_collector,
+            gc_profiling_enabled: gc_profiling_enabled,
+            idle_sampling_helper: idle_sampling_helper,
+            no_signals_workaround_enabled: no_signals_workaround_enabled,
+            dynamic_sampling_rate_enabled: dynamic_sampling_rate_enabled,
+            dynamic_sampling_rate_overhead_target_percentage: dynamic_sampling_rate_overhead_target_percentage,
+            allocation_profiling_enabled: allocation_profiling_enabled,
+            allocation_counting_enabled: allocation_counting_enabled,
+            gvl_profiling_enabled: gvl_profiling_enabled,
+            skip_idle_samples_for_testing: skip_idle_samples_for_testing,
           )
           @worker_thread = nil
           @failure_exception = nil

data/lib/datadog/profiling/collectors/info.rb

@@ -2,6 +2,7 @@
 
 require "set"
 require "time"
+require "libdatadog"
 
 module Datadog
   module Profiling
@@ -62,11 +63,19 @@ module Datadog
         def collect_profiler_info(settings)
           unless @profiler_info
             lib_datadog_gem = ::Gem.loaded_specs["libdatadog"]
+
+            libdatadog_version =
+              if lib_datadog_gem
+                "#{lib_datadog_gem.version}-#{lib_datadog_gem.platform}"
+              else
+                # In some cases, Gem.loaded_specs may not be available, as in
+                # https://github.com/DataDog/dd-trace-rb/pull/1506; let's use the version directly
+                "#{Libdatadog::VERSION}-(unknown)"
+              end
+
             @profiler_info = {
-              # TODO: If profiling is extracted and its version diverges from the datadog gem, this is inaccurate.
-              #       Update if this ever occurs.
               version: Datadog::Core::Environment::Identity.gem_datadog_version,
-              libdatadog: "#{lib_datadog_gem.version}-#{lib_datadog_gem.platform}",
+              libdatadog: libdatadog_version,
               settings: collect_settings_recursively(settings.profiling),
             }.freeze
           end