datadog 2.3.0 → 2.5.0

data/lib/datadog/di/redactor.rb

@@ -0,0 +1,188 @@
+# frozen_string_literal: true
+
+module Datadog
+  module DI
+    # Provides logic to identify sensitive information in snapshots captured
+    # by dynamic instrumentation.
+    #
+    # Redaction can be performed based on identifier or attribute name,
+    # or class name of said identifier or attribute. Redaction does not take
+    # into account variable values.
+    #
+    # There is a built-in list of identifier names which will be subject to
+    # redaction. Additional names can be provided by the user via the
+    # settings.dynamic_instrumentation.redacted_identifiers setting or
+    # the DD_DYNAMIC_INSTRUMENTATION_REDACTED_IDENTIFIERS environment
+    # variable. Currently no class names are subject to redaction by default;
+    # class names can be provided via the
+    # settings.dynamic_instrumentation.redacted_type_names setting or
+    # DD_DYNAMIC_INSTRUMENTATION_REDACTED_TYPES environment variable.
+    #
+    # Redacted identifiers must match exactly to an attribute name, a key
+    # in a hash or a variable name. Redacted types can either be matched
+    # exactly or, if the name is suffixed with an asterisk (*), any class
+    # whose name contains the specified prefix will be subject to redaction.
+    #
+    # When specifying class (type) names to be redacted, user must specify
+    # fully-qualified names. For example, if `Token` or `Token*` are
+    # specified to be redacted, instances of ::Token will be redacted
+    # but instances of ::Foo::Token will not be. To redact the latter,
+    # specify `Foo::Token` or `::Foo::Token` as redacted types.
+    #
+    # This class does not perform redaction itself (i.e., value replacement
+    # with a placeholder). This replacement is performed by Serializer.
+    #
+    # @api private
+    class Redactor
+      def initialize(settings)
+        @settings = settings
+      end
+
+      attr_reader :settings
+
+      def redact_identifier?(name)
+        redacted_identifiers.include?(normalize(name))
+      end
+
+      def redact_type?(value)
+        # Classses can be nameless, do not attempt to redact in that case.
+        if (cls_name = value.class.name)
+          redacted_type_names_regexp.match?(cls_name)
+        else
+          false
+        end
+      end
+
+      private
+
+      def redacted_identifiers
+        @redacted_identifiers ||= begin
+          names = DEFAULT_REDACTED_IDENTIFIERS + settings.dynamic_instrumentation.redacted_identifiers
+          names.map! do |name|
+            normalize(name)
+          end
+          Set.new(names)
+        end
+      end
+
+      def redacted_type_names_regexp
+        @redacted_type_names_regexp ||= begin
+          names = settings.dynamic_instrumentation.redacted_type_names
+          names = names.map do |name|
+            if name.start_with?("::")
+              # :: prefix is redundant, all names are expected to be
+              # fully-qualified.
+              #
+              # Defaulting to empty string is for steep.
+              name = name[2...name.length] || ""
+            end
+            if name.end_with?("*")
+              # Defaulting to empty string is for steep.
+              name = name[0..-2] || ""
+              suffix = ".*"
+            else
+              suffix = ""
+            end
+            Regexp.escape(name) + suffix
+          end.join("|")
+          Regexp.new("\\A(?:#{names})\\z")
+        end
+      end
+
+      # Copied from dd-trace-py
+      DEFAULT_REDACTED_IDENTIFIERS = [
+        "2fa",
+        "accesstoken",
+        "aiohttpsession",
+        "apikey",
+        "apisecret",
+        "apisignature",
+        "appkey",
+        "applicationkey",
+        "auth",
+        "authorization",
+        "authtoken",
+        "ccnumber",
+        "certificatepin",
+        "cipher",
+        "clientid",
+        "clientsecret",
+        "connectionstring",
+        "connectsid",
+        "cookie",
+        "credentials",
+        "creditcard",
+        "csrf",
+        "csrftoken",
+        "cvv",
+        "databaseurl",
+        "dburl",
+        "encryptionkey",
+        "encryptionkeyid",
+        "env",
+        "geolocation",
+        "gpgkey",
+        "ipaddress",
+        "jti",
+        "jwt",
+        "licensekey",
+        "masterkey",
+        "mysqlpwd",
+        "nonce",
+        "oauth",
+        "oauthtoken",
+        "otp",
+        "passhash",
+        "passwd",
+        "password",
+        "passwordb",
+        "pemfile",
+        "pgpkey",
+        "phpsessid",
+        "pin",
+        "pincode",
+        "pkcs8",
+        "privatekey",
+        "publickey",
+        "pwd",
+        "recaptchakey",
+        "refreshtoken",
+        "routingnumber",
+        "salt",
+        "secret",
+        "secretkey",
+        "secrettoken",
+        "securityanswer",
+        "securitycode",
+        "securityquestion",
+        "serviceaccountcredentials",
+        "session",
+        "sessionid",
+        "sessionkey",
+        "setcookie",
+        "signature",
+        "signaturekey",
+        "sshkey",
+        "ssn",
+        "symfony",
+        "token",
+        "transactionid",
+        "twiliotoken",
+        "usersession",
+        "voterid",
+        "xapikey",
+        "xauthtoken",
+        "xcsrftoken",
+        "xforwardedfor",
+        "xrealip",
+        "xsrf",
+        "xsrftoken",
+      ]
+
+      # Input can be a string or a symbol.
+      def normalize(str)
+        str.to_s.strip.downcase.gsub(/[-_$@]/, "")
+      end
+    end
+  end
+end

data/lib/datadog/di/serializer.rb

@@ -0,0 +1,215 @@
+# frozen_string_literal: true
+
+require_relative "redactor"
+
+module Datadog
+  module DI
+    # Serializes captured snapshot to primitive types, which are subsequently
+    # serialized to JSON and sent to the backend.
+    #
+    # This class performs actual removal of sensitive values from the
+    # snapshots. It uses Redactor to determine which values are sensitive
+    # and need to be removed.
+    #
+    # Serializer normally ought not to invoke user (application) code,
+    # to guarantee predictable performance. However, objects like ActiveRecord
+    # models cannot be usefully serialized into primitive types without
+    # custom logic (for example, the attributes are more than 3 levels
+    # down from the top-level object which is the default capture depth,
+    # thus they won't be captured at all). To accommodate complex objects,
+    # there is an extension mechanism implemented permitting registration
+    # of serializer callbacks for arbitrary types. Applications and libraries
+    # definining such serializer callbacks should be very careful to
+    # have predictable performance and avoid exceptions and infinite loops
+    # and other such issues.
+    #
+    # All serialization methods take the names of the variables being
+    # serialized in order to be able to redact values.
+    #
+    # The result of serialization should not reference parameter values when
+    # the values are mutable (currently, this only applies to string values).
+    # Serializer will duplicate such mutable values, so that if method
+    # arguments are captured at entry and then modified during method execution,
+    # the serialized values from entry are correctly preserved.
+    # Alternatively, we could pass a parameter to the serialization methods
+    # which would control whether values are duplicated. This may be more
+    # efficient but there would be additional overhead from passing this
+    # parameter all the time and the API would get more complex.
+    #
+    # @api private
+    class Serializer
+      def initialize(settings, redactor)
+        @settings = settings
+        @redactor = redactor
+      end
+
+      attr_reader :settings
+      attr_reader :redactor
+
+      # Serializes positional and keyword arguments to a method,
+      # as obtained by a method probe.
+      #
+      # UI supports a single argument list only and does not distinguish
+      # between positional and keyword arguments. We convert positional
+      # arguments to keyword arguments ("arg1", "arg2", ...) and ensure
+      # the positional arguments are listed first.
+      def serialize_args(args, kwargs)
+        counter = 0
+        combined = args.each_with_object({}) do |value, c|
+          counter += 1
+          # Conversion to symbol is needed here to put args ahead of
+          # kwargs when they are merged below.
+          c[:"arg#{counter}"] = value
+        end.update(kwargs)
+        serialize_vars(combined)
+      end
+
+      # Serializes variables captured by a line probe.
+      #
+      # These are normally local variables that exist on a particular line
+      # of executed code.
+      def serialize_vars(vars)
+        vars.each_with_object({}) do |(k, v), agg|
+          agg[k] = serialize_value(v, name: k)
+        end
+      end
+
+      # Serializes a single named value.
+      #
+      # The name is needed to perform sensitive data redaction.
+      #
+      # In some cases, the value being serialized does not have a name
+      # (for example, it is the return value of a method).
+      # In this case +name+ can be nil.
+      #
+      # Returns a data structure comprised of only values of basic types
+      # (integers, strings, arrays, hashes).
+      #
+      # Respects string length, collection size and traversal depth limits.
+      def serialize_value(value, name: nil, depth: settings.dynamic_instrumentation.max_capture_depth)
+        if redactor.redact_type?(value)
+          return {type: class_name(value.class), notCapturedReason: "redactedType"}
+        end
+
+        if name && redactor.redact_identifier?(name)
+          return {type: class_name(value.class), notCapturedReason: "redactedIdent"}
+        end
+
+        serialized = {type: class_name(value.class)}
+        case value
+        when NilClass
+          serialized.update(isNull: true)
+        when Integer, Float, TrueClass, FalseClass
+          serialized.update(value: value.to_s)
+        when String, Symbol
+          need_dup = false
+          value = if String === value
+            # This is the only place where we duplicate the value, currently.
+            # All other values are immutable primitives (e.g. numbers).
+            # However, do not duplicate if the string is frozen, or if
+            # it is later truncated.
+            need_dup = !value.frozen?
+            value
+          else
+            value.to_s
+          end
+          max = settings.dynamic_instrumentation.max_capture_string_length
+          if value.length > max
+            serialized.update(truncated: true, size: value.length)
+            value = value[0...max]
+            need_dup = false
+          end
+          value = value.dup if need_dup
+          serialized.update(value: value)
+        when Array
+          if depth < 0
+            serialized.update(notCapturedReason: "depth")
+          else
+            max = settings.dynamic_instrumentation.max_capture_collection_size
+            if max != 0 && value.length > max
+              serialized.update(notCapturedReason: "collectionSize", size: value.length)
+              # same steep failure with array slices.
+              # https://github.com/soutaro/steep/issues/1219
+              value = value[0...max] || []
+            end
+            entries = value.map do |elt|
+              serialize_value(elt, depth: depth - 1)
+            end
+            serialized.update(elements: entries)
+          end
+        when Hash
+          if depth < 0
+            serialized.update(notCapturedReason: "depth")
+          else
+            max = settings.dynamic_instrumentation.max_capture_collection_size
+            cur = 0
+            entries = []
+            value.each do |k, v|
+              if max != 0 && cur >= max
+                serialized.update(notCapturedReason: "collectionSize", size: value.length)
+                break
+              end
+              cur += 1
+              entries << [serialize_value(k, depth: depth - 1), serialize_value(v, name: k, depth: depth - 1)]
+            end
+            serialized.update(entries: entries)
+          end
+        else
+          if depth < 0
+            serialized.update(notCapturedReason: "depth")
+          else
+            fields = {}
+            max = settings.dynamic_instrumentation.max_capture_attribute_count
+            cur = 0
+
+            # MRI and JRuby 9.4.5+ preserve instance variable definition
+            # order when calling #instance_variables. Previous JRuby versions
+            # did not preserve order and returned the variables in arbitrary
+            # order.
+            #
+            # The arbitrary order is problematic because 1) when there are
+            # fewer instance variables than capture limit, the order in which
+            # the variables are shown in UI will change from one capture to
+            # the next and generally will be arbitrary to the user, and
+            # 2) when there are more instance variables than capture limit,
+            # *which* variables are captured will also change meaning user
+            # looking at the UI may have "new" instance variables appear and
+            # existing ones disappear as they are looking at multiple captures.
+            #
+            # For consistency, we should have some kind of stable order of
+            # instance variables on all supported Ruby runtimes, so that the UI
+            # stays consistent. Given that initial implementation of Ruby DI
+            # does not support JRuby, we don't handle JRuby's lack of ordering
+            # of #instance_variables here, but if JRuby is supported in the
+            # future this may need to be addressed.
+            ivars = value.instance_variables
+
+            ivars.each do |ivar|
+              if cur >= max
+                serialized.update(notCapturedReason: "fieldCount", fields: fields)
+                break
+              end
+              cur += 1
+              fields[ivar] = serialize_value(value.instance_variable_get(ivar), name: ivar, depth: depth - 1)
+            end
+            serialized.update(fields: fields)
+          end
+        end
+        serialized
+      end
+
+      private
+
+      # Returns the name for the specified class object.
+      #
+      # Ruby can have nameless classes, e.g. Class.new is a class object
+      # with no name. We return a placeholder for such nameless classes.
+      def class_name(cls)
+        # We could call `cls.to_s` to get the "standard" Ruby inspection of
+        # the class, but it is likely that user code can override #to_s
+        # and we don't want to invoke user code.
+        cls.name || "[Unnamed class]"
+      end
+    end
+  end
+end

data/lib/datadog/di/transport.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+require_relative 'error'
+
+module Datadog
+  module DI
+    # Transport for sending probe statuses and snapshots to local agent.
+    #
+    # Handles encoding of the payloads into multipart posts if necessary,
+    # body formatting/encoding, setting correct headers, etc.
+    #
+    # The transport does not handle batching of statuses or snapshots -
+    # the batching should be implemented upstream of this class.
+    #
+    # Timeout settings are forwarded from agent settings to the Net adapter.
+    #
+    # The send_* methods raise Error::AgentCommunicationError on errors
+    # (network errors and HTTP protocol errors). It is the responsibility
+    # of upstream code to rescue these exceptions appropriately to prevent them
+    # from being propagated to the application.
+    #
+    # @api private
+    class Transport
+      DIAGNOSTICS_PATH = '/debugger/v1/diagnostics'
+      INPUT_PATH = '/debugger/v1/input'
+
+      def initialize(agent_settings)
+        # Note that this uses host, port, timeout and TLS flag from
+        # agent settings.
+        @client = Core::Transport::HTTP::Adapters::Net.new(agent_settings)
+      end
+
+      def send_diagnostics(payload)
+        event_payload = Core::Vendor::Multipart::Post::UploadIO.new(
+          StringIO.new(JSON.dump(payload)), 'application/json', 'event.json'
+        )
+        payload = {'event' => event_payload}
+        send_request('Probe status submission', DIAGNOSTICS_PATH, payload)
+      end
+
+      def send_input(payload)
+        send_request('Probe snapshot submission', INPUT_PATH, payload,
+          headers: {'content-type' => 'application/json'},)
+      end
+
+      private
+
+      attr_reader :client
+
+      def send_request(desc, path, payload, headers: {})
+        # steep:ignore:start
+        env = OpenStruct.new(
+          path: path,
+          form: payload,
+          headers: headers,
+        )
+        # steep:ignore:end
+        response = client.post(env)
+        unless response.ok?
+          raise Error::AgentCommunicationError, "#{desc} failed: #{response.code}: #{response.payload}"
+        end
+      rescue IOError, SystemCallError => exc
+        raise Error::AgentCommunicationError, "#{desc} failed: #{exc.class}: #{exc}"
+      end
+    end
+  end
+end

data/lib/datadog/di/utils.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Datadog
+  module DI
+    module Utils
+      # Returns whether the provided +path+ matches the user-designated
+      # file suffix (of a line probe).
+      #
+      # If suffix is an absolute path (i.e., it starts with a slash), the path
+      # must be identical for it to match.
+      #
+      # If suffix is not an absolute path, the path matches if its suffix is
+      # the provided suffix, at a path component boundary.
+      module_function def path_matches_suffix?(path, suffix)
+        if suffix.start_with?('/')
+          path == suffix
+        else
+          # Exact match is not possible here, meaning any matching path
+          # has to be longer than the suffix. Require full component matches,
+          # meaning either the first character of the suffix is a slash
+          # or the previous character in the path is a slash.
+          # For now only check for forward slashes for Unix-like OSes;
+          # backslash is a legitimate character of a file name in Unix
+          # therefore simply permitting forward or back slash is not
+          # sufficient, we need to perform an OS check to know which
+          # path separator to use.
+          !!
+          if path.length > suffix.length && path.end_with?(suffix)
+            previous_char = path[path.length - suffix.length - 1]
+            previous_char == "/" || suffix[0] == "/"
+          end
+
+          # Alternative implementation using a regular expression:
+          # !!(path =~ %r,(/|\A)#{Regexp.quote(suffix)}\z,)
+        end
+      end
+    end
+  end
+end

data/lib/datadog/di.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+require_relative 'di/error'
+require_relative 'di/configuration'
+require_relative 'di/code_tracker'
+require_relative 'di/extensions'
+require_relative 'di/instrumenter'
+require_relative 'di/probe'
+require_relative 'di/redactor'
+require_relative 'di/serializer'
+require_relative 'di/transport'
+require_relative 'di/utils'
+
+module Datadog
+  # Namespace for Datadog dynamic instrumentation.
+  #
+  # @api private
+  module DI
+    # Expose DI to global shared objects
+    Extensions.activate!
+
+    class << self
+      attr_reader :code_tracker
+
+      # Activates code tracking. Normally this method should be called
+      # when the application starts. If instrumenting third-party code,
+      # code tracking needs to be enabled before the third-party libraries
+      # are loaded. If you definitely will not be instrumenting
+      # third-party libraries, activating tracking after third-party libraries
+      # have been loaded may improve lookup performance.
+      #
+      # TODO test that activating tracker multiple times preserves
+      # existing mappings in the registry
+      def activate_tracking!
+        (@code_tracker ||= CodeTracker.new).start
+      end
+
+      # Deactivates code tracking. In normal usage of DI this method should
+      # never be called, however it is used by DI's test suite to reset
+      # state for individual tests.
+      #
+      # Note that deactivating tracking clears out the registry, losing
+      # the ability to look up files that have been loaded into the process
+      # already.
+      def deactivate_tracking!
+        code_tracker&.stop
+      end
+
+      # Returns whether code tracking is available.
+      # This method should be used instead of querying #code_tracker
+      # because the latter one may be nil.
+      def code_tracking_active?
+        code_tracker&.active? || false
+      end
+    end
+  end
+end

data/lib/datadog/opentelemetry/sdk/propagator.rb

@@ -15,6 +15,7 @@ module Datadog
           setter: ::OpenTelemetry::Context::Propagation.text_map_setter
         )
           unless setter == ::OpenTelemetry::Context::Propagation.text_map_setter
+            # PENDING: Not to report telemetry logs for now
             Datadog.logger.error(
               'Custom setter is not supported. Please inform the `datadog` team at ' \
             ' https://github.com/DataDog/dd-trace-rb of your use case so we can best support you. Using the default ' \
@@ -31,6 +32,7 @@ module Datadog
         )
           if getter != ::OpenTelemetry::Context::Propagation.text_map_getter &&
               getter != ::OpenTelemetry::Common::Propagation.rack_env_getter
+            # PENDING: Not to report telemetry logs for now
             Datadog.logger.error(
               "Custom getter #{getter} is not supported. Please inform the `datadog` team at " \
             ' https://github.com/DataDog/dd-trace-rb of your use case so we can best support you. Using the default ' \

data/lib/datadog/profiling/collectors/cpu_and_wall_time_worker.rb

@@ -22,6 +22,7 @@ module Datadog
           dynamic_sampling_rate_overhead_target_percentage:,
           allocation_profiling_enabled:,
           allocation_counting_enabled:,
+          gvl_profiling_enabled:,
           # **NOTE**: This should only be used for testing; disabling the dynamic sampling rate will increase the
           # profiler overhead!
           dynamic_sampling_rate_enabled: true,
@@ -35,16 +36,17 @@ module Datadog
           end
 
           self.class._native_initialize(
-            self,
-            thread_context_collector,
-            gc_profiling_enabled,
-            idle_sampling_helper,
-            no_signals_workaround_enabled,
-            dynamic_sampling_rate_enabled,
-            dynamic_sampling_rate_overhead_target_percentage,
-            allocation_profiling_enabled,
-            allocation_counting_enabled,
-            skip_idle_samples_for_testing,
+            self_instance: self,
+            thread_context_collector: thread_context_collector,
+            gc_profiling_enabled: gc_profiling_enabled,
+            idle_sampling_helper: idle_sampling_helper,
+            no_signals_workaround_enabled: no_signals_workaround_enabled,
+            dynamic_sampling_rate_enabled: dynamic_sampling_rate_enabled,
+            dynamic_sampling_rate_overhead_target_percentage: dynamic_sampling_rate_overhead_target_percentage,
+            allocation_profiling_enabled: allocation_profiling_enabled,
+            allocation_counting_enabled: allocation_counting_enabled,
+            gvl_profiling_enabled: gvl_profiling_enabled,
+            skip_idle_samples_for_testing: skip_idle_samples_for_testing,
           )
           @worker_thread = nil
           @failure_exception = nil

data/lib/datadog/profiling/collectors/info.rb

@@ -2,6 +2,7 @@
 
 require "set"
 require "time"
+require "libdatadog"
 
 module Datadog
   module Profiling
@@ -62,11 +63,19 @@ module Datadog
         def collect_profiler_info(settings)
           unless @profiler_info
             lib_datadog_gem = ::Gem.loaded_specs["libdatadog"]
+
+            libdatadog_version =
+              if lib_datadog_gem
+                "#{lib_datadog_gem.version}-#{lib_datadog_gem.platform}"
+              else
+                # In some cases, Gem.loaded_specs may not be available, as in
+                # https://github.com/DataDog/dd-trace-rb/pull/1506; let's use the version directly
+                "#{Libdatadog::VERSION}-(unknown)"
+              end
+
             @profiler_info = {
-              # TODO: If profiling is extracted and its version diverges from the datadog gem, this is inaccurate.
-              #       Update if this ever occurs.
               version: Datadog::Core::Environment::Identity.gem_datadog_version,
-              libdatadog: "#{lib_datadog_gem.version}-#{lib_datadog_gem.platform}",
+              libdatadog: libdatadog_version,
               settings: collect_settings_recursively(settings.profiling),
             }.freeze
           end