cw-datadog 2.23.0.2

data/lib/datadog/di/serializer.rb

@@ -0,0 +1,422 @@
+# frozen_string_literal: true
+
+require_relative "redactor"
+
+module Datadog
+  module DI
+    # Serializes captured snapshot to primitive types, which are subsequently
+    # serialized to JSON and sent to the backend.
+    #
+    # This class performs actual removal of sensitive values from the
+    # snapshots. It uses Redactor to determine which values are sensitive
+    # and need to be removed.
+    #
+    # Serializer normally ought not to invoke user (application) code,
+    # to guarantee predictable performance. However, objects like ActiveRecord
+    # models cannot be usefully serialized into primitive types without
+    # custom logic (for example, the attributes are more than 3 levels
+    # down from the top-level object which is the default capture depth,
+    # thus they won't be captured at all). To accommodate complex objects,
+    # there is an extension mechanism implemented permitting registration
+    # of serializer callbacks for arbitrary types. Applications and libraries
+    # definining such serializer callbacks should be very careful to
+    # have predictable performance and avoid exceptions and infinite loops
+    # and other such issues.
+    #
+    # All serialization methods take the names of the variables being
+    # serialized in order to be able to redact values.
+    #
+    # The result of serialization should not reference parameter values when
+    # the values are mutable (currently, this only applies to string values).
+    # Serializer will duplicate such mutable values, so that if method
+    # arguments are captured at entry and then modified during method execution,
+    # the serialized values from entry are correctly preserved.
+    # Alternatively, we could pass a parameter to the serialization methods
+    # which would control whether values are duplicated. This may be more
+    # efficient but there would be additional overhead from passing this
+    # parameter all the time and the API would get more complex.
+    #
+    # Note: "self" cannot be used as a parameter name in Ruby, therefore
+    # there should never be a conflict between instance variable
+    # serialization and method parameters.
+    #
+    # @api private
+    class Serializer
+      # Third-party library integration / custom serializers.
+      #
+      # Dynamic instrumentation has limited payload sizes, and for efficiency
+      # reasons it is not desirable to transmit data to Datadog that will
+      # never contain useful information. Additionally, due to depth limits,
+      # desired data may not even be included in payloads when serialized
+      # with the default, naive serializer. Therefore, custom objects like
+      # ActiveRecord model instances may need custom serializers.
+      #
+      # CUSTOMER NOTE: The API for defining custom serializers is not yet
+      # finalized. Please create an issue at
+      # https://github.com/datadog/dd-trace-rb/issues describing the
+      # object(s) you wish to serialize so that we can ensure your use case
+      # will be supported as the library evolves.
+      #
+      # Note that the current implementation does not permit defining a
+      # serializer for a particular class, which is the simplest use case.
+      # This is because the library itself does not need this functionality
+      # yet, and it won't help for ActiveRecord models (that derive from
+      # a common base class but are all of different classes) or for Mongoid
+      # models (that do not have a common base class at all but include a
+      # standard Mongoid module).
+      #
+      # Important: these serializers are NOT used in log messages.
+      # They are only used for variables that are captured in the snapshots.
+      @@flat_registry = []
+      def self.register(condition: nil, &block)
+        @@flat_registry << {condition: condition, proc: block}
+      end
+
+      def initialize(settings, redactor, telemetry: nil)
+        @settings = settings
+        @redactor = redactor
+        @telemetry = telemetry
+      end
+
+      attr_reader :settings
+      attr_reader :redactor
+      attr_reader :telemetry
+
+      def combine_args(args, kwargs, target_self)
+        counter = 0
+        combined = args.each_with_object({}) do |value, c|
+          counter += 1
+          # Conversion to symbol is needed here to put args ahead of
+          # kwargs when they are merged below.
+          c[:"arg#{counter}"] = value
+        end.update(kwargs)
+        combined[:self] = target_self
+        combined
+      end
+
+      # Serializes positional and keyword arguments to a method,
+      # as obtained by a method probe.
+      #
+      # UI supports a single argument list only and does not distinguish
+      # between positional and keyword arguments. We convert positional
+      # arguments to keyword arguments ("arg1", "arg2", ...) and ensure
+      # the positional arguments are listed first.
+      #
+      # Instance variables are technically a hash just like kwargs,
+      # we take them as a separate parameter to avoid a hash merge
+      # in upstream code.
+      def serialize_args(args, kwargs, target_self,
+        depth: settings.dynamic_instrumentation.max_capture_depth,
+        attribute_count: settings.dynamic_instrumentation.max_capture_attribute_count)
+        combined = combine_args(args, kwargs, target_self)
+        serialize_vars(combined, depth: depth, attribute_count: attribute_count)
+      end
+
+      # Serializes variables captured by a line probe.
+      #
+      # These are normally local variables that exist on a particular line
+      # of executed code.
+      def serialize_vars(vars,
+        depth: settings.dynamic_instrumentation.max_capture_depth,
+        attribute_count: settings.dynamic_instrumentation.max_capture_attribute_count)
+        vars.each_with_object({}) do |(k, v), agg|
+          agg[k] = serialize_value(v, name: k, depth: depth, attribute_count: attribute_count)
+        end
+      end
+
+      # Serializes a single named value.
+      #
+      # The name is needed to perform sensitive data redaction.
+      #
+      # In some cases, the value being serialized does not have a name
+      # (for example, it is the return value of a method).
+      # In this case +name+ can be nil.
+      #
+      # Returns a data structure comprised of only values of basic types
+      # (integers, strings, arrays, hashes).
+      #
+      # Respects string length, collection size and traversal depth limits.
+      def serialize_value(value, name: nil,
+        depth: settings.dynamic_instrumentation.max_capture_depth,
+        attribute_count: nil,
+        type: nil)
+        attribute_count ||= settings.dynamic_instrumentation.max_capture_attribute_count
+        cls = type || value.class
+        begin
+          if redactor.redact_type?(value)
+            return {type: class_name(cls), notCapturedReason: "redactedType"}
+          end
+
+          if name && redactor.redact_identifier?(name)
+            return {type: class_name(cls), notCapturedReason: "redactedIdent"}
+          end
+
+          @@flat_registry.each do |entry|
+            if (condition = entry[:condition]) && condition.call(value)
+              serializer_proc = entry.fetch(:proc)
+              return serializer_proc.call(self, value, name: nil, depth: depth)
+            end
+          end
+
+          serialized = {type: class_name(cls)}
+          # https://github.com/soutaro/steep/issues/1860
+          # @type var serialized: untyped
+          case value
+          when NilClass
+            serialized.update(isNull: true)
+          when Integer, Float, TrueClass, FalseClass
+            serialized.update(value: value.to_s)
+          when Time
+            # This path also handles DateTime values although they do not need
+            # to be explicitly added to the case statement.
+            serialized.update(value: value.iso8601)
+          when Date
+            serialized.update(value: value.to_s)
+          when String, Symbol
+            need_dup = false
+            value = if String === value
+              # This is the only place where we duplicate the value, currently.
+              # All other values are immutable primitives (e.g. numbers).
+              # However, do not duplicate if the string is frozen, or if
+              # it is later truncated.
+              need_dup = !value.frozen?
+              value
+            else
+              value.to_s
+            end
+            max = settings.dynamic_instrumentation.max_capture_string_length
+            if value.length > max
+              serialized.update(truncated: true, size: value.length)
+              value = value[0...max]
+              need_dup = false
+            end
+            value = value.dup if need_dup
+            serialized.update(value: value)
+          when Array
+            if depth < 0
+              serialized.update(notCapturedReason: "depth")
+            else
+              max = settings.dynamic_instrumentation.max_capture_collection_size
+              if max != 0 && value.length > max
+                serialized.update(notCapturedReason: "collectionSize", size: value.length)
+                # same steep failure with array slices.
+                # https://github.com/soutaro/steep/issues/1219
+                value = value[0...max] || []
+              end
+              entries = value.map do |elt|
+                serialize_value(elt, depth: depth - 1)
+              end
+              serialized.update(elements: entries)
+            end
+          when Hash
+            if depth < 0
+              serialized.update(notCapturedReason: "depth")
+            else
+              max = settings.dynamic_instrumentation.max_capture_collection_size
+              cur = 0
+              entries = []
+              value.each do |k, v|
+                if max != 0 && cur >= max
+                  serialized.update(notCapturedReason: "collectionSize", size: value.length)
+                  break
+                end
+                cur += 1
+                entries << [serialize_value(k, depth: depth - 1), serialize_value(v, name: k, depth: depth - 1)]
+              end
+              serialized.update(entries: entries)
+            end
+          else
+            if depth < 0
+              serialized.update(notCapturedReason: "depth")
+            else
+              fields = {}
+              cur = 0
+
+              # MRI and JRuby 9.4.5+ preserve instance variable definition
+              # order when calling #instance_variables. Previous JRuby versions
+              # did not preserve order and returned the variables in arbitrary
+              # order.
+              #
+              # The arbitrary order is problematic because 1) when there are
+              # fewer instance variables than capture limit, the order in which
+              # the variables are shown in UI will change from one capture to
+              # the next and generally will be arbitrary to the user, and
+              # 2) when there are more instance variables than capture limit,
+              # *which* variables are captured will also change meaning user
+              # looking at the UI may have "new" instance variables appear and
+              # existing ones disappear as they are looking at multiple captures.
+              #
+              # For consistency, we should have some kind of stable order of
+              # instance variables on all supported Ruby runtimes, so that the UI
+              # stays consistent. Given that initial implementation of Ruby DI
+              # does not support JRuby, we don't handle JRuby's lack of ordering
+              # of #instance_variables here, but if JRuby is supported in the
+              # future this may need to be addressed.
+              ivars = value.instance_variables
+
+              ivars.each do |ivar|
+                if cur >= attribute_count
+                  serialized.update(notCapturedReason: "fieldCount", fields: fields)
+                  break
+                end
+                cur += 1
+                fields[ivar] = serialize_value(value.instance_variable_get(ivar), name: ivar, depth: depth - 1)
+              end
+              serialized.update(fields: fields)
+            end
+          end
+          serialized
+        rescue => exc
+          telemetry&.report(exc, description: "Error serializing")
+          {type: class_name(cls), notSerializedReason: exc.to_s}
+        end
+      end
+
+      # This method is used for serializing arbitrary values into log messages.
+      # Because the output is meant to be human-readable, we cannot use
+      # the "normal" serialization format which is meant to be machine-readable.
+      # Serialize objects with depth of 1 and include the class name.
+      #
+      # Note that this method does not (currently) utilize the custom
+      # serializers that the "normal" serialization logic uses.
+      #
+      # This serializer differs from the RFC in two ways:
+      # 1. We omit the middle of long strings rather than the end,
+      #    and also the inner entries in arrays/hashes/objects.
+      # 2. We use Ruby-ish syntax for hashes and objects.
+      #
+      # We also use the Ruby-like syntax for symbols, which don't exist
+      # in other languages.
+      def serialize_value_for_message(value, depth = 1)
+        # This method is more verbose than "normal" Ruby code to avoid
+        # array allocations.
+        case value
+        when NilClass
+          'nil'
+        when Integer, Float, TrueClass, FalseClass, Time, Date
+          value.to_s
+        when String
+          serialize_string_or_symbol_for_message(value)
+        when Symbol
+          ':' + serialize_string_or_symbol_for_message(value)
+        when Array
+          return '...' if depth <= 0
+
+          max = max_capture_collection_size_for_message
+          if value.length > max
+            value_ = value[0...max - 1] || []
+            value_ << '...'
+            value_ << value[-1]
+            value = value_
+          end
+          '[' + value.map do |item|
+            serialize_value_for_message(item, depth - 1)
+          end.join(', ') + ']'
+        when Hash
+          return '...' if depth <= 0
+
+          max = max_capture_collection_size_for_message
+          keys = value.keys
+          truncated = false
+          if value.length > max
+            keys_ = keys[0...max - 1] || []
+            keys_ << keys[-1]
+            keys = keys_
+            truncated = true
+          end
+          serialized = keys.map do |key|
+            "#{serialize_value_for_message(key, depth - 1)} => #{serialize_value_for_message(value[key], depth - 1)}"
+          end
+          if truncated
+            serialized[serialized.length] = serialized[serialized.length - 1]
+            serialized[serialized.length - 2] = '...'
+          end
+          "{#{serialized.join(", ")}}"
+        else
+          return '...' if depth <= 0
+
+          vars = value.instance_variables
+          truncated = false
+          max = max_capture_attribute_count_for_message
+          if vars.length > max
+            vars_ = vars[0...max - 1] || []
+            vars_ << vars[-1]
+            truncated = true
+            vars = vars_
+          end
+          serialized = vars.map do |var|
+            # +var+ here is always the instance variable name which is a
+            # symbol, we do not need to run it through our serializer.
+            "#{var}=#{serialize_value_for_message(value.send(:instance_variable_get, var), depth - 1)}"
+          end
+          if truncated
+            serialized << serialized.last
+            serialized[-2] = '...'
+          end
+          serialized = if serialized.any?
+            ' ' + serialized.join(' ')
+          end
+          "#<#{class_name(value.class)}#{serialized}>"
+        end
+      rescue => exc
+        telemetry&.report(exc, description: "Error serializing for message")
+        # TODO class_name(foo) can also fail, which we don't handle here.
+        # Telemetry reporting could potentially also fail?
+        "#<#{class_name(value.class)}: serialization error>"
+      end
+
+      private
+
+      MAX_MESSAGE_COLLECTION_SIZE = 3
+      MAX_MESSAGE_ATTRIBUTE_COUNT = 5
+
+      def max_capture_collection_size_for_message
+        max = settings.dynamic_instrumentation.max_capture_collection_size
+        if max > MAX_MESSAGE_COLLECTION_SIZE
+          max = MAX_MESSAGE_COLLECTION_SIZE
+        end
+        max
+      end
+
+      def max_capture_attribute_count_for_message
+        max = settings.dynamic_instrumentation.max_capture_attribute_count
+        if max > MAX_MESSAGE_ATTRIBUTE_COUNT
+          max = MAX_MESSAGE_ATTRIBUTE_COUNT
+        end
+        max
+      end
+
+      # Returns the name for the specified class object.
+      #
+      # Ruby can have nameless classes, e.g. Class.new is a class object
+      # with no name. We return a placeholder for such nameless classes.
+      def class_name(cls)
+        # We could call `cls.to_s` to get the "standard" Ruby inspection of
+        # the class, but it is likely that user code can override #to_s
+        # and we don't want to invoke user code.
+        cls.name || "[Unnamed class]"
+      end
+
+      def serialize_string_or_symbol_for_message(value)
+        max = settings.dynamic_instrumentation.max_capture_string_length
+        if max > 100
+          max = 100
+        end
+        value = value.to_s
+        if (length = value.length) > max
+          if max < 5
+            value[0...max]
+          else
+            upper = length - max / 2 + 1
+            if max % 2 == 0
+              upper += 1
+            end
+            value[0...max / 2 - 1] + '...' + value[upper...length]
+          end
+        else
+          value
+        end
+      end
+    end
+  end
+end

data/lib/datadog/di/transport/diagnostics.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+require_relative '../../core/transport/parcel'
+require_relative 'http/client'
+
+module Datadog
+  module DI
+    module Transport
+      module Diagnostics
+        class EncodedParcel
+          include Datadog::Core::Transport::Parcel
+        end
+
+        class Request < Datadog::Core::Transport::Request
+        end
+
+        class Transport
+          attr_reader :client, :apis, :default_api, :current_api_id, :logger
+
+          def initialize(apis, default_api, logger:)
+            @apis = apis
+            @logger = logger
+
+            @client = HTTP::Client.new(current_api, logger: logger)
+          end
+
+          def current_api
+            @apis[HTTP::API::DIAGNOSTICS]
+          end
+
+          def send_diagnostics(payload)
+            json = JSON.dump(payload)
+            parcel = EncodedParcel.new(json)
+            request = Request.new(parcel)
+
+            response = @client.send_diagnostics_payload(request)
+            unless response.ok?
+              # TODO Datadog::Core::Transport::InternalErrorResponse
+              # does not have +code+ method, what is the actual API of
+              # these response objects?
+              raise Error::AgentCommunicationError, "send_diagnostics failed: #{begin
+                response.code
+              rescue
+                "???"
+              end}: #{response.payload}"
+            end
+          rescue Error::AgentCommunicationError
+            raise
+          # Datadog::Core::Transport does not perform any exception mapping,
+          # therefore we could have any exception here from failure to parse
+          # agent URI for example.
+          # If we ever implement retries for network errors, we should distinguish
+          # actual network errors from non-network errors that are raised by
+          # transport code.
+          rescue => exc
+            raise Error::AgentCommunicationError, "send_diagnostics failed: #{exc.class}: #{exc}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/di/transport/http/api.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require_relative '../../../core/encoding'
+require_relative '../../../core/transport/http/api/map'
+require_relative '../../../core/transport/http/api/instance'
+require_relative '../../../core/transport/http/api/spec'
+require_relative 'diagnostics'
+require_relative 'input'
+
+module Datadog
+  module DI
+    module Transport
+      module HTTP
+        # Namespace for API components
+        module API
+          # Default API versions
+          DIAGNOSTICS = 'diagnostics'
+          INPUT = 'input'
+
+          module_function
+
+          def defaults
+            Datadog::Core::Transport::HTTP::API::Map[
+              DIAGNOSTICS => Diagnostics::API::Spec.new do |s|
+                s.diagnostics = Diagnostics::API::Endpoint.new(
+                  '/debugger/v1/diagnostics',
+                  Core::Encoding::JSONEncoder,
+                )
+              end,
+              INPUT => Input::API::Spec.new do |s|
+                s.input = Input::API::Endpoint.new(
+                  '/debugger/v1/input',
+                  Core::Encoding::JSONEncoder,
+                )
+              end,
+            ]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/di/transport/http/client.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+require_relative '../../../core/transport/http/env'
+require_relative '../../../core/transport/http/response'
+
+# TODO: Decouple transport/http/client
+#
+# The standard one does `include Transport::HTTP::Statistics` and performs
+# stats updates, which may or may not be desirable in general.
+
+module Datadog
+  module DI
+    module Transport
+      module HTTP
+        # Routes, encodes, and sends DI data to the trace agent via HTTP.
+        class Client
+          attr_reader :api, :logger
+
+          def initialize(api, logger:)
+            @api = api
+            @logger = logger
+          end
+
+          def send_request(request, &block)
+            # Build request into env
+            env = build_env(request)
+
+            # Get responses from API
+            yield(api, env)
+          rescue => e
+            message =
+              "Internal error during #{self.class.name} request. Cause: #{e.class.name} #{e.message} " \
+                "Location: #{Array(e.backtrace).first}"
+
+            logger.debug(message)
+
+            Datadog::Core::Transport::InternalErrorResponse.new(e)
+          end
+
+          def build_env(request)
+            Datadog::Core::Transport::HTTP::Env.new(request)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/di/transport/http/diagnostics.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+require_relative '../../../core/transport/http/api/instance'
+require_relative '../../../core/transport/http/api/spec'
+require_relative 'client'
+
+module Datadog
+  module DI
+    module Transport
+      module HTTP
+        module Diagnostics
+          module Client
+            def send_diagnostics_payload(request)
+              send_request(request) do |api, env|
+                api.send_diagnostics(env)
+              end
+            end
+          end
+
+          module API
+            class Instance < Core::Transport::HTTP::API::Instance
+              def send_diagnostics(env)
+                raise Core::Transport::HTTP::API::Instance::EndpointNotSupportedError.new('diagnostics', self) unless spec.is_a?(Diagnostics::API::Spec)
+
+                spec.send_diagnostics(env) do |request_env|
+                  call(request_env)
+                end
+              end
+            end
+
+            class Spec < Core::Transport::HTTP::API::Spec
+              attr_accessor :diagnostics
+
+              def send_diagnostics(env, &block)
+                raise Core::Transport::HTTP::API::Spec::EndpointNotDefinedError.new('diagnostics', self) if diagnostics.nil?
+
+                diagnostics.call(env, &block)
+              end
+            end
+
+            class Endpoint < Datadog::Core::Transport::HTTP::API::Endpoint
+              attr_reader :encoder
+
+              def initialize(path, encoder)
+                super(:post, path)
+                @encoder = encoder
+              end
+
+              def call(env, &block)
+                event_payload = Core::Vendor::Multipart::Post::UploadIO.new(
+                  StringIO.new(env.request.parcel.data), 'application/json', 'event.json'
+                )
+                env.form = {'event' => event_payload}
+
+                super
+              end
+            end
+          end
+        end
+
+        HTTP::Client.include(Diagnostics::Client)
+      end
+    end
+  end
+end