ddtrace 0.12.0 → 0.13.0

data/lib/ddtrace/contrib/elasticsearch/quantize.rb

@@ -4,28 +4,26 @@ module Datadog
       # Quantize contains ES-specific resource quantization tools.
       module Quantize
         PLACEHOLDER = '?'.freeze
+        ID_PLACEHOLDER = '\1?'.freeze
         EXCLUDE_KEYS = [].freeze
         SHOW_KEYS = [:_index, :_type, :_id].freeze
-        DEFAULT_OPTIONS = { exclude: EXCLUDE_KEYS, show: SHOW_KEYS }.freeze
-
-        ID_REGEXP = %r{\/([0-9]+)([\/\?]|$)}
-        ID_PLACEHOLDER = '/?\2'.freeze
-
-        INDEX_REGEXP = /[0-9]{2,}/
-        INDEX_PLACEHOLDER = '?'.freeze
+        DEFAULT_OPTIONS = {
+          exclude: EXCLUDE_KEYS,
+          show: SHOW_KEYS,
+          placeholder: PLACEHOLDER
+        }.freeze
 
         module_function
 
-        # Very basic quantization, complex processing should be done in the agent
         def format_url(url)
-          quantized_url = url.gsub(ID_REGEXP, ID_PLACEHOLDER)
-          quantized_url.gsub(INDEX_REGEXP, INDEX_PLACEHOLDER)
+          sanitize_fragment_with_id(url)
+            .gsub(/(?:[\d]+)/, PLACEHOLDER)
         end
 
         def format_body(body, options = {})
           format_body!(body, options)
         rescue StandardError
-          PLACEHOLDER
+          options[:placeholder] || PLACEHOLDER
         end
 
         def format_body!(body, options = {})
@@ -36,48 +34,12 @@ module Datadog
 
           # Parse each statement and quantize them.
           statements.collect do |string|
-            reserialize_json(string) do |obj|
-              format_statement(obj, options)
+            reserialize_json(string, options[:placeholder]) do |obj|
+              Datadog::Quantization::Hash.format(obj, options)
             end
           end.join("\n")
         end
 
-        def format_statement(statement, options = {})
-          return statement if options[:show] == :all
-
-          case statement
-          when Hash
-            statement.each_with_object({}) do |(key, value), quantized|
-              if options[:show].include?(key.to_sym)
-                quantized[key] = value
-              elsif !options[:exclude].include?(key.to_sym)
-                quantized[key] = format_value(value, options)
-              end
-            end
-          else
-            format_value(statement, options)
-          end
-        end
-
-        def format_value(value, options = {})
-          return value if options[:show] == :all
-
-          case value
-          when Hash
-            format_statement(value, options)
-          when Array
-            # If any are objects, format them.
-            if value.any? { |v| v.class <= Hash || v.class <= Array }
-              value.collect { |i| format_value(i, options) }
-            # Otherwise short-circuit and return single placeholder
-            else
-              PLACEHOLDER
-            end
-          else
-            PLACEHOLDER
-          end
-        end
-
         def merge_options(original, additional)
           {}.tap do |options|
             # Show
@@ -105,6 +67,13 @@ module Datadog
             fail_value
           end
         end
+
+        # Sanitizes URL fragment by changing it to ? whenever a number is detected
+        # This is meant as simple heuristic that attempts to detect if particular fragment
+        # represents document Id. This is meant to reduce the cardinality in most frequent cases.
+        def sanitize_fragment_with_id(url)
+          url.gsub(%r{^(/?[^/]*/[^/]*/)(?:[^\?/\d]*[\d]+[^\?/]*)}, ID_PLACEHOLDER)
+        end
       end
     end
   end

data/lib/ddtrace/contrib/excon/middleware.rb

@@ -0,0 +1,139 @@
+require 'excon'
+require 'ddtrace/ext/http'
+require 'ddtrace/ext/net'
+require 'ddtrace/ext/distributed'
+require 'ddtrace/propagation/http_propagator'
+
+module Datadog
+  module Contrib
+    module Excon
+      # Middleware implements an excon-middleware for ddtrace instrumentation
+      class Middleware < ::Excon::Middleware::Base
+        SPAN_NAME = 'excon.request'.freeze
+        DEFAULT_ERROR_HANDLER = lambda do |response|
+          Ext::HTTP::ERROR_RANGE.cover?(response[:status])
+        end
+
+        def initialize(stack, options = {})
+          super(stack)
+          @options = Datadog.configuration[:excon].merge(options)
+        end
+
+        def request_call(datum)
+          begin
+            unless datum.key?(:datadog_span)
+              tracer.trace(SPAN_NAME).tap do |span|
+                datum[:datadog_span] = span
+                annotate!(span, datum)
+                propagate!(span, datum) if distributed_tracing?
+              end
+            end
+          rescue StandardError => e
+            Datadog::Tracer.log.debug(e.message)
+          end
+
+          @stack.request_call(datum)
+        end
+
+        def response_call(datum)
+          @stack.response_call(datum).tap do |d|
+            handle_response(d)
+          end
+        end
+
+        def error_call(datum)
+          handle_response(datum)
+          @stack.error_call(datum)
+        end
+
+        # Returns a child class of this trace middleware
+        # With options given as defaults.
+        def self.with(options = {})
+          Class.new(self) do
+            @options = options
+
+            # rubocop:disable Style/TrivialAccessors
+            def self.options
+              @options
+            end
+
+            def initialize(stack)
+              super(stack, self.class.options)
+            end
+          end
+        end
+
+        # Returns a copy of the default stack with the trace middleware injected
+        def self.around_default_stack
+          ::Excon.defaults[:middlewares].dup.tap do |default_stack|
+            # If the default stack contains a version of the trace middleware already...
+            existing_trace_middleware = default_stack.find { |m| m <= Middleware }
+            default_stack.delete(existing_trace_middleware) if existing_trace_middleware
+
+            # Inject after the ResponseParser middleware
+            response_middleware_index = default_stack.index(::Excon::Middleware::ResponseParser).to_i
+            default_stack.insert(response_middleware_index + 1, self)
+          end
+        end
+
+        private
+
+        def tracer
+          @options[:tracer]
+        end
+
+        def distributed_tracing?
+          @options[:distributed_tracing] == true && tracer.enabled
+        end
+
+        def error_handler
+          @options[:error_handler] || DEFAULT_ERROR_HANDLER
+        end
+
+        def split_by_domain?
+          @options[:split_by_domain] == true
+        end
+
+        def annotate!(span, datum)
+          span.resource = datum[:method].to_s.upcase
+          span.service = service_name(datum)
+          span.span_type = Ext::HTTP::TYPE
+          span.set_tag(Ext::HTTP::URL, datum[:path])
+          span.set_tag(Ext::HTTP::METHOD, datum[:method].to_s.upcase)
+          span.set_tag(Ext::NET::TARGET_HOST, datum[:host])
+          span.set_tag(Ext::NET::TARGET_PORT, datum[:port].to_s)
+        end
+
+        def handle_response(datum)
+          if datum.key?(:datadog_span)
+            datum[:datadog_span].tap do |span|
+              return span if span.finished?
+
+              if datum.key?(:response)
+                response = datum[:response]
+                if error_handler.call(response)
+                  span.set_error(["Error #{response[:status]}", response[:body]])
+                end
+                span.set_tag(Ext::HTTP::STATUS_CODE, response[:status])
+              end
+              span.set_error(datum[:error]) if datum.key?(:error)
+              span.finish
+              datum.delete(:datadog_span)
+            end
+          end
+        rescue StandardError => e
+          Datadog::Tracer.log.debug(e.message)
+        end
+
+        def propagate!(span, datum)
+          Datadog::HTTPPropagator.inject!(span.context, datum[:headers])
+        end
+
+        def service_name(datum)
+          # TODO: Change this to implement more sensible multiplexing
+          split_by_domain? ? datum[:host] : @options[:service_name]
+        end
+      end
+    end
+  end
+end

data/lib/ddtrace/contrib/excon/patcher.rb

@@ -0,0 +1,50 @@
+require 'ddtrace/ext/app_types'
+
+module Datadog
+  module Contrib
+    module Excon
+      # Responsible for hooking the instrumentation into Excon
+      module Patcher
+        include Base
+
+        DEFAULT_SERVICE = 'excon'.freeze
+
+        register_as :excon
+        option :tracer, default: Datadog.tracer
+        option :service_name, default: DEFAULT_SERVICE
+        option :distributed_tracing, default: false
+        option :split_by_domain, default: false
+        option :error_handler, default: nil
+
+        @patched = false
+
+        module_function
+
+        def patch
+          return @patched if patched? || !compatible?
+
+          require 'ddtrace/contrib/excon/middleware'
+
+          add_middleware
+
+          @patched = true
+        rescue => e
+          Tracer.log.error("Unable to apply Excon integration: #{e}")
+          @patched
+        end
+
+        def patched?
+          @patched
+        end
+
+        def compatible?
+          defined?(::Excon)
+        end
+
+        def add_middleware
+          ::Excon.defaults[:middlewares] = Middleware.around_default_stack
+        end
+      end
+    end
+  end
+end

data/lib/ddtrace/contrib/grpc/datadog_interceptor/client.rb

@@ -0,0 +1,49 @@
+module Datadog
+  module Contrib
+    module GRPC
+      module DatadogInterceptor
+        # The DatadogInterceptor::Client implements the tracing strategy
+        # for gRPC client-side endpoitns. This middleware compoent will
+        # inject trace context information into gRPC metadata prior to
+        # sending the request to the server.
+        class Client < Base
+          def trace(keywords)
+            keywords[:metadata] ||= {}
+
+            options = {
+              span_type: Datadog::Ext::GRPC::TYPE,
+              service: datadog_pin.service_name,
+              resource: format_resource(keywords[:method])
+            }
+
+            tracer.trace('grpc.client', options) do |span|
+              annotate!(span, keywords[:metadata])
+
+              yield
+            end
+          end
+
+          private
+
+          def annotate!(span, metadata)
+            metadata.each do |header, value|
+              span.set_tag(header, value)
+            end
+
+            Datadog::GRPCPropagator
+              .inject!(span.context, metadata)
+          rescue StandardError => e
+            Datadog::Tracer.log.debug("GRPC client trace failed: #{e}")
+          end
+
+          def format_resource(proto_method)
+            proto_method.downcase
+                        .split('/')
+                        .reject(&:empty?)
+                        .join('.')
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ddtrace/contrib/grpc/datadog_interceptor/server.rb

@@ -0,0 +1,66 @@
+module Datadog
+  module Contrib
+    module GRPC
+      module DatadogInterceptor
+        # The DatadogInterceptor::Server implements the tracing strategy
+        # for gRPC server-side endpoints. When the datadog fields have been
+        # added to the gRPC call metadata, this middleware component will
+        # extract any client-side tracing information, attempting to associate
+        # its tracing context with a parent client-side context
+        class Server < Base
+          def trace(keywords)
+            options = {
+              span_type: Datadog::Ext::GRPC::TYPE,
+              service: datadog_pin.service_name,
+              resource: format_resource(keywords[:method])
+            }
+            metadata = keywords[:call].metadata
+
+            set_distributed_context!(tracer, metadata)
+
+            tracer.trace('grpc.service', options) do |span|
+              annotate!(span, metadata)
+
+              yield
+            end
+          end
+
+          private
+
+          def set_distributed_context!(tracer, metadata)
+            tracer.provider.context = Datadog::GRPCPropagator
+                                      .extract(metadata)
+          rescue StandardError => e
+            Datadog::Tracer.log.debug(
+              "unable to propagate GRPC metadata to context: #{e}"
+            )
+          end
+
+          def annotate!(span, metadata)
+            metadata.each do |header, value|
+              next if reserved_headers.include?(header)
+              span.set_tag(header, value)
+            end
+          rescue StandardError => e
+            Datadog::Tracer.log.debug("GRPC client trace failed: #{e}")
+          end
+
+          def reserved_headers
+            [Datadog::Ext::DistributedTracing::GRPC_METADATA_TRACE_ID,
+             Datadog::Ext::DistributedTracing::GRPC_METADATA_PARENT_ID,
+             Datadog::Ext::DistributedTracing::GRPC_METADATA_SAMPLING_PRIORITY]
+          end
+
+          def format_resource(proto_method)
+            proto_method.owner
+                        .to_s
+                        .downcase
+                        .split('::')
+                        .<<(proto_method.name)
+                        .join('.')
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ddtrace/contrib/grpc/datadog_interceptor.rb

@@ -0,0 +1,65 @@
+module Datadog
+  module Contrib
+    module GRPC
+      # :nodoc:
+      module DatadogInterceptor
+        # :nodoc:
+        class Base < ::GRPC::Interceptor
+          attr_accessor :datadog_pin
+
+          def initialize(options = {})
+            datadog_pin_configuration { |c| yield(c) if block_given? }
+          end
+
+          def request_response(**keywords)
+            trace(keywords) { yield }
+          end
+
+          def client_streamer(**keywords)
+            trace(keywords) { yield }
+          end
+
+          def server_streamer(**keywords)
+            trace(keywords) { yield }
+          end
+
+          def bidi_streamer(**keywords)
+            trace(keywords) { yield }
+          end
+
+          private
+
+          def datadog_pin_configuration
+            pin = default_datadog_pin
+
+            if block_given?
+              pin = Pin.new(
+                pin.service_name,
+                app: pin.app,
+                app_type: pin.app_type,
+                tracer: pin.tracer
+              )
+
+              yield(pin)
+            end
+
+            pin.onto(self)
+
+            pin
+          end
+
+          def default_datadog_pin
+            Pin.get_from(::GRPC)
+          end
+
+          def tracer
+            datadog_pin.tracer
+          end
+        end
+
+        require_relative 'datadog_interceptor/client'
+        require_relative 'datadog_interceptor/server'
+      end
+    end
+  end
+end

data/lib/ddtrace/contrib/grpc/intercept_with_datadog.rb

@@ -0,0 +1,49 @@
+require_relative 'datadog_interceptor'
+
+module Datadog
+  module Contrib
+    module GRPC
+      # :nodoc:
+      # The `#intercept!` method is implemented in gRPC; this module
+      # will be prepended to the original class, effectively injecting
+      # our tracing middleware into the head of the call chain.
+      module InterceptWithDatadog
+        def intercept!(type, args = {})
+          if should_prepend?
+            datadog_interceptor = choose_datadog_interceptor(args)
+
+            @interceptors.unshift(datadog_interceptor.new) if datadog_interceptor
+
+            @trace_started = true
+          end
+
+          super
+        end
+
+        private
+
+        def should_prepend?
+          !trace_started? && !already_prepended?
+        end
+
+        def trace_started?
+          defined?(@trace_started) && @trace_started
+        end
+
+        def already_prepended?
+          @interceptors.any? do |interceptor|
+            interceptor.class.ancestors.include?(Datadog::Contrib::GRPC::DatadogInterceptor::Base)
+          end
+        end
+
+        def choose_datadog_interceptor(args)
+          if args.key?(:metadata)
+            Datadog::Contrib::GRPC::DatadogInterceptor::Client
+          elsif args.key?(:call)
+            Datadog::Contrib::GRPC::DatadogInterceptor::Server
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ddtrace/contrib/grpc/patcher.rb

@@ -0,0 +1,62 @@
+# requirements should be kept minimal as Patcher is a shared requirement.
+
+module Datadog
+  module Contrib
+    module GRPC
+      SERVICE = 'grpc'.freeze
+
+      # Patcher enables patching of 'grpc' module.
+      module Patcher
+        include Base
+        register_as :grpc, auto_patch: true
+        option :tracer, default: Datadog.tracer
+        option :service_name, default: SERVICE
+
+        @patched = false
+
+        module_function
+
+        def patch
+          return false unless compatible?
+          return @patched if @patched
+
+          require 'ddtrace/ext/grpc'
+          require 'ddtrace/propagation/grpc_propagator'
+          require 'ddtrace/contrib/grpc/datadog_interceptor'
+          require 'ddtrace/contrib/grpc/intercept_with_datadog'
+
+          add_pin
+          prepend_interceptor
+
+          @patched = true
+        rescue StandardError => e
+          Datadog::Tracer.log.error("Unable to apply gRPC integration: #{e}")
+        ensure
+          @patched
+        end
+
+        def compatible?
+          defined?(::GRPC::VERSION) && Gem::Version.new(::GRPC::VERSION) >= Gem::Version.new('0.10.0')
+        end
+
+        def patched?
+          @patched
+        end
+
+        def add_pin
+          Pin.new(
+            get_option(:service_name),
+            app: 'grpc',
+            app_type: 'grpc',
+            tracer: get_option(:tracer)
+          ).onto(::GRPC)
+        end
+
+        def prepend_interceptor
+          ::GRPC::InterceptionContext
+            .prepend(Datadog::Contrib::GRPC::InterceptWithDatadog)
+        end
+      end
+    end
+  end
+end

data/lib/ddtrace/contrib/http/patcher.rb

@@ -48,6 +48,8 @@ module Datadog
         include Base
         register_as :http, auto_patch: true
         option :distributed_tracing, default: false
+        option :service_name, default: SERVICE
+        option :tracer, default: Datadog.tracer
 
         @patched = false
 
@@ -64,7 +66,7 @@ module Datadog
               require 'ddtrace/ext/net'
               require 'ddtrace/ext/distributed'
 
-              patch_http()
+              patch_http
 
               @patched = true
             rescue StandardError => e
@@ -74,7 +76,7 @@ module Datadog
           @patched
         end
 
-        # patched? tells wether patch has been successfully applied
+        # patched? tells whether patch has been successfully applied
         def patched?
           @patched
         end
@@ -84,27 +86,27 @@ module Datadog
         # rubocop:disable Metrics/AbcSize
         def patch_http
           ::Net::HTTP.class_eval do
-            alias_method :initialize_without_datadog, :initialize
-            Datadog::Patcher.without_warnings do
-              remove_method :initialize
-            end
-
-            def initialize(*args)
-              pin = Datadog::Pin.new(SERVICE, app: APP, app_type: Datadog::Ext::AppTypes::WEB)
-              pin.onto(self)
-              initialize_without_datadog(*args)
-            end
-
             alias_method :request_without_datadog, :request
             remove_method :request
 
+            def datadog_pin
+              @datadog_pindatadog_pin ||= begin
+                service = Datadog.configuration[:http][:service_name]
+                tracer = Datadog.configuration[:http][:tracer]
+
+                Datadog::Pin.new(service, app: APP, app_type: Datadog::Ext::AppTypes::WEB, tracer: tracer)
+              end
+            end
+
             def request(req, body = nil, &block) # :yield: +response+
-              pin = Datadog::Pin.get_from(self)
+              pin = datadog_pin
               return request_without_datadog(req, body, &block) unless pin && pin.tracer
 
               transport = pin.tracer.writer.transport
-              return request_without_datadog(req, body, &block) if
-                Datadog::Contrib::HTTP.should_skip_tracing?(req, @address, @port, transport, pin)
+
+              if Datadog::Contrib::HTTP.should_skip_tracing?(req, @address, @port, transport, pin)
+                return request_without_datadog(req, body, &block)
+              end
 
               pin.tracer.trace(NAME) do |span|
                 begin