ddtrace 1.0.0 → 1.10.1

data/lib/datadog/tracing/contrib/rack/middlewares.rb

@@ -1,15 +1,14 @@
-# typed: false
-
 require 'date'
 
-require 'datadog/core/environment/variable_helpers'
-require 'datadog/tracing'
-require 'datadog/tracing/metadata/ext'
-require 'datadog/tracing/propagation/http'
-require 'datadog/tracing/contrib/analytics'
-require 'datadog/tracing/contrib/rack/ext'
-require 'datadog/tracing/contrib/rack/request_queue'
-require 'datadog/tracing/contrib/utils/quantization/http'
+require_relative '../../../core/environment/variable_helpers'
+require_relative '../../client_ip'
+require_relative '../../metadata/ext'
+require_relative '../../propagation/http'
+require_relative '../analytics'
+require_relative '../utils/quantization/http'
+require_relative 'ext'
+require_relative 'header_collection'
+require_relative 'request_queue'
 
 module Datadog
   module Tracing
@@ -34,37 +33,52 @@ module Datadog
             request_start = Contrib::Rack::QueueTime.get_request_start(env)
             return if request_start.nil?
 
-            frontend_span = Tracing.trace(
-              Ext::SPAN_HTTP_SERVER_QUEUE,
-              span_type: Tracing::Metadata::Ext::HTTP::TYPE_PROXY,
-              start_time: request_start,
-              service: configuration[:web_service_name]
-            )
-
-            # Tag this span as belonging to Rack
-            frontend_span.set_tag(Tracing::Metadata::Ext::TAG_COMPONENT, Ext::TAG_COMPONENT)
-            frontend_span.set_tag(Tracing::Metadata::Ext::TAG_OPERATION, Ext::TAG_OPERATION_HTTP_SERVER_QUEUE)
-
-            # Set peer service (so its not believed to belong to this app)
-            frontend_span.set_tag(Tracing::Metadata::Ext::TAG_PEER_SERVICE, configuration[:web_service_name])
-
-            frontend_span
+            case configuration[:request_queuing]
+            when true, :include_request # DEV: Switch `true` to `:exclude_request` in v2.0
+              queue_span = trace_http_server(Ext::SPAN_HTTP_SERVER_QUEUE, start_time: request_start)
+              # Tag this span as belonging to Rack
+              queue_span.set_tag(Tracing::Metadata::Ext::TAG_COMPONENT, Ext::TAG_COMPONENT)
+              queue_span.set_tag(Tracing::Metadata::Ext::TAG_OPERATION, Ext::TAG_OPERATION_HTTP_SERVER_QUEUE)
+              queue_span.set_tag(Tracing::Metadata::Ext::TAG_KIND, Tracing::Metadata::Ext::SpanKind::TAG_SERVER)
+              queue_span
+            when :exclude_request
+              request_span = trace_http_server(Ext::SPAN_HTTP_PROXY_REQUEST, start_time: request_start)
+              request_span.set_tag(Tracing::Metadata::Ext::TAG_COMPONENT, Ext::TAG_COMPONENT_HTTP_PROXY)
+              request_span.set_tag(Tracing::Metadata::Ext::TAG_OPERATION, Ext::TAG_OPERATION_HTTP_PROXY_REQUEST)
+              request_span.set_tag(Tracing::Metadata::Ext::TAG_KIND, Tracing::Metadata::Ext::SpanKind::TAG_PROXY)
+
+              queue_span = trace_http_server(Ext::SPAN_HTTP_PROXY_QUEUE, start_time: request_start)
+
+              # Measure service stats
+              Contrib::Analytics.set_measured(queue_span)
+
+              queue_span.set_tag(Tracing::Metadata::Ext::TAG_COMPONENT, Ext::TAG_COMPONENT_HTTP_PROXY)
+              queue_span.set_tag(Tracing::Metadata::Ext::TAG_OPERATION, Ext::TAG_OPERATION_HTTP_PROXY_QUEUE)
+              queue_span.set_tag(Tracing::Metadata::Ext::TAG_KIND, Tracing::Metadata::Ext::SpanKind::TAG_PROXY)
+              # finish the `queue` span now to record only the time spent *in queue*,
+              # excluding the time spent processing the request itself
+              queue_span.finish
+
+              request_span
+            end
           end
 
           def call(env)
             # Find out if this is rack within rack
             previous_request_span = env[Ext::RACK_ENV_REQUEST_SPAN]
 
+            return @app.call(env) if previous_request_span
+
             # Extract distributed tracing context before creating any spans,
             # so that all spans will be added to the distributed trace.
-            if configuration[:distributed_tracing] && previous_request_span.nil?
+            if configuration[:distributed_tracing]
               trace_digest = Tracing::Propagation::HTTP.extract(env)
               Tracing.continue_trace!(trace_digest)
             end
 
             # Create a root Span to keep track of frontend web servers
             # (i.e. Apache, nginx) if the header is properly set
-            frontend_span = compute_queue_time(env) if previous_request_span.nil?
+            frontend_span = compute_queue_time(env)
 
             trace_options = { span_type: Tracing::Metadata::Ext::HTTP::TYPE_INBOUND }
             trace_options[:service] = configuration[:service_name] if configuration[:service_name]
@@ -113,45 +127,48 @@ module Datadog
               request_span.finish
             end
 
-            frontend_span.finish unless frontend_span.nil?
+            frontend_span.finish if frontend_span
           end
           # rubocop:enable Lint/RescueException
 
-          def resource_name_for(env, status)
-            if configuration[:middleware_names] && env['RESPONSE_MIDDLEWARE']
-              "#{env['RESPONSE_MIDDLEWARE']}##{env['REQUEST_METHOD']}"
-            else
-              "#{env['REQUEST_METHOD']} #{status}".strip
-            end
-          end
-
           # rubocop:disable Metrics/AbcSize
           # rubocop:disable Metrics/CyclomaticComplexity
           # rubocop:disable Metrics/PerceivedComplexity
           # rubocop:disable Metrics/MethodLength
           def set_request_tags!(trace, request_span, env, status, headers, response, original_env)
-            # http://www.rubydoc.info/github/rack/rack/file/SPEC
-            # The source of truth in Rack is the PATH_INFO key that holds the
-            # URL for the current request; but some frameworks may override that
-            # value, especially during exception handling.
-            #
-            # Because of this, we prefer to use REQUEST_URI, if available, which is the
-            # relative path + query string, and doesn't mutate.
-            #
-            # REQUEST_URI is only available depending on what web server is running though.
-            # So when its not available, we want the original, unmutated PATH_INFO, which
-            # is just the relative path without query strings.
-            url = env['REQUEST_URI'] || original_env['PATH_INFO']
-            request_headers = parse_request_headers(env)
-            response_headers = parse_response_headers(headers || {})
-
-            request_span.resource ||= resource_name_for(env, status)
+            request_header_collection = Header::RequestHeaderCollection.new(env)
+            request_headers_tags = parse_request_headers(request_header_collection)
+            response_headers_tags = parse_response_headers(headers || {})
+
+            # Since it could be mutated, it would be more accurate to fetch from the original env,
+            # e.g. ActionDispatch::ShowExceptions middleware with Rails exceptions_app configuration
+            original_request_method = original_env['REQUEST_METHOD']
+
+            # request_headers is subject to filtering and configuration so we
+            # get the user agent separately
+            user_agent = parse_user_agent_header(request_header_collection)
+
+            # The priority
+            # 1. User overrides span.resource
+            # 2. Configuration
+            # 3. Nested App override trace.resource
+            # 4. Fallback with verb + status, eq `GET 200`
+            request_span.resource ||=
+              if configuration[:middleware_names] && env['RESPONSE_MIDDLEWARE']
+                "#{env['RESPONSE_MIDDLEWARE']}##{original_request_method}"
+              elsif trace.resource_override?
+                trace.resource
+              else
+                "#{original_request_method} #{status}".strip
+              end
 
-            # Set trace name if it hasn't been set yet (name == resource)
-            trace.resource = request_span.resource if trace.resource == trace.name
+            # Overrides the trace resource if it never been set
+            # Otherwise, the getter method would delegate to its root span
+            trace.resource = request_span.resource unless trace.resource_override?
 
             request_span.set_tag(Tracing::Metadata::Ext::TAG_COMPONENT, Ext::TAG_COMPONENT)
             request_span.set_tag(Tracing::Metadata::Ext::TAG_OPERATION, Ext::TAG_OPERATION_REQUEST)
+            request_span.set_tag(Tracing::Metadata::Ext::TAG_KIND, Tracing::Metadata::Ext::SpanKind::TAG_SERVER)
 
             # Set analytics sample rate
             if Contrib::Analytics.enabled?(configuration[:analytics_enabled])
@@ -162,11 +179,17 @@ module Datadog
             Contrib::Analytics.set_measured(request_span)
 
             if request_span.get_tag(Tracing::Metadata::Ext::HTTP::TAG_METHOD).nil?
-              request_span.set_tag(Tracing::Metadata::Ext::HTTP::TAG_METHOD, env['REQUEST_METHOD'])
+              request_span.set_tag(Tracing::Metadata::Ext::HTTP::TAG_METHOD, original_request_method)
             end
 
+            url = parse_url(env, original_env)
+
             if request_span.get_tag(Tracing::Metadata::Ext::HTTP::TAG_URL).nil?
-              options = configuration[:quantize]
+              options = configuration[:quantize] || {}
+
+              # Quantization::HTTP.url base defaults to :show, but we are transitioning
+              options[:base] ||= :exclude
+
               request_span.set_tag(
                 Tracing::Metadata::Ext::HTTP::TAG_URL,
                 Contrib::Utils::Quantization::HTTP.url(url, options)
@@ -174,29 +197,43 @@ module Datadog
             end
 
             if request_span.get_tag(Tracing::Metadata::Ext::HTTP::TAG_BASE_URL).nil?
-              request_obj = ::Rack::Request.new(env)
+              options = configuration[:quantize]
 
-              base_url = if request_obj.respond_to?(:base_url)
-                           request_obj.base_url
-                         else
-                           # Compatibility for older Rack versions
-                           request_obj.url.chomp(request_obj.fullpath)
-                         end
+              unless options[:base] == :show
+                base_url = Contrib::Utils::Quantization::HTTP.base_url(url)
+
+                unless base_url.empty?
+                  request_span.set_tag(
+                    Tracing::Metadata::Ext::HTTP::TAG_BASE_URL,
+                    base_url
+                  )
+                end
+              end
+            end
 
-              request_span.set_tag(Tracing::Metadata::Ext::HTTP::TAG_BASE_URL, base_url)
+            if request_span.get_tag(Tracing::Metadata::Ext::HTTP::TAG_CLIENT_IP).nil?
+              Tracing::ClientIp.set_client_ip_tag(
+                request_span,
+                headers: request_header_collection,
+                remote_ip: env['REMOTE_ADDR']
+              )
             end
 
             if request_span.get_tag(Tracing::Metadata::Ext::HTTP::TAG_STATUS_CODE).nil? && status
               request_span.set_tag(Tracing::Metadata::Ext::HTTP::TAG_STATUS_CODE, status)
             end
 
+            if request_span.get_tag(Tracing::Metadata::Ext::HTTP::TAG_USER_AGENT).nil? && user_agent
+              request_span.set_tag(Tracing::Metadata::Ext::HTTP::TAG_USER_AGENT, user_agent)
+            end
+
             # Request headers
-            request_headers.each do |name, value|
+            request_headers_tags.each do |name, value|
               request_span.set_tag(name, value) if request_span.get_tag(name).nil?
             end
 
             # Response headers
-            response_headers.each do |name, value|
+            response_headers_tags.each do |name, value|
               request_span.set_tag(name, value) if request_span.get_tag(name).nil?
             end
 
@@ -215,14 +252,73 @@ module Datadog
             Datadog.configuration.tracing[:rack]
           end
 
-          def parse_request_headers(env)
-            {}.tap do |result|
-              whitelist = configuration[:headers][:request] || []
-              whitelist.each do |header|
-                rack_header = header_to_rack_header(header)
-                if env.key?(rack_header)
-                  result[Tracing::Metadata::Ext::HTTP::RequestHeaders.to_tag(header)] = env[rack_header]
-                end
+          def trace_http_server(span_name, start_time:)
+            span = Tracing.trace(
+              span_name,
+              span_type: Tracing::Metadata::Ext::HTTP::TYPE_PROXY,
+              start_time: start_time,
+              service: configuration[:web_service_name]
+            )
+
+            # Set peer service (so its not believed to belong to this app)
+            span.set_tag(Tracing::Metadata::Ext::TAG_PEER_SERVICE, configuration[:web_service_name])
+
+            span
+          end
+
+          def parse_url(env, original_env)
+            request_obj = ::Rack::Request.new(env)
+
+            # scheme, host, and port
+            base_url = if request_obj.respond_to?(:base_url)
+                         request_obj.base_url
+                       else
+                         # Compatibility for older Rack versions
+                         request_obj.url.chomp(request_obj.fullpath)
+                       end
+
+            # https://github.com/rack/rack/blob/main/SPEC.rdoc
+            #
+            # The source of truth in Rack is the PATH_INFO key that holds the
+            # URL for the current request; but some frameworks may override that
+            # value, especially during exception handling.
+            #
+            # Because of this, we prefer to use REQUEST_URI, if available, which is the
+            # relative path + query string, and doesn't mutate.
+            #
+            # REQUEST_URI is only available depending on what web server is running though.
+            # So when its not available, we want the original, unmutated PATH_INFO, which
+            # is just the relative path without query strings.
+            #
+            # SCRIPT_NAME is the first part of the request URL path, so that
+            # the application can know its virtual location. It should be
+            # prepended to PATH_INFO to reflect the correct user visible path.
+            request_uri = env['REQUEST_URI'].to_s
+            fullpath = if request_uri.empty?
+                         query_string = original_env['QUERY_STRING'].to_s
+                         path = original_env['SCRIPT_NAME'].to_s + original_env['PATH_INFO'].to_s
+
+                         query_string.empty? ? path : "#{path}?#{query_string}"
+                       else
+                         # normally REQUEST_URI starts at the path, but it
+                         # might contain the full URL in some cases (e.g WEBrick)
+                         request_uri.sub(/^#{base_url}/, '')
+                       end
+
+            base_url + fullpath
+          end
+
+          def parse_user_agent_header(headers)
+            headers.get(Tracing::Metadata::Ext::HTTP::HEADER_USER_AGENT)
+          end
+
+          def parse_request_headers(headers)
+            whitelist = configuration[:headers][:request] || []
+            whitelist.each_with_object({}) do |header, result|
+              header_value = headers.get(header)
+              unless header_value.nil?
+                header_tag = Tracing::Metadata::Ext::HTTP::RequestHeaders.to_tag(header)
+                result[header_tag] = header_value
               end
             end
           end
@@ -244,10 +340,6 @@ module Datadog
               end
             end
           end
-
-          def header_to_rack_header(name)
-            "HTTP_#{name.to_s.upcase.gsub(/[-\s]/, '_')}"
-          end
         end
       end
     end

data/lib/datadog/tracing/contrib/rack/patcher.rb

@@ -1,5 +1,3 @@
-# typed: false
-
 module Datadog
   module Tracing
     module Contrib
@@ -91,10 +89,12 @@ module Datadog
               if get_option(:application)
                 MiddlewareNamePatcher.patch
               else
-                Datadog.logger.warn(%(
+                Datadog.logger.warn(
+                  %(
                 Rack :middleware_names requires you to also pass :application.
                 Middleware names have NOT been patched; please provide :application.
-                e.g. use: :rack, middleware_names: true, application: my_rack_app).freeze)
+                e.g. use: :rack, middleware_names: true, application: my_rack_app).freeze
+                )
               end
             end
           end
@@ -102,6 +102,14 @@ module Datadog
           def get_option(option)
             Datadog.configuration.tracing[:rack].get_option(option)
           end
+
+          def patch_successful
+            MiddlewarePatcher.patch_successful || MiddlewareNamePatcher.patch_successful
+          end
+
+          def patch_error_result
+            MiddlewarePatcher.patch_error_result || MiddlewareNamePatcher.patch_error_result
+          end
         end
       end
     end

data/lib/datadog/tracing/contrib/rack/request_queue.rb

@@ -1,5 +1,3 @@
-# typed: true
-
 module Datadog
   module Tracing
     module Contrib

data/lib/datadog/tracing/contrib/rails/auto_instrument_railtie.rb

@@ -1,6 +1,4 @@
-# typed: ignore
-
-require 'datadog/tracing/contrib/auto_instrument'
+require_relative '../auto_instrument'
 
 # Railtie to include AutoInstrumentation in rails loading
 class DatadogAutoInstrumentRailtie < Rails::Railtie

data/lib/datadog/tracing/contrib/rails/configuration/settings.rb

@@ -1,6 +1,4 @@
-# typed: false
-
-require 'datadog/tracing/contrib/configuration/settings'
+require_relative '../../configuration/settings'
 
 module Datadog
   module Tracing
@@ -47,6 +45,9 @@ module Datadog
             end
 
             option :distributed_tracing, default: true
+
+            option :request_queuing, default: false
+
             option :exception_controller do |o|
               o.on_set do |value|
                 # Update ActionPack exception controller too

data/lib/datadog/tracing/contrib/rails/ext.rb

@@ -1,5 +1,3 @@
-# typed: true
-
 module Datadog
   module Tracing
     module Contrib

data/lib/datadog/tracing/contrib/rails/framework.rb

@@ -1,17 +1,14 @@
-# typed: false
-
-require 'datadog/tracing'
-require 'datadog/tracing/contrib/action_cable/integration'
-require 'datadog/tracing/contrib/action_mailer/integration'
-require 'datadog/tracing/contrib/action_pack/integration'
-require 'datadog/tracing/contrib/action_view/integration'
-require 'datadog/tracing/contrib/active_record/integration'
-require 'datadog/tracing/contrib/active_support/integration'
-require 'datadog/tracing/contrib/grape/endpoint'
-require 'datadog/tracing/contrib/lograge/integration'
-require 'datadog/tracing/contrib/rails/ext'
-require 'datadog/tracing/contrib/rails/utils'
-require 'datadog/tracing/contrib/semantic_logger/integration'
+require_relative '../action_cable/integration'
+require_relative '../action_mailer/integration'
+require_relative '../action_pack/integration'
+require_relative '../action_view/integration'
+require_relative '../active_record/integration'
+require_relative '../active_support/integration'
+require_relative '../grape/endpoint'
+require_relative '../lograge/integration'
+require_relative 'ext'
+require_relative 'utils'
+require_relative '../semantic_logger/integration'
 
 module Datadog
   module Tracing
@@ -36,18 +33,14 @@ module Datadog
             #       used to reconfigure tracer components with Rails-sourced defaults.
             #       This is a trade-off we take to get nice defaults.
             Datadog.configure do |datadog_config|
+              rails_config = datadog_config.tracing[:rails]
+
               # By default, default service would be guessed from the script
               # being executed, but here we know better, get it from Rails config.
               # Don't set this if service has been explicitly provided by the user.
-              rails_service_name = Datadog.configuration.tracing[:rails][:service_name] \
-                                    || Datadog.configuration.service_without_fallback \
-                                    || Utils.app_name
-
-              datadog_config.service ||= rails_service_name
-            end
-
-            Datadog.configure do |datadog_config|
-              rails_config = datadog_config.tracing[:rails]
+              if datadog_config.service_without_fallback.nil?
+                datadog_config.service = rails_config[:service_name] || Utils.app_name
+              end
 
               activate_rack!(datadog_config, rails_config)
               activate_action_cable!(datadog_config, rails_config)
@@ -68,7 +61,8 @@ module Datadog
               application: ::Rails.application,
               service_name: rails_config[:service_name],
               middleware_names: rails_config[:middleware_names],
-              distributed_tracing: rails_config[:distributed_tracing]
+              distributed_tracing: rails_config[:distributed_tracing],
+              request_queuing: rails_config[:request_queuing]
             )
           end
 

data/lib/datadog/tracing/contrib/rails/integration.rb

@@ -1,10 +1,8 @@
-# typed: false
+require_relative '../integration'
 
-require 'datadog/tracing/contrib/integration'
-
-require 'datadog/tracing/contrib/rails/ext'
-require 'datadog/tracing/contrib/rails/configuration/settings'
-require 'datadog/tracing/contrib/rails/patcher'
+require_relative 'ext'
+require_relative 'configuration/settings'
+require_relative 'patcher'
 
 module Datadog
   module Tracing

data/lib/datadog/tracing/contrib/rails/log_injection.rb

@@ -1,7 +1,3 @@
-# typed: false
-
-require 'datadog/tracing'
-
 module Datadog
   module Tracing
     module Contrib
@@ -22,24 +18,11 @@ module Datadog
             end
           rescue StandardError => e
             # TODO: can we use Datadog.logger at this point?
-            Datadog.logger.warn("Unable to add Datadog Trace context to ActiveSupport::TaggedLogging: #{e.message}")
+            Datadog.logger.warn(
+              "Unable to add Datadog Trace context to ActiveSupport::TaggedLogging: #{e.class.name} #{e.message}"
+            )
             false
           end
-
-          def datadog_trace_log_hash(correlation)
-            {
-              # Adds IDs as tags to log output
-              dd: {
-                # To preserve precision during JSON serialization, use strings for large numbers
-                trace_id: correlation.trace_id.to_s,
-                span_id: correlation.span_id.to_s,
-                env: correlation.env.to_s,
-                service: correlation.service.to_s,
-                version: correlation.version.to_s
-              },
-              ddsource: ['ruby']
-            }
-          end
         end
       end
     end

data/lib/datadog/tracing/contrib/rails/middlewares.rb

@@ -1,7 +1,4 @@
-# typed: true
-
-require 'datadog/tracing'
-require 'datadog/tracing/contrib/action_pack/utils'
+require_relative '../action_pack/utils'
 
 module Datadog
   module Tracing

data/lib/datadog/tracing/contrib/rails/patcher.rb

@@ -1,13 +1,10 @@
-# typed: ignore
-
-require 'datadog/core/utils/only_once'
-require 'datadog/tracing'
-require 'datadog/tracing/contrib/rack/middlewares'
-require 'datadog/tracing/contrib/rails/framework'
-require 'datadog/tracing/contrib/rails/log_injection'
-require 'datadog/tracing/contrib/rails/middlewares'
-require 'datadog/tracing/contrib/rails/utils'
-require 'datadog/tracing/contrib/semantic_logger/patcher'
+require_relative '../../../core/utils/only_once'
+require_relative '../rack/middlewares'
+require_relative 'framework'
+require_relative 'log_injection'
+require_relative 'middlewares'
+require_relative 'utils'
+require_relative '../semantic_logger/patcher'
 
 module Datadog
   module Tracing

data/lib/datadog/tracing/contrib/rails/railtie.rb

@@ -1,8 +1,6 @@
-# typed: ignore
-
-require 'datadog/tracing/contrib/rails/framework'
-require 'datadog/tracing/contrib/rails/middlewares'
-require 'datadog/tracing/contrib/rack/middlewares'
+require_relative 'framework'
+require_relative 'middlewares'
+require_relative '../rack/middlewares'
 
 module Datadog
   # Railtie class initializes

data/lib/datadog/tracing/contrib/rails/utils.rb

@@ -1,6 +1,4 @@
-# typed: false
-
-require 'datadog/tracing/contrib/analytics'
+require_relative '../analytics'
 
 module Datadog
   module Tracing

data/lib/datadog/tracing/contrib/rake/configuration/settings.rb

@@ -1,7 +1,7 @@
-# typed: false
+require 'set'
 
-require 'datadog/tracing/contrib/configuration/settings'
-require 'datadog/tracing/contrib/rake/ext'
+require_relative '../../configuration/settings'
+require_relative '../ext'
 
 module Datadog
   module Tracing
@@ -28,6 +28,19 @@ module Datadog
 
             option :quantize, default: {}
             option :service_name
+
+            # A list of rake tasks, using their string names, to be instrumented.
+            # An empty list, or not setting this option means no task is instrumented.
+            # Automatically instrumenting all Rake tasks can lead to long-running tasks
+            # causing undue memory accumulation, as the trace for such tasks is never flushed.
+            option :tasks do |o|
+              o.default { [] }
+              o.lazy
+              o.on_set do |value|
+                # DEV: It should be possible to modify the value after it's set. E.g. for normalization.
+                options[:tasks].instance_variable_set(:@value, value.map(&:to_s).to_set)
+              end
+            end
           end
         end
       end

data/lib/datadog/tracing/contrib/rake/ext.rb

@@ -1,5 +1,3 @@
-# typed: true
-
 module Datadog
   module Tracing
     module Contrib