ddtrace 1.4.2 → 1.5.0

data/lib/datadog/tracing/contrib/aws/services.rb

@@ -3,7 +3,6 @@
 module Datadog
   module Tracing
     module Contrib
-      # rubocop:disable Metrics/ModuleLength:
       module Aws
         SERVICES = %w[
           ACM
@@ -117,7 +116,6 @@ module Datadog
           XRay
         ].freeze
       end
-      # rubocop:enable Metrics/ModuleLength:
     end
   end
 end

data/lib/datadog/tracing/contrib/ethon/easy_patch.rb

@@ -18,7 +18,6 @@ module Datadog
           end
 
           # InstanceMethods - implementing instrumentation
-          # rubocop:disable Metrics/ModuleLength
           module InstanceMethods
             include Contrib::HttpAnnotationHelper
 
@@ -168,7 +167,6 @@ module Datadog
               datadog_configuration[:analytics_sample_rate]
             end
           end
-          # rubocop:enable Metrics/ModuleLength
         end
       end
     end

data/lib/datadog/tracing/contrib/ext.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Tracing
+    module Contrib
+      # Contrib specific constants
+      module Ext
+        # @public_api
+        module DB
+          TAG_INSTANCE = 'db.instance'
+          TAG_USER = 'db.user'
+          TAG_SYSTEM = 'db.system'
+          TAG_STATEMENT = 'db.statement'
+          TAG_ROW_COUNT = 'db.row_count'
+        end
+      end
+    end
+  end
+end

data/lib/datadog/tracing/contrib/faraday/middleware.rb

@@ -39,8 +39,7 @@ module Datadog
 
           def annotate!(span, env, options)
             span.resource = resource_name(env)
-            service_name(env[:url].host, options)
-            span.service = options[:split_by_domain] ? env[:url].host : options[:service_name]
+            span.service = service_name(env[:url].host, options)
             span.span_type = Tracing::Metadata::Ext::HTTP::TYPE_OUTBOUND
 
             span.set_tag(Tracing::Metadata::Ext::TAG_COMPONENT, Ext::TAG_COMPONENT)

data/lib/datadog/tracing/contrib/grape/endpoint.rb

@@ -9,7 +9,6 @@ module Datadog
   module Tracing
     module Contrib
       module Grape
-        # rubocop:disable Metrics/ModuleLength
         # Endpoint module includes a list of subscribers to create
         # traces when a Grape endpoint is hit
         module Endpoint
@@ -245,7 +244,6 @@ module Datadog
             end
           end
         end
-        # rubocop:enable Metrics/ModuleLength
       end
     end
   end

data/lib/datadog/tracing/contrib/graphql/configuration/settings.rb

@@ -27,7 +27,7 @@ module Datadog
             end
 
             option :schemas
-            option :service_nam
+            option :service_name
           end
         end
       end

data/lib/datadog/tracing/contrib/pg/instrumentation.rb

@@ -2,6 +2,7 @@
 
 require_relative '../../metadata/ext'
 require_relative '../analytics'
+require_relative '../ext'
 require_relative 'ext'
 
 module Datadog
@@ -95,9 +96,9 @@ module Datadog
               span.set_tag(Tracing::Metadata::Ext::TAG_PEER_SERVICE, service)
               span.set_tag(Tracing::Metadata::Ext::TAG_PEER_HOSTNAME, host)
 
-              span.set_tag(Tracing::Metadata::Ext::DB::TAG_INSTANCE, db)
-              span.set_tag(Tracing::Metadata::Ext::DB::TAG_USER, user)
-              span.set_tag(Tracing::Metadata::Ext::DB::TAG_SYSTEM, Ext::SPAN_SYSTEM)
+              span.set_tag(Contrib::Ext::DB::TAG_INSTANCE, db)
+              span.set_tag(Contrib::Ext::DB::TAG_USER, user)
+              span.set_tag(Contrib::Ext::DB::TAG_SYSTEM, Ext::SPAN_SYSTEM)
 
               span.set_tag(Tracing::Metadata::Ext::NET::TAG_TARGET_HOST, host)
               span.set_tag(Tracing::Metadata::Ext::NET::TAG_TARGET_PORT, port)
@@ -106,7 +107,7 @@ module Datadog
             end
 
             def annotate_span_with_result!(span, result)
-              span.set_tag(Tracing::Metadata::Ext::DB::TAG_ROW_COUNT, result.ntuples)
+              span.set_tag(Contrib::Ext::DB::TAG_ROW_COUNT, result.ntuples)
             end
 
             def datadog_configuration

data/lib/datadog/tracing/contrib/rack/header_collection.rb

@@ -0,0 +1,35 @@
+require_relative '../../../core/header_collection'
+
+module Datadog
+  module Tracing
+    module Contrib
+      module Rack
+        # Classes and utilities for handling headers in Rack.
+        module Header
+          # An implementation of a header collection that looks up headers from a Rack environment.
+          class RequestHeaderCollection < Datadog::Core::HeaderCollection
+            # Creates a header collection from a rack environment.
+            def initialize(env)
+              super()
+              @env = env
+            end
+
+            # Gets the value of the header with the given name.
+            def get(header_name)
+              @env[Header.to_rack_header(header_name)]
+            end
+
+            # Tests whether a header with the given name exists in the environment.
+            def key?(header_name)
+              @env.key?(Header.to_rack_header(header_name))
+            end
+          end
+
+          def self.to_rack_header(name)
+            "HTTP_#{name.to_s.upcase.gsub(/[-\s]/, '_')}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/tracing/contrib/rack/middlewares.rb

@@ -3,12 +3,14 @@
 require 'date'
 
 require_relative '../../../core/environment/variable_helpers'
+require_relative '../../client_ip'
 require_relative '../../metadata/ext'
 require_relative '../../propagation/http'
 require_relative '../analytics'
+require_relative '../utils/quantization/http'
 require_relative 'ext'
+require_relative 'header_collection'
 require_relative 'request_queue'
-require_relative '../utils/quantization/http'
 
 module Datadog
   module Tracing
@@ -121,20 +123,13 @@ module Datadog
           # rubocop:disable Metrics/PerceivedComplexity
           # rubocop:disable Metrics/MethodLength
           def set_request_tags!(trace, request_span, env, status, headers, response, original_env)
-            # http://www.rubydoc.info/github/rack/rack/file/SPEC
-            # The source of truth in Rack is the PATH_INFO key that holds the
-            # URL for the current request; but some frameworks may override that
-            # value, especially during exception handling.
-            #
-            # Because of this, we prefer to use REQUEST_URI, if available, which is the
-            # relative path + query string, and doesn't mutate.
-            #
-            # REQUEST_URI is only available depending on what web server is running though.
-            # So when its not available, we want the original, unmutated PATH_INFO, which
-            # is just the relative path without query strings.
-            url = env['REQUEST_URI'] || original_env['PATH_INFO']
-            request_headers = parse_request_headers(env)
-            response_headers = parse_response_headers(headers || {})
+            request_header_collection = Header::RequestHeaderCollection.new(env)
+            request_headers_tags = parse_request_headers(request_header_collection)
+            response_headers_tags = parse_response_headers(headers || {})
+
+            # request_headers is subject to filtering and configuration so we
+            # get the user agent separately
+            user_agent = parse_user_agent_header(request_header_collection)
 
             # The priority
             # 1. User overrides span.resource
@@ -169,8 +164,14 @@ module Datadog
               request_span.set_tag(Tracing::Metadata::Ext::HTTP::TAG_METHOD, env['REQUEST_METHOD'])
             end
 
+            url = parse_url(env, original_env)
+
             if request_span.get_tag(Tracing::Metadata::Ext::HTTP::TAG_URL).nil?
-              options = configuration[:quantize]
+              options = configuration[:quantize] || {}
+
+              # Quantization::HTTP.url base defaults to :show, but we are transitioning
+              options[:base] ||= :exclude
+
               request_span.set_tag(
                 Tracing::Metadata::Ext::HTTP::TAG_URL,
                 Contrib::Utils::Quantization::HTTP.url(url, options)
@@ -178,29 +179,43 @@ module Datadog
             end
 
             if request_span.get_tag(Tracing::Metadata::Ext::HTTP::TAG_BASE_URL).nil?
-              request_obj = ::Rack::Request.new(env)
+              options = configuration[:quantize]
 
-              base_url = if request_obj.respond_to?(:base_url)
-                           request_obj.base_url
-                         else
-                           # Compatibility for older Rack versions
-                           request_obj.url.chomp(request_obj.fullpath)
-                         end
+              unless options[:base] == :show
+                base_url = Contrib::Utils::Quantization::HTTP.base_url(url)
 
-              request_span.set_tag(Tracing::Metadata::Ext::HTTP::TAG_BASE_URL, base_url)
+                unless base_url.empty?
+                  request_span.set_tag(
+                    Tracing::Metadata::Ext::HTTP::TAG_BASE_URL,
+                    base_url
+                  )
+                end
+              end
+            end
+
+            if request_span.get_tag(Tracing::Metadata::Ext::HTTP::TAG_CLIENT_IP).nil?
+              Tracing::ClientIp.set_client_ip_tag(
+                request_span,
+                headers: request_header_collection,
+                remote_ip: env['REMOTE_ADDR']
+              )
             end
 
             if request_span.get_tag(Tracing::Metadata::Ext::HTTP::TAG_STATUS_CODE).nil? && status
               request_span.set_tag(Tracing::Metadata::Ext::HTTP::TAG_STATUS_CODE, status)
             end
 
+            if request_span.get_tag(Tracing::Metadata::Ext::HTTP::TAG_USER_AGENT).nil? && user_agent
+              request_span.set_tag(Tracing::Metadata::Ext::HTTP::TAG_USER_AGENT, user_agent)
+            end
+
             # Request headers
-            request_headers.each do |name, value|
+            request_headers_tags.each do |name, value|
               request_span.set_tag(name, value) if request_span.get_tag(name).nil?
             end
 
             # Response headers
-            response_headers.each do |name, value|
+            response_headers_tags.each do |name, value|
               request_span.set_tag(name, value) if request_span.get_tag(name).nil?
             end
 
@@ -219,14 +234,57 @@ module Datadog
             Datadog.configuration.tracing[:rack]
           end
 
-          def parse_request_headers(env)
-            {}.tap do |result|
-              whitelist = configuration[:headers][:request] || []
-              whitelist.each do |header|
-                rack_header = header_to_rack_header(header)
-                if env.key?(rack_header)
-                  result[Tracing::Metadata::Ext::HTTP::RequestHeaders.to_tag(header)] = env[rack_header]
-                end
+          def parse_url(env, original_env)
+            request_obj = ::Rack::Request.new(env)
+
+            # scheme, host, and port
+            base_url = if request_obj.respond_to?(:base_url)
+                         request_obj.base_url
+                       else
+                         # Compatibility for older Rack versions
+                         request_obj.url.chomp(request_obj.fullpath)
+                       end
+
+            # https://github.com/rack/rack/blob/main/SPEC.rdoc
+            #
+            # The source of truth in Rack is the PATH_INFO key that holds the
+            # URL for the current request; but some frameworks may override that
+            # value, especially during exception handling.
+            #
+            # Because of this, we prefer to use REQUEST_URI, if available, which is the
+            # relative path + query string, and doesn't mutate.
+            #
+            # REQUEST_URI is only available depending on what web server is running though.
+            # So when its not available, we want the original, unmutated PATH_INFO, which
+            # is just the relative path without query strings.
+            #
+            # SCRIPT_NAME is the first part of the request URL path, so that
+            # the application can know its virtual location. It should be
+            # prepended to PATH_INFO to reflect the correct user visible path.
+            request_uri = env['REQUEST_URI'].to_s
+            fullpath = if request_uri.empty?
+                         query_string = original_env['QUERY_STRING'].to_s
+                         path = original_env['SCRIPT_NAME'].to_s + original_env['PATH_INFO'].to_s
+
+                         query_string.empty? ? path : "#{path}?#{query_string}"
+                       else
+                         request_uri
+                       end
+
+            base_url + fullpath
+          end
+
+          def parse_user_agent_header(headers)
+            headers.get(Tracing::Metadata::Ext::HTTP::HEADER_USER_AGENT)
+          end
+
+          def parse_request_headers(headers)
+            whitelist = configuration[:headers][:request] || []
+            whitelist.each_with_object({}) do |header, result|
+              header_value = headers.get(header)
+              unless header_value.nil?
+                header_tag = Tracing::Metadata::Ext::HTTP::RequestHeaders.to_tag(header)
+                result[header_tag] = header_value
               end
             end
           end
@@ -248,10 +306,6 @@ module Datadog
               end
             end
           end
-
-          def header_to_rack_header(name)
-            "HTTP_#{name.to_s.upcase.gsub(/[-\s]/, '_')}"
-          end
         end
       end
     end

data/lib/datadog/tracing/contrib/utils/quantization/http.rb

@@ -22,6 +22,14 @@ module Datadog
               options[:placeholder] || PLACEHOLDER
             end
 
+            def base_url(url, options = {})
+              URI.parse(url).tap do |uri|
+                uri.path = ''
+                uri.query = nil
+                uri.fragment = nil
+              end.to_s
+            end
+
             def url!(url, options = {})
               options ||= {}
 
@@ -32,8 +40,14 @@ module Datadog
                   uri.query = (!query.nil? && query.empty? ? nil : query)
                 end
 
-                # Remove any URI framents
+                # Remove any URI fragments
                 uri.fragment = nil unless options[:fragment] == :show
+
+                if options[:base] == :exclude
+                  uri.host = nil
+                  uri.port = nil
+                  uri.scheme = nil
+                end
               end.to_s
             end
 
@@ -45,22 +59,26 @@ module Datadog
 
             def query!(query, options = {})
               options ||= {}
-              options[:show] = options[:show] || []
+              options[:obfuscate] = {} if options[:obfuscate] == :internal
+              options[:show] = options[:show] || (options[:obfuscate] ? :all : [])
               options[:exclude] = options[:exclude] || []
 
               # Short circuit if query string is meant to exclude everything
               # or if the query string is meant to include everything
               return '' if options[:exclude] == :all
-              return query if options[:show] == :all
 
-              collect_query(query, uniq: true) do |key, value|
-                if options[:exclude].include?(key)
-                  [nil, nil]
-                else
-                  value = options[:show].include?(key) ? value : nil
-                  [key, value]
+              unless options[:show] == :all && !(options[:obfuscate] && options[:exclude])
+                query = collect_query(query, uniq: true) do |key, value|
+                  if options[:exclude].include?(key)
+                    [nil, nil]
+                  else
+                    value = options[:show] == :all || options[:show].include?(key) ? value : nil
+                    [key, value]
+                  end
                 end
               end
+
+              options[:obfuscate] ? obfuscate_query(query, options[:obfuscate]) : query
             end
 
             # Iterate over each key value pair, yielding to the block given.
@@ -91,6 +109,62 @@ module Datadog
             end
 
             private_class_method :collect_query
+
+            # Scans over the query string and obfuscates sensitive data by
+            # replacing matches with an opaque value
+            def obfuscate_query(query, options = {})
+              options[:regex] = nil if options[:regex] == :internal
+              re = options[:regex] || OBFUSCATOR_REGEX
+              with = options[:with] || OBFUSCATOR_WITH
+
+              query.gsub(re, with)
+            end
+
+            private_class_method :obfuscate_query
+
+            OBFUSCATOR_WITH = '<redacted>'.freeze
+
+            # rubocop:disable Layout/LineLength
+            OBFUSCATOR_REGEX = %r{
+              (?: # JSON-ish leading quote
+                 (?:"|%22)?
+              )
+              (?: # common keys
+                 (?:old_?|new_?)?p(?:ass)?w(?:or)?d(?:1|2)? # pw, password variants
+                |pass(?:_?phrase)?  # pass, passphrase variants
+                |secret
+                |(?: # key, key_id variants
+                     api_?
+                    |private_?
+                    |public_?
+                    |access_?
+                    |secret_?
+                 )key(?:_?id)?
+                |token
+                |consumer_?(?:id|key|secret)
+                |sign(?:ed|ature)?
+                |auth(?:entication|orization)?
+              )
+              (?:
+                 # '=' query string separator, plus value til next '&' separator
+                 (?:\s|%20)*(?:=|%3D)[^&]+
+                 # JSON-ish '": "somevalue"', key being handled with case above, without the opening '"'
+                |(?:"|%22)                                     # closing '"' at end of key
+                 (?:\s|%20)*(?::|%3A)(?:\s|%20)*               # ':' key-value spearator, with surrounding spaces
+                 (?:"|%22)                                     # opening '"' at start of value
+                 (?:%2[^2]|%[^2]|[^"%])+                       # value
+                 (?:"|%22)                                     # closing '"' at end of value
+              )
+             |(?: # other common secret values
+                 bearer(?:\s|%20)+[a-z0-9._\-]+
+                |token(?::|%3A)[a-z0-9]{13}
+                |gh[opsu]_[0-9a-zA-Z]{36}
+                |ey[I-L](?:[\w=-]|%3D)+\.ey[I-L](?:[\w=-]|%3D)+(?:\.(?:[\w.+/=-]|%3D|%2F|%2B)+)?
+                |-{5}BEGIN(?:[a-z\s]|%20)+PRIVATE(?:\s|%20)KEY-{5}[^\-]+-{5}END(?:[a-z\s]|%20)+PRIVATE(?:\s|%20)KEY(?:-{5})?(?:\n|%0A)?
+                |(?:ssh-(?:rsa|dss)|ecdsa-[a-z0-9]+-[a-z0-9]+)(?:\s|%20)*(?:[a-z0-9/.+]|%2F|%5C|%2B){100,}(?:=|%3D)*(?:(?:\s+)[a-z0-9._-]+)?
+              )
+            }ix.freeze
+            # rubocop:enable Layout/LineLength
           end
         end
       end

data/lib/datadog/tracing/flush.rb

@@ -3,71 +3,93 @@
 module Datadog
   module Tracing
     module Flush
-      # Consumes only completed traces (where all spans have finished)
-      class Finished
-        # Consumes and returns completed traces (where all spans have finished)
-        # from the provided \trace_op, if any.
+      # Consumes and returns a {TraceSegment} to be flushed, from
+      # the provided {TraceSegment}.
+      #
+      # Only finished spans are consumed. Any spans consumed are
+      # removed from +trace_op+ as a side effect. Unfinished spans are
+      # unaffected.
+      #
+      # @abstract
+      class Base
+        # Consumes and returns a {TraceSegment} to be flushed, from
+        # the provided {TraceSegment}.
         #
-        # Any traces consumed are removed from +trace_op+ as a side effect.
+        # Only finished spans are consumed. Any spans consumed are
+        # removed from +trace_op+ as a side effect. Unfinished spans are
+        # unaffected.
         #
+        # @param [TraceOperation] trace_op
         # @return [TraceSegment] trace to be flushed, or +nil+ if the trace is not finished
         def consume!(trace_op)
-          return unless full_flush?(trace_op)
+          return unless flush?(trace_op)
 
           get_trace(trace_op)
         end
 
-        def full_flush?(trace_op)
-          trace_op && trace_op.sampled? && trace_op.finished?
+        # Should we consume spans from the +trace_op+?
+        # @abstract
+        def flush?(trace_op)
+          raise NotImplementedError
         end
 
         protected
 
+        # Consumes all finished spans from trace.
+        # @return [TraceSegment]
         def get_trace(trace_op)
-          trace_op.flush!
+          trace_op.flush! do |spans|
+            spans.select! { |span| single_sampled?(span) } unless trace_op.sampled?
+
+            spans
+          end
+        end
+
+        # Single Span Sampling has chosen to keep this span
+        # regardless of the trace-level sampling decision
+        def single_sampled?(span)
+          span.get_metric(Sampling::Span::Ext::TAG_MECHANISM) == Sampling::Span::Ext::MECHANISM_SPAN_SAMPLING_RATE
+        end
+      end
+
+      # Consumes and returns completed traces (where all spans have finished),
+      # if any, from the provided +trace_op+.
+      #
+      # Spans consumed are removed from +trace_op+ as a side effect.
+      class Finished < Base
+        # Are all spans finished?
+        def flush?(trace_op)
+          trace_op && trace_op.finished?
         end
       end
 
-      # Performs partial trace flushing to avoid large traces residing in memory for too long
-      class Partial
+      # Consumes and returns completed or partially completed
+      # traces from the provided +trace_op+, if any.
+      #
+      # Partial trace flushing avoids large traces residing in memory for too long.
+      #
+      # Partially completed traces, where not all spans have finished,
+      # will only be returned if there are at least
+      # +@min_spans_for_partial+ finished spans.
+      #
+      # Spans consumed are removed from +trace_op+ as a side effect.
+      class Partial < Base
         # Start flushing partial trace after this many active spans in one trace
         DEFAULT_MIN_SPANS_FOR_PARTIAL_FLUSH = 500
 
         attr_reader :min_spans_for_partial
 
         def initialize(options = {})
+          super()
           @min_spans_for_partial = options.fetch(:min_spans_before_partial_flush, DEFAULT_MIN_SPANS_FOR_PARTIAL_FLUSH)
         end
 
-        # Consumes and returns completed or partially completed
-        # traces from the provided +trace_op+, if any.
-        #
-        # Partially completed traces, where not all spans have finished,
-        # will only be returned if there are at least
-        # +@min_spans_for_partial+ finished spans.
-        #
-        # Any spans consumed are removed from +trace_op+ as a side effect.
-        #
-        # @return [TraceSegment] partial or complete trace to be flushed, or +nil+ if no spans are finished
-        def consume!(trace_op)
-          return unless partial_flush?(trace_op)
-
-          get_trace(trace_op)
-        end
-
-        def partial_flush?(trace_op)
-          return false unless trace_op.sampled?
+        def flush?(trace_op)
           return true if trace_op.finished?
           return false if trace_op.finished_span_count < @min_spans_for_partial
 
           true
         end
-
-        protected
-
-        def get_trace(trace_op)
-          trace_op.flush!
-        end
       end
     end
   end

data/lib/datadog/tracing/metadata/ext.rb

@@ -63,11 +63,14 @@ module Datadog
           TAG_BASE_URL = 'http.base_url'
           TAG_METHOD = 'http.method'
           TAG_STATUS_CODE = 'http.status_code'
+          TAG_USER_AGENT = 'http.useragent'
           TAG_URL = 'http.url'
           TYPE_INBOUND = AppTypes::TYPE_WEB.freeze
           TYPE_OUTBOUND = 'http'
           TYPE_PROXY = 'proxy'
           TYPE_TEMPLATE = 'template'
+          TAG_CLIENT_IP = 'http.client_ip'
+          HEADER_USER_AGENT = 'User-Agent'
 
           # General header functionality
           module Headers
@@ -153,15 +156,6 @@ module Datadog
           TAG_QUERY = 'sql.query'
         end
 
-        # @public_api
-        module DB
-          TAG_INSTANCE = 'db.instance'
-          TAG_USER = 'db.user'
-          TAG_SYSTEM = 'db.system'
-          TAG_STATEMENT = 'db.statement'
-          TAG_ROW_COUNT = 'db.row_count'
-        end
-
         # @public_api
         module SpanKind
           TAG_SERVER = 'server'

data/lib/datadog/tracing/metadata/tagging.rb

@@ -65,6 +65,15 @@ module Datadog
           tags.each { |k, v| set_tag(k, v) }
         end
 
+        # Returns true if the provided `tag` was set to a non-nil value.
+        # False otherwise.
+        #
+        # @param [String] tag the tag or metric to check for presence
+        # @return [Boolean] if the tag is present and not nil
+        def has_tag?(tag) # rubocop:disable Naming/PredicateName
+          !get_tag(tag).nil? # nil is considered not present, thus we can't use `Hash#has_key?`
+        end
+
         # This method removes a tag for the given key.
         def clear_tag(key)
           meta.delete(key)

data/lib/datadog/tracing/sampling/rate_limiter.rb

@@ -39,6 +39,9 @@ module Datadog
         def initialize(rate, max_tokens = rate)
           super()
 
+          raise ArgumentError, "rate must be a number: #{rate}" unless rate.is_a?(Numeric)
+          raise ArgumentError, "max_tokens must be a number: #{max_tokens}" unless max_tokens.is_a?(Numeric)
+
           @rate = rate
           @max_tokens = max_tokens