ddtrace 0.12.0.beta2 → 0.12.0.rc1

data/lib/ddtrace/contrib/rails/framework.rb

@@ -1,13 +1,13 @@
 require 'ddtrace/pin'
 require 'ddtrace/ext/app_types'
 
+require 'ddtrace/contrib/active_record/patcher'
 require 'ddtrace/contrib/grape/endpoint'
 require 'ddtrace/contrib/rack/middlewares'
 
 require 'ddtrace/contrib/rails/core_extensions'
 require 'ddtrace/contrib/rails/action_controller'
 require 'ddtrace/contrib/rails/action_view'
-require 'ddtrace/contrib/rails/active_record'
 require 'ddtrace/contrib/rails/active_support'
 require 'ddtrace/contrib/rails/utils'
 
@@ -21,38 +21,53 @@ module Datadog
       module Framework
         # configure Datadog settings
         def self.setup
-          config = Datadog.configuration[:rails]
-          config[:service_name] ||= Utils.app_name
-          tracer = config[:tracer]
+          config = config_with_defaults
+
+          activate_rack!(config)
+          activate_active_record!(config)
+          set_service_info!(config)
+
+          # By default, default service would be guessed from the script
+          # being executed, but here we know better, get it from Rails config.
+          config[:tracer].default_service = config[:service_name]
+        end
 
+        def self.config_with_defaults
+          # We set defaults here instead of in the patcher because we need to wait
+          # for the Rails application to be fully initialized.
+          Datadog.configuration[:rails].tap do |config|
+            config[:service_name] ||= Utils.app_name
+            config[:database_service] ||= "#{config[:service_name]}-#{Contrib::ActiveRecord::Utils.adapter_name}"
+            config[:controller_service] ||= config[:service_name]
+            config[:cache_service] ||= "#{config[:service_name]}-cache"
+          end
+        end
+
+        def self.activate_rack!(config)
           Datadog.configuration.use(
             :rack,
-            tracer: tracer,
+            tracer: config[:tracer],
+            application: ::Rails.application,
             service_name: config[:service_name],
+            middleware_names: config[:middleware_names],
             distributed_tracing: config[:distributed_tracing]
           )
-
-          config[:controller_service] ||= config[:service_name]
-          config[:cache_service] ||= "#{config[:service_name]}-cache"
-
-          tracer.set_service_info(config[:controller_service], 'rails', Ext::AppTypes::WEB)
-          tracer.set_service_info(config[:cache_service], 'rails', Ext::AppTypes::CACHE)
-          set_database_service
-
-          # By default, default service would be guessed from the script
-          # being executed, but here we know better, get it from Rails config.
-          tracer.default_service = config[:service_name]
         end
 
-        def self.set_database_service
+        def self.activate_active_record!(config)
           return unless defined?(::ActiveRecord)
 
-          config = Datadog.configuration[:rails]
-          adapter_name = Utils.adapter_name
-          config[:database_service] ||= "#{config[:service_name]}-#{adapter_name}"
-          config[:tracer].set_service_info(config[:database_service], adapter_name, Ext::AppTypes::DB)
-        rescue => e
-          Tracer.log.warn("Unable to get database config (#{e}), skipping ActiveRecord instrumentation")
+          Datadog.configuration.use(
+            :active_record,
+            service_name: config[:database_service],
+            tracer: config[:tracer]
+          )
+        end
+
+        def self.set_service_info!(config)
+          tracer = config[:tracer]
+          tracer.set_service_info(config[:controller_service], 'rails', Ext::AppTypes::WEB)
+          tracer.set_service_info(config[:cache_service], 'rails', Ext::AppTypes::CACHE)
         end
       end
     end

data/lib/ddtrace/contrib/rails/middlewares.rb

@@ -22,8 +22,13 @@ module Datadog
         # SignalException::Interrupt would still bubble up.
         rescue Exception => e
           tracer = Datadog.configuration[:rails][:tracer]
-          span = tracer.active_span()
-          span.set_error(e) unless span.nil?
+          span = tracer.active_span
+          unless span.nil?
+            # Only set error if it's supposed to be flagged as such
+            # e.g. we don't want to flag 404s.
+            # You can add custom errors via `config.action_dispatch.rescue_responses`
+            span.set_error(e) if Utils.exception_is_error?(e)
+          end
           raise e
         end
       end

data/lib/ddtrace/contrib/rails/patcher.rb

@@ -1,3 +1,5 @@
+require 'ddtrace/contrib/rails/utils'
+
 module Datadog
   module Contrib
     module Rails
@@ -9,7 +11,13 @@ module Datadog
         option :service_name
         option :controller_service
         option :cache_service
-        option :database_service
+        option :database_service, depends_on: [:service_name] do |value|
+          value.tap do
+            # Update ActiveRecord service name too
+            Datadog.configuration[:active_record][:service_name] = value
+          end
+        end
+        option :middleware_names, default: false
         option :distributed_tracing, default: false
         option :template_base_path, default: 'views/'
         option :exception_controller, default: nil
@@ -36,11 +44,6 @@ module Datadog
 
             defined?(::Rails::VERSION) && ::Rails::VERSION::MAJOR.to_i >= 3
           end
-
-          def active_record_instantiation_tracing_supported?
-            Gem.loaded_specs['activerecord'] \
-              && Gem.loaded_specs['activerecord'].version >= Gem::Version.new('4.2')
-          end
         end
       end
     end

data/lib/ddtrace/contrib/rails/railtie.rb

@@ -6,13 +6,15 @@ module Datadog
   # Railtie class initializes
   class Railtie < Rails::Railtie
     config.app_middleware.insert_before(0, Datadog::Contrib::Rack::TraceMiddleware)
-    config.app_middleware.use(Datadog::Contrib::Rails::ExceptionMiddleware)
+    # Insert right after Rails exception handling middleware, because if it's before,
+    # it catches and swallows the error. If it's too far after, custom middleware can find itself
+    # between, and raise exceptions that don't end up getting tagged on the request properly (e.g lost stack trace.)
+    config.app_middleware.insert_after(ActionDispatch::ShowExceptions, Datadog::Contrib::Rails::ExceptionMiddleware)
 
     config.after_initialize do
       Datadog::Contrib::Rails::Framework.setup
       Datadog::Contrib::Rails::ActionController.instrument
       Datadog::Contrib::Rails::ActionView.instrument
-      Datadog::Contrib::Rails::ActiveRecord.instrument
       Datadog::Contrib::Rails::ActiveSupport.instrument
     end
   end

data/lib/ddtrace/contrib/rails/utils.rb

@@ -23,21 +23,6 @@ module Datadog
           return name.to_s
         end
 
-        # TODO: Consider moving this out of Rails.
-        # Return a canonical name for a type of database
-        def self.normalize_vendor(vendor)
-          case vendor
-          when nil
-            'defaultdb'
-          when 'sqlite3'
-            'sqlite'
-          when 'postgresql'
-            'postgres'
-          else
-            vendor
-          end
-        end
-
         def self.app_name
           if ::Rails::VERSION::MAJOR >= 4
             ::Rails.application.class.parent_name.underscore
@@ -46,33 +31,17 @@ module Datadog
           end
         end
 
-        def self.adapter_name
-          normalize_vendor(connection_config[:adapter])
-        end
-
-        def self.database_name
-          connection_config[:database]
-        end
-
-        def self.adapter_host
-          connection_config[:host]
-        end
-
-        def self.adapter_port
-          connection_config[:port]
-        end
-
-        def self.connection_config
-          @connection_config ||= begin
-            if defined?(::ActiveRecord::Base.connection_config)
-              ::ActiveRecord::Base.connection_config
-            else
-              ::ActiveRecord::Base.connection_pool.spec.config
-            end
+        def self.exception_is_error?(exception)
+          if defined?(::ActionDispatch::ExceptionWrapper)
+            # Gets the equivalent status code for the exception (not all are 5XX)
+            # You can add custom errors via `config.action_dispatch.rescue_responses`
+            status = ::ActionDispatch::ExceptionWrapper.status_code_for_exception(exception.class.name)
+            # Only 5XX exceptions are actually errors (e.g. don't flag 404s)
+            status.to_s.starts_with?('5')
+          else
+            true
           end
         end
-
-        private_class_method :connection_config
       end
     end
   end

data/lib/ddtrace/patcher.rb

@@ -1,18 +1,40 @@
 module Datadog
   # Defines some useful patching methods for integrations
   module Patcher
-    module_function
+    def self.included(base)
+      base.send(:extend, CommonMethods)
+      base.send(:include, CommonMethods)
+    end
 
-    def without_warnings
-      # This is typically used when monkey patching functions such as
-      # intialize, which Ruby advices you not to. Use cautiously.
-      v = $VERBOSE
-      $VERBOSE = nil
-      begin
-        yield
-      ensure
-        $VERBOSE = v
+    # Defines some common methods for patching, that can be used
+    # at the instance, class, or module level.
+    module CommonMethods
+      def without_warnings
+        # This is typically used when monkey patching functions such as
+        # intialize, which Ruby advices you not to. Use cautiously.
+        v = $VERBOSE
+        $VERBOSE = nil
+        begin
+          yield
+        ensure
+          $VERBOSE = v
+        end
+      end
+
+      def do_once(key = nil)
+        # If already done, don't do again
+        @done_once ||= {}
+        return @done_once[key] if @done_once.key?(key)
+
+        # Otherwise 'do'
+        yield.tap do
+          # Then add the key so we don't do again.
+          @done_once[key] = true
+        end
       end
     end
+
+    # Extend the common methods so they're available as a module function.
+    extend(CommonMethods)
   end
 end

data/lib/ddtrace/quantization/http.rb

@@ -0,0 +1,86 @@
+require 'uri'
+require 'set'
+
+module Datadog
+  module Quantization
+    # Quantization for HTTP resources
+    module HTTP
+      PLACEHOLDER = '?'.freeze
+
+      module_function
+
+      def url(url, options = {})
+        url!(url, options)
+      rescue StandardError
+        options[:placeholder] || PLACEHOLDER
+      end
+
+      def url!(url, options = {})
+        options ||= {}
+
+        URI.parse(url).tap do |uri|
+          # Format the query string
+          if uri.query
+            query = query(uri.query, options[:query])
+            uri.query = (!query.nil? && query.empty? ? nil : query)
+          end
+
+          # Remove any URI framents
+          uri.fragment = nil unless options[:fragment] == :show
+        end.to_s
+      end
+
+      def query(query, options = {})
+        query!(query, options)
+      rescue StandardError
+        options[:placeholder] || PLACEHOLDER
+      end
+
+      def query!(query, options = {})
+        options ||= {}
+        options[:show] = options[:show] || []
+        options[:exclude] = options[:exclude] || []
+
+        # Short circuit if query string is meant to exclude everything
+        # or if the query string is meant to include everything
+        return '' if options[:exclude] == :all
+        return query if options[:show] == :all
+
+        collect_query(query, uniq: true) do |key, value|
+          if options[:exclude].include?(key)
+            [nil, nil]
+          else
+            value = options[:show].include?(key) ? value : nil
+            [key, value]
+          end
+        end
+      end
+
+      # Iterate over each key value pair, yielding to the block given.
+      # Accepts :uniq option, which keeps uniq copies of keys without values.
+      # e.g. Reduces "foo&bar=bar&bar=bar&foo" to "foo&bar=bar&bar=bar"
+      def collect_query(query, options = {})
+        return query unless block_given?
+        uniq = options[:uniq].nil? ? false : options[:uniq]
+        keys = Set.new
+
+        delims = query.scan(/(^|&|;)/).flatten
+        query.split(/[&;]/).collect.with_index do |pairs, i|
+          key, value = pairs.split('=', 2)
+          key, value = yield(key, value, delims[i])
+          if uniq && keys.include?(key)
+            ''
+          elsif key && value
+            "#{delims[i]}#{key}=#{value}"
+          elsif key
+            "#{delims[i]}#{key}".tap { keys << key }
+          else
+            ''
+          end
+        end.join.sub(/^[&;]/, '')
+      end
+
+      private_class_method :collect_query
+    end
+  end
+end

data/lib/ddtrace/tracer.rb

@@ -5,6 +5,7 @@ require 'pathname'
 
 require 'ddtrace/span'
 require 'ddtrace/context'
+require 'ddtrace/context_flush'
 require 'ddtrace/provider'
 require 'ddtrace/logger'
 require 'ddtrace/writer'
@@ -97,6 +98,8 @@ module Datadog
       @provider = options.fetch(:context_provider, Datadog::DefaultContextProvider.new)
       @provider ||= Datadog::DefaultContextProvider.new # @provider should never be nil
 
+      @context_flush = options[:partial_flush] ? Datadog::ContextFlush.new(options) : nil
+
       @mutex = Mutex.new
       @services = {}
       @tags = {}
@@ -117,8 +120,13 @@ module Datadog
       enabled = options.fetch(:enabled, nil)
       hostname = options.fetch(:hostname, nil)
       port = options.fetch(:port, nil)
+
+      # Those are rare "power-user" options.
       sampler = options.fetch(:sampler, nil)
       priority_sampling = options[:priority_sampling]
+      max_spans_before_partial_flush = options.fetch(:max_spans_before_partial_flush, nil)
+      min_spans_before_partial_flush = options.fetch(:min_spans_before_partial_flush, nil)
+      partial_flush_timeout = options.fetch(:partial_flush_timeout, nil)
 
       @enabled = enabled unless enabled.nil?
       @sampler = sampler unless sampler.nil?
@@ -130,6 +138,10 @@ module Datadog
 
       @writer.transport.hostname = hostname unless hostname.nil?
       @writer.transport.port = port unless port.nil?
+
+      @context_flush = Datadog::ContextFlush.new(options) unless min_spans_before_partial_flush.nil? &&
+                                                                 max_spans_before_partial_flush.nil? &&
+                                                                 partial_flush_timeout.nil?
     end
 
     # Set the information about the given service. A valid example is:
@@ -295,8 +307,23 @@ module Datadog
       context = context.context if context.is_a?(Datadog::Span)
       return if context.nil?
       trace, sampled = context.get
-      ready = !trace.nil? && !trace.empty? && sampled
-      write(trace) if ready
+
+      # If context flushing is configured...
+      if @context_flush
+        if sampled
+          if trace.nil? || trace.empty?
+            @context_flush.each_partial_trace(context) do |t|
+              write(t)
+            end
+          else
+            write(trace)
+          end
+        end
+      # Default behavior
+      else
+        ready = !trace.nil? && !trace.empty? && sampled
+        write(trace) if ready
+      end
     end
 
     # Return the current active span or +nil+.

data/lib/ddtrace/transport.rb

@@ -8,6 +8,7 @@ module Datadog
   # Transport class that handles the spans delivery to the
   # local trace-agent. The class wraps a Net:HTTP instance
   # so that the Transport is thread-safe.
+  # rubocop:disable Metrics/ClassLength
   class HTTPTransport
     attr_accessor :hostname, :port
     attr_reader :traces_endpoint, :services_endpoint
@@ -21,18 +22,21 @@ module Datadog
 
     API = {
       V4 = 'v0.4'.freeze => {
+        version: V4,
         traces_endpoint: '/v0.4/traces'.freeze,
         services_endpoint: '/v0.4/services'.freeze,
         encoder: Encoding::MsgpackEncoder,
         fallback: 'v0.3'.freeze
       }.freeze,
       V3 = 'v0.3'.freeze => {
+        version: V3,
         traces_endpoint: '/v0.3/traces'.freeze,
         services_endpoint: '/v0.3/services'.freeze,
         encoder: Encoding::MsgpackEncoder,
         fallback: 'v0.2'.freeze
       }.freeze,
       V2 = 'v0.2'.freeze => {
+        version: V2,
         traces_endpoint: '/v0.2/traces'.freeze,
         services_endpoint: '/v0.2/services'.freeze,
         encoder: Encoding::JSONEncoder
@@ -71,34 +75,45 @@ module Datadog
       case endpoint
       when :services
         payload = @encoder.encode_services(data)
-        status_code = post(@api[:services_endpoint], payload)
+        status_code = post(@api[:services_endpoint], payload) do |response|
+          process_callback(:services, response)
+        end
       when :traces
         count = data.length
         payload = @encoder.encode_traces(data)
-        status_code = post(@api[:traces_endpoint], payload, count)
+        status_code = post(@api[:traces_endpoint], payload, count) do |response|
+          process_callback(:traces, response)
+        end
       else
         Datadog::Tracer.log.error("Unsupported endpoint: #{endpoint}")
         return nil
       end
 
-      downgrade! && send(endpoint, data) if downgrade?(status_code)
-
-      status_code
+      if downgrade?(status_code)
+        downgrade!
+        send(endpoint, data)
+      else
+        status_code
+      end
     end
 
     # send data to the trace-agent; the method is thread-safe
     def post(url, data, count = nil)
-      Datadog::Tracer.log.debug("Sending data from process: #{Process.pid}")
-      headers = count.nil? ? {} : { TRACE_COUNT_HEADER => count.to_s }
-      headers = headers.merge(@headers)
-      request = Net::HTTP::Post.new(url, headers)
-      request.body = data
-
-      response = Net::HTTP.start(@hostname, @port, read_timeout: TIMEOUT) { |http| http.request(request) }
-      handle_response(response)
-    rescue StandardError => e
-      Datadog::Tracer.log.error(e.message)
-      500
+      begin
+        Datadog::Tracer.log.debug("Sending data from process: #{Process.pid}")
+        headers = count.nil? ? {} : { TRACE_COUNT_HEADER => count.to_s }
+        headers = headers.merge(@headers)
+        request = Net::HTTP::Post.new(url, headers)
+        request.body = data
+
+        response = Net::HTTP.start(@hostname, @port, read_timeout: TIMEOUT) { |http| http.request(request) }
+        handle_response(response)
+      rescue StandardError => e
+        Datadog::Tracer.log.error(e.message)
+        500
+      end.tap do
+        yield(response) if block_given?
+      end
     end
 
     # Downgrade the connection to a compatibility version of the HTTPTransport;
@@ -166,8 +181,6 @@ module Datadog
         @mutex.synchronize { @count_server_error += 1 }
       end
 
-      process_callback(response)
-
       status_code
     rescue StandardError => e
       Datadog::Tracer.log.error(e.message)
@@ -188,10 +201,10 @@ module Datadog
 
     private
 
-    def process_callback(response)
+    def process_callback(action, response)
       return unless @response_callback && @response_callback.respond_to?(:call)
 
-      @response_callback.call(response)
+      @response_callback.call(action, response, @api)
     rescue => e
       Tracer.log.debug("Error processing callback: #{e}")
     end