ddtrace 0.12.0.beta2 → 0.12.0.rc1

data/gemfiles/contrib.gemfile

@@ -13,13 +13,13 @@ gem "redis", "< 4.0"
 gem "hiredis"
 gem "sinatra"
 gem "sqlite3"
-gem "activerecord"
+gem "activerecord", "< 5.1.5"
 gem "sidekiq"
 gem "aws-sdk"
 gem "sucker_punch"
 gem "dalli"
 gem "resque", "< 2.0"
 gem "racecar", ">= 0.3.5"
-gem "mysql2", platform: :ruby
+gem "mysql2", "< 0.5", platform: :ruby
 
 gemspec path: "../"

data/gemfiles/rails4_mysql2.gemfile

@@ -4,7 +4,7 @@ source "https://rubygems.org"
 
 gem "pry-nav", git: "https://github.com/nixme/pry-nav.git", branch: "master"
 gem "rails", "4.2.7.1"
-gem "mysql2", platform: :ruby
+gem "mysql2", "< 0.5", platform: :ruby
 gem "activerecord-jdbcmysql-adapter", platform: :jruby
 
 gemspec path: "../"

data/gemfiles/rails5_mysql2.gemfile

@@ -3,7 +3,7 @@
 source "https://rubygems.org"
 
 gem "pry-nav", git: "https://github.com/nixme/pry-nav.git", branch: "master"
-gem "rails", "5.0.1"
-gem "mysql2", platform: :ruby
+gem "rails", "5.1.5"
+gem "mysql2", "< 0.5", platform: :ruby
 
 gemspec path: "../"

data/gemfiles/rails5_postgres.gemfile

@@ -3,7 +3,7 @@
 source "https://rubygems.org"
 
 gem "pry-nav", git: "https://github.com/nixme/pry-nav.git", branch: "master"
-gem "rails", "5.0.1"
+gem "rails", "5.1.5"
 gem "pg", "< 1.0", platform: :ruby
 
 gemspec path: "../"

data/gemfiles/rails5_postgres_redis.gemfile

@@ -3,7 +3,7 @@
 source "https://rubygems.org"
 
 gem "pry-nav", git: "https://github.com/nixme/pry-nav.git", branch: "master"
-gem "rails", "5.0.1"
+gem "rails", "5.1.5"
 gem "pg", "< 1.0", platform: :ruby
 gem "redis-rails"
 gem "redis"

data/gemfiles/rails5_postgres_sidekiq.gemfile

@@ -3,7 +3,7 @@
 source "https://rubygems.org"
 
 gem "pry-nav", git: "https://github.com/nixme/pry-nav.git", branch: "master"
-gem "rails", "5.0.1"
+gem "rails", "5.1.5"
 gem "pg", "< 1.0", platform: :ruby
 gem "sidekiq"
 gem "activejob"

data/lib/ddtrace.rb

@@ -4,6 +4,7 @@ require 'ddtrace/registry'
 require 'ddtrace/pin'
 require 'ddtrace/tracer'
 require 'ddtrace/error'
+require 'ddtrace/quantization/http'
 require 'ddtrace/pipeline'
 require 'ddtrace/configuration'
 require 'ddtrace/patcher'

data/lib/ddtrace/context.rb

@@ -13,23 +13,22 @@ module Datadog
   # \Context, it will be related to the original trace.
   #
   # This data structure is thread-safe.
+  # rubocop:disable Metrics/ClassLength
   class Context
+    # 100k spans is about a 100Mb footprint
+    DEFAULT_MAX_LENGTH = 100_000
+
+    attr_reader :max_length
+
     # Initialize a new thread-safe \Context.
     def initialize(options = {})
       @mutex = Mutex.new
+      # max_length is the amount of spans above which, for a given trace,
+      # the context will simply drop and ignore spans, avoiding high memory usage.
+      @max_length = options.fetch(:max_length, DEFAULT_MAX_LENGTH)
       reset(options)
     end
 
-    def reset(options = {})
-      @trace = []
-      @parent_trace_id = options.fetch(:trace_id, nil)
-      @parent_span_id = options.fetch(:span_id, nil)
-      @sampled = options.fetch(:sampled, false)
-      @sampling_priority = options.fetch(:sampling_priority, nil)
-      @finished_spans = 0
-      @current_span = nil
-    end
-
     def trace_id
       @mutex.synchronize do
         @parent_trace_id
@@ -64,20 +63,19 @@ module Datadog
       end
     end
 
-    def set_current_span(span)
-      @current_span = span
-      if span
-        @parent_trace_id = span.trace_id
-        @parent_span_id = span.span_id
-        @sampled = span.sampled
-      else
-        @parent_span_id = nil
-      end
-    end
-
     # Add a span to the context trace list, keeping it as the last active span.
     def add_span(span)
       @mutex.synchronize do
+        # If hitting the hard limit, just drop spans. This is really a rare case
+        # as it means despite the soft limit, the hard limit is reached, so the trace
+        # by default has 10000 spans, all of which belong to unfinished parts of a
+        # larger trace. This is a catch-all to reduce global memory usage.
+        if @max_length > 0 && @trace.length >= @max_length
+          Datadog::Tracer.log.debug("context full, ignoring span #{span.name}")
+          # Detach the span from any context, it's being dropped and ignored.
+          span.context = nil
+          return
+        end
         set_current_span(span)
         @trace << span
         span.context = self
@@ -105,12 +103,6 @@ module Datadog
       end
     end
 
-    # Returns if the trace for the current Context is finished or not.
-    # Low-level internal function, not thread-safe.
-    def check_finished_spans
-      @finished_spans > 0 && @trace.length == @finished_spans
-    end
-
     # Returns if the trace for the current Context is finished or not. A \Context
     # is considered finished if all spans in this context are finished.
     def finished?
@@ -135,14 +127,16 @@ module Datadog
     # This operation is thread-safe.
     def get
       @mutex.synchronize do
-        return nil, nil unless check_finished_spans
-
         trace = @trace
         sampled = @sampled
+
         attach_sampling_priority if sampled && @sampling_priority
 
+        # still return sampled attribute, even if context is not finished
+        return nil, sampled unless check_finished_spans()
+
         reset
-        return trace, sampled
+        [trace, sampled]
       end
     end
 
@@ -154,6 +148,35 @@ module Datadog
       end
     end
 
+    private
+
+    def reset(options = {})
+      @trace = []
+      @parent_trace_id = options.fetch(:trace_id, nil)
+      @parent_span_id = options.fetch(:span_id, nil)
+      @sampled = options.fetch(:sampled, false)
+      @sampling_priority = options.fetch(:sampling_priority, nil)
+      @finished_spans = 0
+      @current_span = nil
+    end
+
+    def set_current_span(span)
+      @current_span = span
+      if span
+        @parent_trace_id = span.trace_id
+        @parent_span_id = span.span_id
+        @sampled = span.sampled
+      else
+        @parent_span_id = nil
+      end
+    end
+
+    # Returns if the trace for the current Context is finished or not.
+    # Low-level internal function, not thread-safe.
+    def check_finished_spans
+      @finished_spans > 0 && @trace.length == @finished_spans
+    end
+
     def attach_sampling_priority
       @trace.first.set_metric(
         Ext::DistributedTracing::SAMPLING_PRIORITY_KEY,
@@ -161,10 +184,49 @@ module Datadog
       )
     end
 
-    private :reset
-    private :check_finished_spans
-    private :set_current_span
-    private :attach_sampling_priority
+    # Return the start time of the root span, or nil if there are no spans or this is undefined.
+    def start_time
+      @mutex.synchronize do
+        return nil if @trace.empty?
+        @trace[0].start_time
+      end
+    end
+
+    # Return the length of the current trace held by this context.
+    def length
+      @mutex.synchronize do
+        @trace.length
+      end
+    end
+
+    # Iterate on each span within the trace. This is thread safe.
+    def each_span
+      @mutex.synchronize do
+        @trace.each do |span|
+          yield span
+        end
+      end
+    end
+
+    # Delete any span matching the condition. This is thread safe.
+    def delete_span_if
+      @mutex.synchronize do
+        @trace.delete_if do |span|
+          finished = span.finished?
+          delete_span = yield span
+          if delete_span
+            # We need to detach the span from the context, else, some code
+            # finishing it afterwards would mess up with the number of
+            # finished_spans and possibly cause other side effects.
+            span.context = nil
+            # Acknowledge there's one span less to finish, if needed.
+            # It's very important to keep this balanced.
+            @finished_spans -= 1 if finished
+          end
+          delete_span
+        end
+      end
+    end
   end
 
   # ThreadLocalContext can be used as a tracer global reference to create

data/lib/ddtrace/context_flush.rb

@@ -0,0 +1,132 @@
+require 'set'
+
+require 'ddtrace/context'
+
+module Datadog
+  # \ContextFlush is used to cap context size and avoid it using too much memory.
+  # It performs memory flushes when required.
+  class ContextFlush
+    # by default, soft and hard limits are the same
+    DEFAULT_MAX_SPANS_BEFORE_PARTIAL_FLUSH = Datadog::Context::DEFAULT_MAX_LENGTH
+    # by default, never do a partial flush
+    DEFAULT_MIN_SPANS_BEFORE_PARTIAL_FLUSH = Datadog::Context::DEFAULT_MAX_LENGTH
+    # timeout should be lower than the trace agent window
+    DEFAULT_PARTIAL_FLUSH_TIMEOUT = 10
+
+    private_constant :DEFAULT_MAX_SPANS_BEFORE_PARTIAL_FLUSH
+    private_constant :DEFAULT_MIN_SPANS_BEFORE_PARTIAL_FLUSH
+    private_constant :DEFAULT_PARTIAL_FLUSH_TIMEOUT
+
+    def initialize(options = {})
+      # max_spans_before_partial_flush is the amount of spans collected before
+      # the context starts to partially flush parts of traces. With a setting of 10k,
+      # the memory overhead is about 10Mb per thread/context (depends on spans metadata,
+      # this is just an order of magnitude).
+      @max_spans_before_partial_flush = options.fetch(:max_spans_before_partial_flush,
+                                                      DEFAULT_MAX_SPANS_BEFORE_PARTIAL_FLUSH)
+      # min_spans_before_partial_flush is the minimum number of spans required
+      # for a partial flush to happen on a timeout. This is to prevent partial flush
+      # of traces which last a very long time but yet have few spans.
+      @min_spans_before_partial_flush = options.fetch(:min_spans_before_partial_flush,
+                                                      DEFAULT_MIN_SPANS_BEFORE_PARTIAL_FLUSH)
+      # partial_flush_timeout is the limit (in seconds) above which the context
+      # considers flushing parts of the trace. Partial flushes should not be done too
+      # late else the agent rejects them with a "too far in the past" error.
+      @partial_flush_timeout = options.fetch(:partial_flush_timeout,
+                                             DEFAULT_PARTIAL_FLUSH_TIMEOUT)
+      @partial_traces = []
+    end
+
+    def add_children(m, spans, ids, leaf)
+      spans << leaf
+      ids.add(leaf.span_id)
+
+      if m[leaf.span_id]
+        m[leaf.span_id].each do |sub|
+          add_children(m, spans, ids, sub)
+        end
+      end
+    end
+
+    def partial_traces(context)
+      # 1st step, taint all parents of an unfinished span as unflushable
+      unflushable_ids = Set.new
+
+      context.send(:each_span) do |span|
+        next if span.finished? || unflushable_ids.include?(span.span_id)
+        unflushable_ids.add span.span_id
+        while span.parent
+          span = span.parent
+          unflushable_ids.add span.span_id
+        end
+      end
+
+      # 2nd step, find all spans which are at the border between flushable and unflushable
+      # Along the road, collect a reverse-tree which allows direct walking from parents to
+      # children but only for the ones we're interested it.
+      roots = []
+      children_map = {}
+      context.send(:each_span) do |span|
+        # There's no point in trying to put the real root in those partial roots, if
+        # it's flushable, the default algorithm would figure way more quickly.
+        if span.parent && !unflushable_ids.include?(span.span_id)
+          if unflushable_ids.include?(span.parent.span_id)
+            # span is flushable but is parent is not
+            roots << span
+          else
+            # span is flushable and its parent is too, build the reverse
+            # parent to child map for this one, it will be useful
+            children_map[span.parent.span_id] ||= []
+            children_map[span.parent.span_id] << span
+          end
+        end
+      end
+
+      # 3rd step, find all children, as this can be costly, only perform it for partial roots
+      partial_traces = []
+      all_ids = Set.new
+      roots.each do |root|
+        spans = []
+        add_children(children_map, spans, all_ids, root)
+        partial_traces << spans
+      end
+
+      return [nil, nil] if partial_traces.empty?
+      [partial_traces, all_ids]
+    end
+
+    def partial_flush(context)
+      traces, flushed_ids = partial_traces(context)
+      return nil unless traces && flushed_ids
+
+      # We need to reject by span ID and not by value, because a span
+      # value may be altered (typical example: it's finished by some other thread)
+      # since we lock only the context, not all the spans which belong to it.
+      context.send(:delete_span_if) { |span| flushed_ids.include? span.span_id }
+      traces
+    end
+
+    # Performs an operation which each partial trace it can get from the context.
+    def each_partial_trace(context)
+      start_time = context.send(:start_time)
+      length = context.send(:length)
+      # Stop and do not flush anything if there are not enough spans.
+      return if length <= @min_spans_before_partial_flush
+      # If there are enough spans, but not too many, check for start time.
+      # If timeout is not given or 0, then wait
+      return if length <= @max_spans_before_partial_flush &&
+                (@partial_flush_timeout.nil? || @partial_flush_timeout <= 0 ||
+                 (start_time && start_time > Time.now.utc - @partial_flush_timeout))
+      # Here, either the trace is old or we have too many spans, flush it.
+      traces = partial_flush(context)
+      return unless traces
+      traces.each do |trace|
+        yield trace
+      end
+    end
+
+    private :add_children
+    private :partial_traces
+    private :partial_flush
+  end
+end

data/lib/ddtrace/contrib/active_record/patcher.rb

@@ -1,5 +1,7 @@
 require 'ddtrace/ext/sql'
 require 'ddtrace/ext/app_types'
+require 'ddtrace/contrib/active_record/utils'
+require 'ddtrace/contrib/active_support/notifications/subscriber'
 
 module Datadog
   module Contrib
@@ -7,15 +9,51 @@ module Datadog
       # Patcher enables patching of 'active_record' module.
       module Patcher
         include Base
+        include ActiveSupport::Notifications::Subscriber
+
+        NAME_SQL = 'sql.active_record'.freeze
+        NAME_INSTANTIATION = 'instantiation.active_record'.freeze
+
         register_as :active_record, auto_patch: false
-        option :service_name do |value|
-          value.tap { @database_service_name = nil }
+        option :service_name, depends_on: [:tracer] do |value|
+          (value || Utils.adapter_name).tap do |v|
+            get_option(:tracer).set_service_info(v, 'active_record', Ext::AppTypes::DB)
+          end
         end
         option :orm_service_name
-        option :tracer, default: Datadog.tracer
+        option :tracer, default: Datadog.tracer do |value|
+          (value || Datadog.tracer).tap do |v|
+            # Make sure to update tracers of all subscriptions
+            subscriptions.each do |subscription|
+              subscription.tracer = v
+            end
+          end
+        end
 
         @patched = false
 
+        on_subscribe do
+          # sql.active_record
+          subscribe(
+            self::NAME_SQL,                         # Event name
+            'active_record.sql',                    # Span name
+            { service: get_option(:service_name) }, # Span options
+            get_option(:tracer),                    # Tracer
+            &method(:sql)                           # Handler
+          )
+
+          # instantiation.active_record
+          if instantiation_tracing_supported?
+            subscribe(
+              self::NAME_INSTANTIATION,               # Event name
+              'active_record.instantiation',          # Span name
+              { service: get_option(:service_name) }, # Span options
+              get_option(:tracer),                    # Tracer
+              &method(:instantiation)                 # Handler
+            )
+          end
+        end
+
         module_function
 
         # patched? tells whether patch has been successfully applied
@@ -26,9 +64,7 @@ module Datadog
         def patch
           if !@patched && defined?(::ActiveRecord)
             begin
-              require 'ddtrace/contrib/rails/utils'
-
-              patch_active_record
+              subscribe!
               @patched = true
             rescue StandardError => e
               Datadog::Tracer.log.error("Unable to apply Active Record integration: #{e}")
@@ -38,84 +74,33 @@ module Datadog
           @patched
         end
 
-        def patch_active_record
-          # subscribe when the active record query has been processed
-          ::ActiveSupport::Notifications.subscribe('sql.active_record') do |*args|
-            sql(*args)
-          end
-
-          if instantiation_tracing_supported?
-            # subscribe when the active record instantiates objects
-            ::ActiveSupport::Notifications.subscribe('instantiation.active_record') do |*args|
-              instantiation(*args)
-            end
-          end
-        end
-
         def instantiation_tracing_supported?
           Gem.loaded_specs['activerecord'] \
             && Gem.loaded_specs['activerecord'].version >= Gem::Version.new('4.2')
         end
 
-        # NOTE: Resolve this here instead of in the option defaults,
-        #       because resolving adapter name as a default causes ActiveRecord to connect,
-        #       which isn't a good idea at initialization time.
-        def self.database_service_name
-          @database_service_name ||= (get_option(:service_name) || adapter_name).tap do |name|
-            get_option(:tracer).set_service_info(name, 'active_record', Ext::AppTypes::DB)
-          end
-        end
-
-        def self.adapter_name
-          @adapter_name ||= Datadog::Contrib::Rails::Utils.adapter_name
-        end
-
-        def self.database_name
-          @database_name ||= Datadog::Contrib::Rails::Utils.database_name
-        end
-
-        def self.adapter_host
-          @adapter_host ||= Datadog::Contrib::Rails::Utils.adapter_host
-        end
-
-        def self.adapter_port
-          @adapter_port ||= Datadog::Contrib::Rails::Utils.adapter_port
-        end
-
-        def self.sql(_name, start, finish, _id, payload)
-          span = get_option(:tracer).trace(
-            "#{adapter_name}.query",
-            resource: payload.fetch(:sql),
-            service: database_service_name,
-            span_type: Datadog::Ext::SQL::TYPE
-          )
+        def sql(span, event, _id, payload)
+          connection_config = Utils.connection_config(payload[:connection_id])
+          span.name = "#{connection_config[:adapter_name]}.query"
+          span.service = get_option(:service_name)
+          span.resource = payload.fetch(:sql)
+          span.span_type = Datadog::Ext::SQL::TYPE
 
           # Find out if the SQL query has been cached in this request. This meta is really
           # helpful to users because some spans may have 0ns of duration because the query
           # is simply cached from memory, so the notification is fired with start == finish.
           cached = payload[:cached] || (payload[:name] == 'CACHE')
 
-          # the span should have the query ONLY in the Resource attribute,
-          # so that the ``sql.query`` tag will be set in the agent with an
-          # obfuscated version
-          span.set_tag('active_record.db.vendor', adapter_name)
-          span.set_tag('active_record.db.name', database_name)
+          span.set_tag('active_record.db.vendor', connection_config[:adapter_name])
+          span.set_tag('active_record.db.name', connection_config[:database_name])
           span.set_tag('active_record.db.cached', cached) if cached
-          span.set_tag('out.host', adapter_host)
-          span.set_tag('out.port', adapter_port)
-          span.start_time = start
-          span.finish(finish)
+          span.set_tag('out.host', connection_config[:adapter_host])
+          span.set_tag('out.port', connection_config[:adapter_port])
         rescue StandardError => e
           Datadog::Tracer.log.debug(e.message)
         end
 
-        def self.instantiation(_name, start, finish, _id, payload)
-          span = get_option(:tracer).trace(
-            'active_record.instantiation',
-            resource: payload.fetch(:class_name),
-            span_type: 'custom'
-          )
-
+        def instantiation(span, event, _id, payload)
           # Inherit service name from parent, if available.
           span.service = if get_option(:orm_service_name)
                            get_option(:orm_service_name)
@@ -125,10 +110,10 @@ module Datadog
                            'active_record'
                          end
 
+          span.resource = payload.fetch(:class_name)
+          span.span_type = 'custom'
           span.set_tag('active_record.instantiation.class_name', payload.fetch(:class_name))
           span.set_tag('active_record.instantiation.record_count', payload.fetch(:record_count))
-          span.start_time = start
-          span.finish(finish)
         rescue StandardError => e
           Datadog::Tracer.log.debug(e.message)
         end