ddtrace 0.13.0 → 0.47.0

data/lib/ddtrace/context.rb

@@ -1,4 +1,8 @@
-require 'thread'
+require 'ddtrace/diagnostics/health'
+
+require 'ddtrace/context_flush'
+require 'ddtrace/context_provider'
+require 'ddtrace/utils/forking'
 
 module Datadog
   # \Context is used to keep track of a hierarchy of spans for the current
@@ -15,6 +19,8 @@ module Datadog
   # This data structure is thread-safe.
   # rubocop:disable Metrics/ClassLength
   class Context
+    include Datadog::Utils::Forking
+
     # 100k spans is about a 100Mb footprint
     DEFAULT_MAX_LENGTH = 100_000
 
@@ -53,6 +59,18 @@ module Datadog
       end
     end
 
+    def origin
+      @mutex.synchronize do
+        @origin
+      end
+    end
+
+    def origin=(origin)
+      @mutex.synchronize do
+        @origin = origin
+      end
+    end
+
     # Return the last active span that corresponds to the last inserted
     # item in the trace list. This cannot be considered as the current active
     # span in asynchronous environments, because some spans can be closed
@@ -63,6 +81,12 @@ module Datadog
       end
     end
 
+    def current_root_span
+      @mutex.synchronize do
+        return @current_root_span
+      end
+    end
+
     # Add a span to the context trace list, keeping it as the last active span.
     def add_span(span)
       @mutex.synchronize do
@@ -71,12 +95,21 @@ module Datadog
         # by default has 10000 spans, all of which belong to unfinished parts of a
         # larger trace. This is a catch-all to reduce global memory usage.
         if @max_length > 0 && @trace.length >= @max_length
-          Datadog::Tracer.log.debug("context full, ignoring span #{span.name}")
           # Detach the span from any context, it's being dropped and ignored.
           span.context = nil
+          Datadog.logger.debug("context full, ignoring span #{span.name}")
+
+          # If overflow has already occurred, don't send this metric.
+          # Prevents metrics spam if buffer repeatedly overflows for the same trace.
+          unless @overflow
+            Datadog.health_metrics.error_context_overflow(1, tags: ["max_length:#{@max_length}"])
+            @overflow = true
+          end
+
           return
         end
         set_current_span(span)
+        @current_root_span = span if @trace.empty?
         @trace << span
         span.context = self
       end
@@ -92,12 +125,19 @@ module Datadog
         # on per-instrumentation code to retrieve handle parent/child relations.
         set_current_span(span.parent)
         return if span.tracer.nil?
-        return unless Datadog::Tracer.debug_logging
-        if span.parent.nil? && !check_finished_spans
-          opened_spans = @trace.length - @finished_spans
-          Datadog::Tracer.log.debug("root span #{span.name} closed but has #{opened_spans} unfinished spans:")
-          @trace.each do |s|
-            Datadog::Tracer.log.debug("unfinished span: #{s}") unless s.finished?
+
+        if span.parent.nil? && !all_spans_finished?
+          if Datadog.configuration.diagnostics.debug
+            opened_spans = @trace.length - @finished_spans
+            Datadog.logger.debug("root span #{span.name} closed but has #{opened_spans} unfinished spans:")
+          end
+
+          @trace.reject(&:finished?).group_by(&:name).each do |unfinished_span_name, unfinished_spans|
+            Datadog.logger.debug("unfinished span: #{unfinished_spans.first}") if Datadog.configuration.diagnostics.debug
+            Datadog.health_metrics.error_unfinished_spans(
+              unfinished_spans.length,
+              tags: ["name:#{unfinished_span_name}"]
+            )
           end
         end
       end
@@ -107,7 +147,14 @@ module Datadog
     # is considered finished if all spans in this context are finished.
     def finished?
       @mutex.synchronize do
-        return check_finished_spans
+        return all_spans_finished?
+      end
+    end
+
+    # @@return [Numeric] numbers of finished spans
+    def finished_span_count
+      @mutex.synchronize do
+        @finished_spans
       end
     end
 
@@ -120,34 +167,89 @@ module Datadog
     end
 
     # Returns both the trace list generated in the current context and
-    # if the context is sampled or not. It returns nil, nil if the ``Context`` is
-    # not finished. If a trace is returned, the \Context will be reset so that it
+    # if the context is sampled or not.
+    #
+    # It returns +[nil,@sampled]+ if the \Context is
+    # not finished.
+    #
+    # If a trace is returned, the \Context will be reset so that it
     # can be re-used immediately.
     #
     # This operation is thread-safe.
+    #
+    # @return [Array<Array<Span>, Boolean>] finished trace and sampled flag
     def get
       @mutex.synchronize do
         trace = @trace
         sampled = @sampled
 
-        attach_sampling_priority if sampled && @sampling_priority
-
         # still return sampled attribute, even if context is not finished
-        return nil, sampled unless check_finished_spans()
+        return nil, sampled unless all_spans_finished?
+
+        # Root span is finished at this point, we can configure it
+        annotate_for_flush!(@current_root_span)
 
         reset
         [trace, sampled]
       end
     end
 
+    # Delete any span matching the condition. This is thread safe.
+    #
+    # @return [Array<Span>] deleted spans
+    def delete_span_if
+      @mutex.synchronize do
+        [].tap do |deleted_spans|
+          @trace.delete_if do |span|
+            finished = span.finished?
+
+            next unless yield span
+
+            deleted_spans << span
+
+            # We need to detach the span from the context, else, some code
+            # finishing it afterwards would mess up with the number of
+            # finished_spans and possibly cause other side effects.
+            span.context = nil
+            # Acknowledge there's one span less to finish, if needed.
+            # It's very important to keep this balanced.
+            @finished_spans -= 1 if finished
+
+            true
+          end
+        end
+      end
+    end
+
+    # Set tags to root span required for flush
+    def annotate_for_flush!(span)
+      attach_sampling_priority(span) if @sampled && @sampling_priority
+      attach_origin(span) if @origin
+    end
+
     # Return a string representation of the context.
     def to_s
       @mutex.synchronize do
-        # rubocop:disable Metrics/LineLength
+        # rubocop:disable Layout/LineLength
         "Context(trace.length:#{@trace.length},sampled:#{@sampled},finished_spans:#{@finished_spans},current_span:#{@current_span})"
       end
     end
 
+    # Generates equivalent context for forked processes.
+    #
+    # When Context from parent process is forked, child process
+    # should have a Context belonging to the same trace but not
+    # have the parent process spans.
+    def fork_clone
+      self.class.new(
+        trace_id: trace_id,
+        span_id: span_id,
+        sampled: sampled?,
+        sampling_priority: sampling_priority,
+        origin: origin
+      )
+    end
+
     private
 
     def reset(options = {})
@@ -156,8 +258,11 @@ module Datadog
       @parent_span_id = options.fetch(:span_id, nil)
       @sampled = options.fetch(:sampled, false)
       @sampling_priority = options.fetch(:sampling_priority, nil)
+      @origin = options.fetch(:origin, nil)
       @finished_spans = 0
       @current_span = nil
+      @current_root_span = nil
+      @overflow = false
     end
 
     def set_current_span(span)
@@ -173,21 +278,29 @@ module Datadog
 
     # Returns if the trace for the current Context is finished or not.
     # Low-level internal function, not thread-safe.
-    def check_finished_spans
+    def all_spans_finished?
       @finished_spans > 0 && @trace.length == @finished_spans
     end
 
-    def attach_sampling_priority
-      @trace.first.set_metric(
+    def attach_sampling_priority(span)
+      span.set_metric(
         Ext::DistributedTracing::SAMPLING_PRIORITY_KEY,
         @sampling_priority
       )
     end
 
+    def attach_origin(span)
+      span.set_tag(
+        Ext::DistributedTracing::ORIGIN_KEY,
+        @origin
+      )
+    end
+
     # Return the start time of the root span, or nil if there are no spans or this is undefined.
     def start_time
       @mutex.synchronize do
         return nil if @trace.empty?
+
         @trace[0].start_time
       end
     end
@@ -200,56 +313,10 @@ module Datadog
     end
 
     # Iterate on each span within the trace. This is thread safe.
-    def each_span
+    def each_span(&block)
       @mutex.synchronize do
-        @trace.each do |span|
-          yield span
-        end
+        @trace.each(&block)
       end
     end
-
-    # Delete any span matching the condition. This is thread safe.
-    def delete_span_if
-      @mutex.synchronize do
-        @trace.delete_if do |span|
-          finished = span.finished?
-          delete_span = yield span
-          if delete_span
-            # We need to detach the span from the context, else, some code
-            # finishing it afterwards would mess up with the number of
-            # finished_spans and possibly cause other side effects.
-            span.context = nil
-            # Acknowledge there's one span less to finish, if needed.
-            # It's very important to keep this balanced.
-            @finished_spans -= 1 if finished
-          end
-          delete_span
-        end
-      end
-    end
-  end
-
-  # ThreadLocalContext can be used as a tracer global reference to create
-  # a different \Context for each thread. In synchronous tracer, this
-  # is required to prevent multiple threads sharing the same \Context
-  # in different executions.
-  class ThreadLocalContext
-    # ThreadLocalContext can be used as a tracer global reference to create
-    # a different \Context for each thread. In synchronous tracer, this
-    # is required to prevent multiple threads sharing the same \Context
-    # in different executions.
-    def initialize
-      self.local = Datadog::Context.new
-    end
-
-    # Override the thread-local context with a new context.
-    def local=(ctx)
-      Thread.current[:datadog_context] = ctx
-    end
-
-    # Return the thread-local context.
-    def local
-      Thread.current[:datadog_context] ||= Datadog::Context.new
-    end
   end
 end

data/lib/ddtrace/context_flush.rb

@@ -1,132 +1,69 @@
-require 'set'
-
-require 'ddtrace/context'
-
 module Datadog
-  # \ContextFlush is used to cap context size and avoid it using too much memory.
-  # It performs memory flushes when required.
-  class ContextFlush
-    # by default, soft and hard limits are the same
-    DEFAULT_MAX_SPANS_BEFORE_PARTIAL_FLUSH = Datadog::Context::DEFAULT_MAX_LENGTH
-    # by default, never do a partial flush
-    DEFAULT_MIN_SPANS_BEFORE_PARTIAL_FLUSH = Datadog::Context::DEFAULT_MAX_LENGTH
-    # timeout should be lower than the trace agent window
-    DEFAULT_PARTIAL_FLUSH_TIMEOUT = 10
-
-    private_constant :DEFAULT_MAX_SPANS_BEFORE_PARTIAL_FLUSH
-    private_constant :DEFAULT_MIN_SPANS_BEFORE_PARTIAL_FLUSH
-    private_constant :DEFAULT_PARTIAL_FLUSH_TIMEOUT
-
-    def initialize(options = {})
-      # max_spans_before_partial_flush is the amount of spans collected before
-      # the context starts to partially flush parts of traces. With a setting of 10k,
-      # the memory overhead is about 10Mb per thread/context (depends on spans metadata,
-      # this is just an order of magnitude).
-      @max_spans_before_partial_flush = options.fetch(:max_spans_before_partial_flush,
-                                                      DEFAULT_MAX_SPANS_BEFORE_PARTIAL_FLUSH)
-      # min_spans_before_partial_flush is the minimum number of spans required
-      # for a partial flush to happen on a timeout. This is to prevent partial flush
-      # of traces which last a very long time but yet have few spans.
-      @min_spans_before_partial_flush = options.fetch(:min_spans_before_partial_flush,
-                                                      DEFAULT_MIN_SPANS_BEFORE_PARTIAL_FLUSH)
-      # partial_flush_timeout is the limit (in seconds) above which the context
-      # considers flushing parts of the trace. Partial flushes should not be done too
-      # late else the agent rejects them with a "too far in the past" error.
-      @partial_flush_timeout = options.fetch(:partial_flush_timeout,
-                                             DEFAULT_PARTIAL_FLUSH_TIMEOUT)
-      @partial_traces = []
+  module ContextFlush
+    # Consumes only completed traces (where all spans have finished)
+    class Finished
+      # Consumes and returns completed traces (where all spans have finished)
+      # from the provided +context+, if any.
+      #
+      # Any traces consumed are removed from +context+ as a side effect.
+      #
+      # @return [Array<Span>] trace to be flushed, or +nil+ if the trace is not finished
+      def consume!(context)
+        trace, sampled = context.get
+        trace if sampled
+      end
     end
 
-    def add_children(m, spans, ids, leaf)
-      spans << leaf
-      ids.add(leaf.span_id)
+    # Performs partial trace flushing to avoid large traces residing in memory for too long
+    class Partial
+      # Start flushing partial trace after this many active spans in one trace
+      DEFAULT_MIN_SPANS_FOR_PARTIAL_FLUSH = 500
 
-      if m[leaf.span_id]
-        m[leaf.span_id].each do |sub|
-          add_children(m, spans, ids, sub)
-        end
+      def initialize(options = {})
+        @min_spans_for_partial = options.fetch(:min_spans_before_partial_flush, DEFAULT_MIN_SPANS_FOR_PARTIAL_FLUSH)
       end
-    end
 
-    def partial_traces(context)
-      # 1st step, taint all parents of an unfinished span as unflushable
-      unflushable_ids = Set.new
+      # Consumes and returns completed or partially completed
+      # traces from the provided +context+, if any.
+      #
+      # Partially completed traces, where not all spans have finished,
+      # will only be returned if there are at least
+      # +@min_spans_for_partial+ finished spans.
+      #
+      # Any spans consumed are removed from +context+ as a side effect.
+      #
+      # @return [Array<Span>] partial or complete trace to be flushed, or +nil+ if no spans are finished
+      def consume!(context)
+        trace, sampled = context.get
 
-      context.send(:each_span) do |span|
-        next if span.finished? || unflushable_ids.include?(span.span_id)
-        unflushable_ids.add span.span_id
-        while span.parent
-          span = span.parent
-          unflushable_ids.add span.span_id
-        end
-      end
+        return nil unless sampled
+        return trace if trace && !trace.empty?
 
-      # 2nd step, find all spans which are at the border between flushable and unflushable
-      # Along the road, collect a reverse-tree which allows direct walking from parents to
-      # children but only for the ones we're interested it.
-      roots = []
-      children_map = {}
-      context.send(:each_span) do |span|
-        # There's no point in trying to put the real root in those partial roots, if
-        # it's flushable, the default algorithm would figure way more quickly.
-        if span.parent && !unflushable_ids.include?(span.span_id)
-          if unflushable_ids.include?(span.parent.span_id)
-            # span is flushable but is parent is not
-            roots << span
-          else
-            # span is flushable and its parent is too, build the reverse
-            # parent to child map for this one, it will be useful
-            children_map[span.parent.span_id] ||= []
-            children_map[span.parent.span_id] << span
-          end
-        end
+        partial_trace(context)
       end
 
-      # 3rd step, find all children, as this can be costly, only perform it for partial roots
-      partial_traces = []
-      all_ids = Set.new
-      roots.each do |root|
-        spans = []
-        add_children(children_map, spans, all_ids, root)
-        partial_traces << spans
-      end
+      private
 
-      return [nil, nil] if partial_traces.empty?
-      [partial_traces, all_ids]
-    end
+      def partial_trace(context)
+        return nil if context.finished_span_count < @min_spans_for_partial
+
+        finished_spans(context)
+      end
 
-    def partial_flush(context)
-      traces, flushed_ids = partial_traces(context)
-      return nil unless traces && flushed_ids
+      def finished_spans(context)
+        trace = context.delete_span_if(&:finished?)
 
-      # We need to reject by span ID and not by value, because a span
-      # value may be altered (typical example: it's finished by some other thread)
-      # since we lock only the context, not all the spans which belong to it.
-      context.send(:delete_span_if) { |span| flushed_ids.include? span.span_id }
-      traces
-    end
+        # Ensure that the first span in a partial trace has
+        # sampling and origin information.
+        if trace[0]
+          context.annotate_for_flush!(trace[0])
+        else
+          Datadog.logger.debug('Tried to retrieve trace from context, but got nothing. ' \
+            "Is there another consumer for this context? #{context.trace_id}")
+        end
 
-    # Performs an operation which each partial trace it can get from the context.
-    def each_partial_trace(context)
-      start_time = context.send(:start_time)
-      length = context.send(:length)
-      # Stop and do not flush anything if there are not enough spans.
-      return if length <= @min_spans_before_partial_flush
-      # If there are enough spans, but not too many, check for start time.
-      # If timeout is not given or 0, then wait
-      return if length <= @max_spans_before_partial_flush &&
-                (@partial_flush_timeout.nil? || @partial_flush_timeout <= 0 ||
-                 (start_time && start_time > Time.now.utc - @partial_flush_timeout))
-      # Here, either the trace is old or we have too many spans, flush it.
-      traces = partial_flush(context)
-      return unless traces
-      traces.each do |trace|
-        yield trace
+        trace unless trace.empty?
       end
     end
-
-    private :add_children
-    private :partial_traces
-    private :partial_flush
   end
 end