ddtrace 0.38.0 → 0.43.0

data/lib/ddtrace/ext/diagnostics.rb

@@ -2,7 +2,8 @@ module Datadog
   module Ext
     module Diagnostics
       DD_TRACE_STARTUP_LOGS = 'DD_TRACE_STARTUP_LOGS'.freeze
-
+      DD_TRACE_DEBUG = 'DD_TRACE_DEBUG'.freeze
+      DD_TRACE_ENABLED = 'DD_TRACE_ENABLED'.freeze
       # Health
       module Health
         # Metrics

data/lib/ddtrace/ext/distributed.rb

@@ -20,8 +20,14 @@ module Datadog
       PROPAGATION_STYLE_DATADOG = 'Datadog'.freeze
       PROPAGATION_STYLE_B3 = 'B3'.freeze
       PROPAGATION_STYLE_B3_SINGLE_HEADER = 'B3 single header'.freeze
-      PROPAGATION_INJECT_STYLE_ENV = 'DD_PROPAGATION_INJECT_STYLE'.freeze
-      PROPAGATION_EXTRACT_STYLE_ENV = 'DD_PROPAGATION_EXTRACT_STYLE'.freeze
+      PROPAGATION_STYLE_INJECT_ENV = 'DD_PROPAGATION_STYLE_INJECT'.freeze
+      PROPAGATION_STYLE_EXTRACT_ENV = 'DD_PROPAGATION_STYLE_EXTRACT'.freeze
+      # Note: the below inject/extract values are deprecated and were defined erronously
+      # they were never part of the datadog language client standard or documentation
+      # some users may already be relying on them, but we should look to remove these in the future
+      # or before 1.0.
+      PROPAGATION_INJECT_STYLE_ENV_OLD = 'DD_PROPAGATION_INJECT_STYLE'.freeze
+      PROPAGATION_EXTRACT_STYLE_ENV_OLD = 'DD_PROPAGATION_EXTRACT_STYLE'.freeze
 
       # gRPC metadata keys for distributed tracing. https://github.com/grpc/grpc-go/blob/v1.10.x/Documentation/grpc-metadata.md
       GRPC_METADATA_TRACE_ID = 'x-datadog-trace-id'.freeze

data/lib/ddtrace/ext/git.rb

@@ -0,0 +1,12 @@
+module Datadog
+  module Ext
+    # Defines constants for Git tags
+    module Git
+      TAG_BRANCH = 'git.branch'.freeze
+      TAG_COMMIT_SHA = 'git.commit.sha'.freeze
+      TAG_DEPRECATED_COMMIT_SHA = 'git.commit_sha'.freeze
+      TAG_REPOSITORY_URL = 'git.repository_url'.freeze
+      TAG_TAG = 'git.tag'.freeze
+    end
+  end
+end

data/lib/ddtrace/ext/integration.rb

@@ -0,0 +1,8 @@
+module Datadog
+  module Ext
+    module Integration
+      # Name of external service that performed the work
+      TAG_PEER_SERVICE = 'peer.service'.freeze
+    end
+  end
+end

data/lib/ddtrace/ext/runtime.rb

@@ -7,8 +7,10 @@ module Datadog
       LANG = 'ruby'.freeze
       LANG_INTERPRETER = (RUBY_ENGINE + '-' + RUBY_PLATFORM).freeze
       LANG_VERSION = RUBY_VERSION
+      RUBY_ENGINE =  ::RUBY_ENGINE # e.g. 'ruby', 'jruby', 'truffleruby'
       TRACER_VERSION = Datadog::VERSION::STRING
 
+      TAG_ID = 'runtime-id'.freeze
       TAG_LANG = 'language'.freeze
 
       # Metrics

data/lib/ddtrace/ext/test.rb

@@ -0,0 +1,24 @@
+module Datadog
+  module Ext
+    # Defines constants for test tags
+    module Test
+      TAG_ARGUMENTS = 'test.arguments'.freeze
+      TAG_FRAMEWORK = 'test.framework'.freeze
+      TAG_NAME = 'test.name'.freeze
+      TAG_SKIP_REASON = 'test.skip_reason'.freeze
+      TAG_STATUS = 'test.status'.freeze
+      TAG_SUITE = 'test.suite'.freeze
+      TAG_TRAITS = 'test.traits'.freeze
+      TAG_TYPE = 'test.type'.freeze
+
+      # TODO: is there a better place for SPAN_KIND?
+      TAG_SPAN_KIND = 'span.kind'.freeze
+
+      module Status
+        PASS = 'pass'.freeze
+        FAIL = 'fail'.freeze
+        SKIP = 'skip'.freeze
+      end
+    end
+  end
+end

data/lib/ddtrace/ext/transport.rb

@@ -6,6 +6,7 @@ module Datadog
         DEFAULT_PORT = 8126
         ENV_DEFAULT_HOST = 'DD_AGENT_HOST'.freeze
         ENV_DEFAULT_PORT = 'DD_TRACE_AGENT_PORT'.freeze
+        ENV_DEFAULT_URL = 'DD_TRACE_AGENT_URL'.freeze
         HEADER_CONTAINER_ID = 'Datadog-Container-ID'.freeze
         HEADER_META_LANG = 'Datadog-Meta-Lang'.freeze
         HEADER_META_LANG_VERSION = 'Datadog-Meta-Lang-Version'.freeze

data/lib/ddtrace/logger.rb

@@ -10,7 +10,7 @@ module Datadog
     def initialize(*args, &block)
       super
       self.progname = PREFIX
-      self.level = ::Logger::WARN
+      self.level = ::Logger::INFO
     end
 
     def add(severity, message = nil, progname = nil, &block)

data/lib/ddtrace/opentracer/distributed_headers.rb

@@ -44,7 +44,7 @@ module Datadog
 
       def id(header)
         value = @carrier[header].to_i
-        return if value.zero? || value >= Datadog::Span::MAX_ID
+        return if value.zero? || value >= Datadog::Span::EXTERNAL_MAX_ID
         value < 0 ? value + 0x1_0000_0000_0000_0000 : value
       end
     end

data/lib/ddtrace/propagation/grpc_propagator.rb

@@ -38,24 +38,36 @@ module Datadog
       end
 
       def trace_id
-        value = @metadata[GRPC_METADATA_TRACE_ID].to_i
-        value if (1..Span::MAX_ID).cover? value
+        value = metadata_for_key(GRPC_METADATA_TRACE_ID).to_i
+        value if (1..Span::EXTERNAL_MAX_ID).cover? value
       end
 
       def parent_id
-        value = @metadata[GRPC_METADATA_PARENT_ID].to_i
-        value if (1..Span::MAX_ID).cover? value
+        value = metadata_for_key(GRPC_METADATA_PARENT_ID).to_i
+        value if (1..Span::EXTERNAL_MAX_ID).cover? value
       end
 
       def sampling_priority
-        value = @metadata[GRPC_METADATA_SAMPLING_PRIORITY]
+        value = metadata_for_key(GRPC_METADATA_SAMPLING_PRIORITY)
         value && value.to_i
       end
 
       def origin
-        value = @metadata[GRPC_METADATA_ORIGIN]
+        value = metadata_for_key(GRPC_METADATA_ORIGIN)
         value if value != ''
       end
+
+      private
+
+      def metadata_for_key(key)
+        # metadata values can be arrays (multiple headers with the same key)
+        value = @metadata[key]
+        if value.is_a?(Array)
+          value.first
+        else
+          value
+        end
+      end
     end
   end
 end

data/lib/ddtrace/runtime/identity.rb

@@ -1,22 +1,21 @@
 require 'securerandom'
 require 'ddtrace/ext/runtime'
+require 'ddtrace/utils/forking'
 
 module Datadog
   module Runtime
     # For runtime identity
     module Identity
+      extend Datadog::Utils::Forking
+
       module_function
 
       # Retrieves number of classes from runtime
       def id
-        @pid ||= Process.pid
         @id ||= SecureRandom.uuid
 
         # Check if runtime has changed, e.g. forked.
-        if Process.pid != @pid
-          @pid = Process.pid
-          @id = SecureRandom.uuid
-        end
+        after_fork! { @id = SecureRandom.uuid }
 
         @id
       end

data/lib/ddtrace/runtime/metrics.rb

@@ -1,3 +1,4 @@
+require 'ddtrace/ext/integration'
 require 'ddtrace/ext/runtime'
 
 require 'ddtrace/metrics'
@@ -25,8 +26,11 @@ module Datadog
         # Register service as associated with metrics
         register_service(span.service) unless span.service.nil?
 
-        # Tag span with language and runtime ID for association with metrics
-        span.set_tag(Ext::Runtime::TAG_LANG, Runtime::Identity.lang)
+        # Tag span with language and runtime ID for association with metrics.
+        # We only tag spans that performed internal application work.
+        unless span.get_tag(Datadog::Ext::Integration::TAG_PEER_SERVICE)
+          span.set_tag(Ext::Runtime::TAG_LANG, Runtime::Identity.lang)
+        end
       end
 
       # Associate service with runtime metrics
@@ -55,10 +59,8 @@ module Datadog
 
       def gc_metrics
         Hash[
-          GC.stat.map do |k, v|
-            next if v.is_a?(Hash) # TODO: JRuby supports additional nested metrics
-
-            ["#{Ext::Runtime::Metrics::METRIC_GC_PREFIX}.#{k}", v]
+          GC.stat.flat_map do |k, v|
+            nested_gc_metric(Ext::Runtime::Metrics::METRIC_GC_PREFIX, k, v)
           end
         ]
       end
@@ -91,6 +93,22 @@ module Datadog
           "#{Ext::Runtime::Metrics::TAG_SERVICE}:#{service}".freeze
         end
       end
+
+      def nested_gc_metric(prefix, k, v)
+        path = "#{prefix}.#{k}"
+
+        if v.is_a?(Hash)
+          v.flat_map do |key, value|
+            nested_gc_metric(path, key, value)
+          end
+        else
+          [[to_metric_name(path), v]]
+        end
+      end
+
+      def to_metric_name(str)
+        str.downcase.gsub(/[-\s]/, '_')
+      end
     end
   end
 end

data/lib/ddtrace/sampler.rb

@@ -61,11 +61,11 @@ module Datadog
 
     def sample_rate=(sample_rate)
       @sample_rate = sample_rate
-      @sampling_id_threshold = sample_rate * Span::MAX_ID
+      @sampling_id_threshold = sample_rate * Span::EXTERNAL_MAX_ID
     end
 
     def sample?(span)
-      ((span.trace_id * KNUTH_FACTOR) % Datadog::Span::MAX_ID) <= @sampling_id_threshold
+      ((span.trace_id * KNUTH_FACTOR) % Datadog::Span::EXTERNAL_MAX_ID) <= @sampling_id_threshold
     end
 
     def sample!(span)

data/lib/ddtrace/sampling/rate_limiter.rb

@@ -38,6 +38,10 @@ module Datadog
         @tokens = max_tokens
         @total_messages = 0
         @conforming_messages = 0
+        @prev_conforming_messages = nil
+        @prev_total_messages = nil
+        @current_window = nil
+
         @last_refill = Utils::Time.get_time
       end
 
@@ -47,28 +51,17 @@ module Datadog
       # If it does, return +true+ and remove +size+
       # tokens from the bucket.
       # If it does not, return +false+ without affecting
-      # the tokens form the bucket.
+      # the tokens from the bucket.
       #
       # @return [Boolean] +true+ if message conforms with current bucket limit
       def allow?(size)
-        return false if @rate.zero?
-        return true if @rate < 0
-
-        refill_since_last_message
-
-        increment_total_count
-
-        return false if @tokens < size
-
-        increment_conforming_count
-
-        @tokens -= size
-
-        true
+        allowed = should_allow?(size)
+        update_rate_counts(allowed)
+        allowed
       end
 
       # Ratio of 'conformance' per 'total messages' checked
-      # on this bucket.
+      # averaged for the past 2 buckets
       #
       # Returns +1.0+ when no messages have been checked yet.
       #
@@ -77,6 +70,20 @@ module Datadog
         return 0.0 if @rate.zero?
         return 1.0 if @rate < 0 || @total_messages.zero?
 
+        return current_window_rate if @prev_conforming_messages.nil? || @prev_total_messages.nil?
+
+        (@conforming_messages.to_f + @prev_conforming_messages.to_f) / (@total_messages + @prev_total_messages)
+      end
+
+      # Ratio of 'conformance' per 'total messages' checked
+      # on this bucket
+      #
+      # Returns +1.0+ when no messages have been checked yet.
+      #
+      # @return [Float] Conformance ratio, between +[0,1]+
+      def current_window_rate
+        return 1.0 if @total_messages.zero?
+
         @conforming_messages.to_f / @total_messages
       end
 
@@ -91,6 +98,8 @@ module Datadog
         now = Utils::Time.get_time
         elapsed = now - @last_refill
 
+        # Update the number of available tokens, but ensure we do not exceed the max
+        # we return the min of tokens + rate*elapsed, or max tokens
         refill_tokens(@rate * elapsed)
 
         @last_refill = now
@@ -108,6 +117,46 @@ module Datadog
       def increment_conforming_count
         @conforming_messages += 1
       end
+
+      def should_allow?(size)
+        # rate limit of 0 blocks everything
+        return false if @rate.zero?
+
+        # negative rate limit disables rate limiting
+        return true if @rate < 0
+
+        refill_since_last_message
+
+        # if tokens < 1 we don't allow?
+        return false if @tokens < size
+
+        @tokens -= size
+
+        true
+      end
+
+      # Sets and Updates the past two 1 second windows for which
+      # the rate limiter must compute it's rate over and updates
+      # the total count, and conforming message count if +allowed+
+      def update_rate_counts(allowed)
+        now = Utils::Time.get_time
+
+        # No tokens have been seen yet, start a new window
+        if @current_window.nil?
+          @current_window = now
+        # If more than 1 second has past since last window, reset
+        elsif now - @current_window >= 1
+          @prev_conforming_messages = @conforming_messages
+          @prev_total_messages = @total_messages
+          @conforming_messages = 0
+          @total_messages = 0
+          @current_window = now
+        end
+
+        increment_conforming_count if allowed
+
+        increment_total_count
+      end
     end
 
     # \RateLimiter that accepts all resources,

data/lib/ddtrace/span.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'time'
 require 'thread'
 
@@ -8,6 +10,7 @@ require 'ddtrace/environment'
 require 'ddtrace/analytics'
 require 'ddtrace/forced_tracing'
 require 'ddtrace/diagnostics/health'
+require 'ddtrace/utils/time'
 
 module Datadog
   # Represents a logical unit of work in the system. Each trace consists of one or more spans.
@@ -24,24 +27,26 @@ module Datadog
     # The max value for a \Span identifier.
     # Span and trace identifiers should be strictly positive and strictly inferior to this limit.
     #
-    # Limited to 63-bit positive integers, as some other languages might be limited to this,
-    # and IDs need to be easy to port across various languages and platforms.
-    MAX_ID = 2**63
+    # Limited to +2<<62-1+ positive integers, as Ruby is able to represent such numbers "inline",
+    # inside a +VALUE+ scalar, thus not requiring memory allocation.
+    #
+    # The range of IDs also has to consider portability across different languages and platforms.
+    RUBY_MAX_ID = (1 << 62) - 1
 
     # While we only generate 63-bit integers due to limitations in other languages, we support
     # parsing 64-bit integers for distributed tracing since an upstream system may generate one
-    EXTERNAL_MAX_ID = 2**64
+    EXTERNAL_MAX_ID = 1 << 64
 
     # This limit is for numeric tags because uint64 could end up rounded.
-    NUMERIC_TAG_SIZE_RANGE = (-2**53..2**53)
+    NUMERIC_TAG_SIZE_RANGE = (-1 << 53..1 << 53)
 
     attr_accessor :name, :service, :resource, :span_type,
-                  :start_time, :end_time,
                   :span_id, :trace_id, :parent_id,
                   :status, :sampled,
-                  :tracer, :context
+                  :tracer, :context, :duration, :start_time, :end_time
 
     attr_reader :parent
+
     # Create a new span linked to the given tracer. Call the \Tracer method <tt>start_span()</tt>
     # and then <tt>finish()</tt> once the tracer operation is over.
     #
@@ -72,11 +77,20 @@ module Datadog
       @parent = nil
       @sampled = true
 
-      @start_time = nil # set by Tracer.start_span
-      @end_time = nil # set by Span.finish
-
       @allocation_count_start = now_allocations
       @allocation_count_finish = @allocation_count_start
+
+      # start_time and end_time track wall clock. In Ruby, wall clock
+      # has less accuracy than monotonic clock, so if possible we look to only use wall clock
+      # to measure duration when a time is supplied by the user, or if monotonic clock
+      # is unsupported.
+      @start_time = nil
+      @end_time = nil
+
+      # duration_start and duration_end track monotonic clock, and may remain nil in cases where it
+      # is known that we have to use wall clock to measure duration.
+      @duration_start = nil
+      @duration_end = nil
     end
 
     # Set the given key / value tag pair on the span. Keys and values
@@ -160,6 +174,28 @@ module Datadog
       set_tag(Ext::Errors::STACK, e.backtrace) unless e.backtrace.empty?
     end
 
+    # Mark the span started at the current time.
+    def start(start_time = nil)
+      # A span should not be started twice. However, this is existing
+      # behavior and so we maintain it for backward compatibility for those
+      # who are using async manual instrumentation that may rely on this
+
+      @start_time = start_time || Time.now.utc
+      @duration_start = start_time.nil? ? duration_marker : nil
+
+      self
+    end
+
+    # for backwards compatibility
+    def start_time=(time)
+      time.tap { start(time) }
+    end
+
+    # for backwards compatibility
+    def end_time=(time)
+      time.tap { finish(time) }
+    end
+
     # Mark the span finished at the current time and submit it.
     def finish(finish_time = nil)
       # A span should not be finished twice. Note that this is not thread-safe,
@@ -170,12 +206,15 @@ module Datadog
 
       @allocation_count_finish = now_allocations
 
-      # Provide a default start_time if unset, but this should have been set by start_span.
-      # Using now here causes 0-duration spans, still, this is expected, as we never
-      # explicitely say when it started.
-      @start_time ||= Time.now.utc
+      now = Time.now.utc
+
+      # Provide a default start_time if unset.
+      # Using `now` here causes duration to be 0; this is expected
+      # behavior when start_time is unknown.
+      start(finish_time || now) unless started?
 
-      @end_time = finish_time.nil? ? Time.now.utc : finish_time # finish this
+      @end_time = finish_time || now
+      @duration_end = finish_time.nil? ? duration_marker : nil
 
       # Finish does not really do anything if the span is not bound to a tracer and a context.
       return self if @tracer.nil? || @context.nil?
@@ -195,11 +234,6 @@ module Datadog
       self
     end
 
-    # Return whether the span is finished or not.
-    def finished?
-      !@end_time.nil?
-    end
-
     # Return a string representation of the span.
     def to_s
       "Span(name:#{@name},sid:#{@span_id},tid:#{@trace_id},pid:#{@parent_id})"
@@ -246,19 +280,76 @@ module Datadog
         error: @status
       }
 
-      if !@start_time.nil? && !@end_time.nil?
-        h[:start] = (@start_time.to_f * 1e9).to_i
-        h[:duration] = ((@end_time - @start_time) * 1e9).to_i
+      if finished?
+        h[:start] = start_time_nano
+        h[:duration] = duration_nano
       end
 
       h
     end
 
+    # MessagePack serializer interface. Making this object
+    # respond to `#to_msgpack` allows it to be automatically
+    # serialized by MessagePack.
+    #
+    # This is more efficient than doing +MessagePack.pack(span.to_hash)+
+    # as we don't have to create an intermediate Hash.
+    #
+    # @param packer [MessagePack::Packer] serialization buffer, can be +nil+ with JRuby
+    def to_msgpack(packer = nil)
+      # As of 1.3.3, JRuby implementation doesn't pass an existing packer
+      packer ||= MessagePack::Packer.new
+
+      if finished?
+        packer.write_map_header(13) # Set header with how many elements in the map
+
+        packer.write('start')
+        packer.write(start_time_nano)
+
+        packer.write('duration')
+        packer.write(duration_nano)
+      else
+        packer.write_map_header(11) # Set header with how many elements in the map
+      end
+
+      # DEV: We use strings as keys here, instead of symbols, as
+      # DEV: MessagePack will ultimately convert them to strings.
+      # DEV: By providing strings directly, we skip this indirection operation.
+      packer.write('span_id')
+      packer.write(@span_id)
+      packer.write('parent_id')
+      packer.write(@parent_id)
+      packer.write('trace_id')
+      packer.write(@trace_id)
+      packer.write('name')
+      packer.write(@name)
+      packer.write('service')
+      packer.write(@service)
+      packer.write('resource')
+      packer.write(@resource)
+      packer.write('type')
+      packer.write(@span_type)
+      packer.write('meta')
+      packer.write(@meta)
+      packer.write('metrics')
+      packer.write(@metrics)
+      packer.write('allocations')
+      packer.write(allocations)
+      packer.write('error')
+      packer.write(@status)
+      packer
+    end
+
+    # JSON serializer interface.
+    # Used by older version of the transport.
+    def to_json(*args)
+      to_hash.to_json(*args)
+    end
+
     # Return a human readable version of the span
     def pretty_print(q)
-      start_time = (@start_time.to_f * 1e9).to_i rescue '-'
-      end_time = (@end_time.to_f * 1e9).to_i rescue '-'
-      duration = ((@end_time - @start_time) * 1e9).to_i rescue 0
+      start_time = (self.start_time.to_f * 1e9).to_i
+      end_time = (self.end_time.to_f * 1e9).to_i
       q.group 0 do
         q.breakable
         q.text "Name: #{@name}\n"
@@ -271,7 +362,7 @@ module Datadog
         q.text "Error: #{@status}\n"
         q.text "Start: #{start_time}\n"
         q.text "End: #{end_time}\n"
-        q.text "Duration: #{duration}\n"
+        q.text "Duration: #{duration.to_f if finished?}\n"
         q.text "Allocations: #{allocations}\n"
         q.group(2, 'Tags: [', "]\n") do
           q.breakable
@@ -288,8 +379,30 @@ module Datadog
       end
     end
 
+    # Return whether the duration is started or not
+    def started?
+      !@start_time.nil?
+    end
+
+    # Return whether the duration is finished or not.
+    def finished?
+      !@end_time.nil?
+    end
+
+    def duration
+      if @duration_end.nil? || @duration_start.nil?
+        @end_time - @start_time
+      else
+        @duration_end - @duration_start
+      end
+    end
+
     private
 
+    def duration_marker
+      Utils::Time.get_time
+    end
+
     if defined?(JRUBY_VERSION) || Gem::Version.new(RUBY_VERSION) < Gem::Version.new(VERSION::MINIMUM_RUBY_VERSION)
       def now_allocations
         0
@@ -303,5 +416,17 @@ module Datadog
         GC.stat(:total_allocated_objects)
       end
     end
+
+    # Used for serialization
+    # @return [Integer] in nanoseconds since Epoch
+    def start_time_nano
+      @start_time.to_i * 1000000000 + @start_time.nsec
+    end
+
+    # Used for serialization
+    # @return [Integer] in nanoseconds since Epoch
+    def duration_nano
+      (duration * 1e9).to_i
+    end
   end
 end