ddtrace 0.40.0 → 0.41.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 38d23bcce474aa2b77029ef98bcc73a47a929aa75e6b1c2779671248b0f238e2
-  data.tar.gz: 0723d895065ae562813f7a9d9474a330cb8e8902e0f5b6ed4a75370fdabe1a80
+  metadata.gz: 0bbda933f09b8f29d20edc57d6825f6491158161b7e5164252deb006931e0e7d
+  data.tar.gz: b3aaa6718e835fbfe97db64faa62f82b2f3ff76aefadb53cb27fb9b236b842e3
 SHA512:
-  metadata.gz: 28c1958972c5e046ae96670b6dcc6b454e8272f5c7ee6b6d0362f51d149db852d09de4fd76bf1eeac76a483c30ad9464845ec2cf3f052b95f432210c2f92da9e
-  data.tar.gz: d5b07614bc418f759fac5cf8be3fedf39d6354dd18f086c2461e5d494970b7c1b270585bea37066fd06843f8278ed7fa9863b883b4fdd7062ca4bc8e37c8e663
+  metadata.gz: 60d239de8c63091ede6a810127ad580629428b96181982cf543e77f45a89087dfdd238b98418fe72f0e27ebefae6d5f730208c5f6149541dfe1d1c6923579826
+  data.tar.gz: 1d3657bc28458d0b50d0c2c68d3397b20ef6b2d4c2a0adf2ec834f701fd2c7701bcdac7f6fbce913743482e5eb2492a6409baf7c9c37c3467677ead705fd752e

data/CHANGELOG.md

@@ -2,6 +2,38 @@
 
 ## [Unreleased]
 
+## [0.41.0] - 2020-09-30
+
+Release notes: https://github.com/DataDog/dd-trace-rb/releases/tag/v0.41.0
+
+Git diff: https://github.com/DataDog/dd-trace-rb/compare/v0.40.0...v0.41.0
+
+### Added
+
+- Improve duration counting using monotonic clock (#424, #1173) (@soulcutter)
+
+### Changed
+
+- Add peer.service tag to external services and skip tagging external services with language tag for runtime metrics (#934, #935, #1180)
+  - This helps support the way runtime metrics are associated with spans in the UI.
+- Faster TraceBuffer for CRuby (#1172)
+- Reduce memory usage during gem statup (#1090)
+- Reduce memory usage of the HTTP transport (#1165)
+
+### Fixed
+
+- Improved prepared statement support for Sequel  integrations (#1186)
+- Fix Sequel instrumentation when executing literal strings (#1185) (@matchbookmac)
+- Remove explicit `Logger` class verification (#1181) (@bartekbsh)
+  - This allows users to pass in a custom logger that does not inherit from `Logger` class.
+- Correct tracer buffer metric counting (#1182)
+- Fix Span#pretty_print for empty duration (#1183)
+
+### Refactored
+
+- Improvements to test suite & CI (#1179, #1184, #1177, #1178, #1176)
+- Reduce generated Span ID range to fit in Fixnum (#1189)
+
 ## [0.40.0] - 2020-09-08
 
 Release notes: https://github.com/DataDog/dd-trace-rb/releases/tag/v0.40.0
@@ -1395,7 +1427,8 @@ Release notes: https://github.com/DataDog/dd-trace-rb/releases/tag/v0.3.1
 
 Git diff: https://github.com/DataDog/dd-trace-rb/compare/v0.3.0...v0.3.1
 
-[Unreleased]: https://github.com/DataDog/dd-trace-rb/compare/v0.40.0...master
+[Unreleased]: https://github.com/DataDog/dd-trace-rb/compare/v0.41.0...master
+[0.41.0]: https://github.com/DataDog/dd-trace-rb/compare/v0.40.0...v0.41.0
 [0.40.0]: https://github.com/DataDog/dd-trace-rb/compare/v0.39.0...v0.40.0
 [0.39.0]: https://github.com/DataDog/dd-trace-rb/compare/v0.38.0...v0.39.0
 [0.38.0]: https://github.com/DataDog/dd-trace-rb/compare/v0.37.0...v0.38.0

data/Rakefile

@@ -209,8 +209,6 @@ task :ci do
     declare 'bundle exec rake test:main'
     declare 'bundle exec rake spec:main'
     declare 'bundle exec rake spec:contrib'
-    # Benchmarks
-    declare 'bundle exec rake spec:benchmark'
 
     if RUBY_PLATFORM != 'java'
       # Contrib minitests
@@ -265,8 +263,6 @@ task :ci do
     declare 'bundle exec rake spec:main'
     declare 'bundle exec rake spec:contrib'
     declare 'bundle exec rake spec:opentracer'
-    # Benchmarks
-    declare 'bundle exec rake spec:benchmark'
 
     if RUBY_PLATFORM != 'java'
       # Contrib minitests
@@ -328,8 +324,6 @@ task :ci do
     declare 'bundle exec rake spec:main'
     declare 'bundle exec rake spec:contrib'
     declare 'bundle exec rake spec:opentracer'
-    # Benchmarks
-    declare 'bundle exec rake spec:benchmark'
 
     if RUBY_PLATFORM != 'java'
       # Contrib minitests
@@ -403,8 +397,6 @@ task :ci do
     declare 'bundle exec rake spec:main'
     declare 'bundle exec rake spec:contrib'
     declare 'bundle exec rake spec:opentracer'
-    # Benchmarks
-    declare 'bundle exec rake spec:benchmark'
 
     if RUBY_PLATFORM != 'java'
       # Contrib minitests
@@ -481,12 +473,8 @@ task :ci do
     declare 'bundle exec rake spec:contrib'
     declare 'bundle exec rake spec:opentracer'
     declare 'bundle exec rake spec:opentelemetry'
-    # Benchmarks
-    declare 'bundle exec rake spec:benchmark'
 
     if RUBY_PLATFORM != 'java'
-      # Benchmarks
-      declare 'bundle exec rake benchmark'
       # Contrib minitests
       # Contrib specs
       declare 'bundle exec appraisal contrib rake spec:action_pack'
@@ -546,9 +534,6 @@ task :ci do
     declare 'bundle exec rake spec:contrib'
     declare 'bundle exec rake spec:opentracer'
     declare 'bundle exec rake spec:opentelemetry'
-    # Benchmarks
-    declare 'bundle exec rake spec:benchmark' if RUBY_PLATFORM != 'java' # Too slow due to repeated JVM instantiation
-    declare 'bundle exec rake benchmark'
     # Contrib minitests
     # Contrib specs
     declare 'bundle exec appraisal contrib rake spec:action_pack'
@@ -617,12 +602,8 @@ task :ci do
     declare 'bundle exec rake spec:contrib'
     declare 'bundle exec rake spec:opentracer'
     declare 'bundle exec rake spec:opentelemetry'
-    # Benchmarks
-    declare 'bundle exec rake spec:benchmark'
 
     if RUBY_PLATFORM != 'java'
-      # Benchmarks
-      declare 'bundle exec rake benchmark'
       # Contrib minitests
       # Contrib specs
       declare 'bundle exec appraisal contrib rake spec:action_pack'
@@ -691,12 +672,8 @@ task :ci do
     declare 'bundle exec rake spec:contrib'
     declare 'bundle exec rake spec:opentracer'
     declare 'bundle exec rake spec:opentelemetry'
-    # Benchmarks
-    declare 'bundle exec rake spec:benchmark'
 
     if RUBY_PLATFORM != 'java'
-      # Benchmarks
-      declare 'bundle exec rake benchmark'
       # Contrib minitests
       # Contrib specs
       declare 'bundle exec appraisal contrib rake spec:action_pack'

data/ddtrace.gemspec

@@ -41,6 +41,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'opentracing', '>= 0.4.1'
 
   # Development dependencies
+  spec.add_development_dependency 'concurrent-ruby' # Leave it open as we also have it as an integration and want Appraisal to control the version under test.
   spec.add_development_dependency 'rake', '>= 10.5'
   spec.add_development_dependency 'rubocop', '= 0.49.1' if RUBY_VERSION >= '2.1.0'
   spec.add_development_dependency 'rspec', '~> 3.0'

data/lib/ddtrace/buffer.rb

@@ -2,23 +2,79 @@ require 'thread'
 require 'ddtrace/diagnostics/health'
 require 'ddtrace/runtime/object_space'
 
+# Trace buffer that accumulates traces for a consumer.
+# Consumption can happen from a different thread.
 module Datadog
-  # Trace buffer that stores application traces. The buffer has a maximum size and when
-  # the buffer is full, a random trace is discarded. This class is thread-safe and is used
-  # automatically by the ``Tracer`` instance when a ``Span`` is finished.
-  class TraceBuffer
-    def initialize(max_size)
-      @max_size = max_size
+  # Aggregate metrics:
+  # They reflect buffer activity since last #pop.
+  # These may not be as accurate or as granular, but they
+  # don't use as much network traffic as live stats.
+  class MeasuredBuffer
+    def initialize
+      @buffer_accepted = 0
+      @buffer_accepted_lengths = 0
+      @buffer_dropped = 0
+      @buffer_spans = 0
+    end
 
-      @mutex = Mutex.new()
-      @traces = []
-      @closed = false
+    def measure_accept(trace)
+      @buffer_accepted += 1
+      @buffer_accepted_lengths += trace.length
+
+      @buffer_spans += trace.length
+    rescue StandardError => e
+      Datadog.logger.debug("Failed to measure queue accept. Cause: #{e.message} Source: #{e.backtrace.first}")
+    end
 
-      # Initialize metric values
+    def measure_drop(trace)
+      @buffer_dropped += 1
+
+      @buffer_spans -= trace.length
+    rescue StandardError => e
+      Datadog.logger.debug("Failed to measure queue drop. Cause: #{e.message} Source: #{e.backtrace.first}")
+    end
+
+    def measure_pop(traces)
+      # Accepted, cumulative totals
+      Datadog.health_metrics.queue_accepted(@buffer_accepted)
+      Datadog.health_metrics.queue_accepted_lengths(@buffer_accepted_lengths)
+
+      # Dropped, cumulative totals
+      Datadog.health_metrics.queue_dropped(@buffer_dropped)
+      # TODO: are we missing a +queue_dropped_lengths+ metric?
+
+      # Queue gauges, current values
+      Datadog.health_metrics.queue_max_length(@max_size)
+      Datadog.health_metrics.queue_spans(@buffer_spans)
+      Datadog.health_metrics.queue_length(traces.length)
+
+      # Reset aggregated metrics
       @buffer_accepted = 0
       @buffer_accepted_lengths = 0
       @buffer_dropped = 0
       @buffer_spans = 0
+    rescue StandardError => e
+      Datadog.logger.debug("Failed to measure queue. Cause: #{e.message} Source: #{e.backtrace.first}")
+    end
+  end
+
+  # Trace buffer that stores application traces and
+  # can be safely used concurrently on any environment.
+  #
+  # This implementation uses a {Mutex} around public methods, incurring
+  # overhead in order to ensure full thread-safety.
+  #
+  # This is implementation is recommended for non-CRuby environments.
+  # If using CRuby, {Datadog::CRubyTraceBuffer} is a faster implementation with minimal compromise.
+  class ThreadSafeBuffer < MeasuredBuffer
+    def initialize(max_size)
+      super()
+
+      @max_size = max_size
+
+      @mutex = Mutex.new()
+      @traces = []
+      @closed = false
     end
 
     # Add a new ``trace`` in the local queue. This method doesn't block the execution
@@ -72,48 +128,103 @@ module Datadog
         @closed = true
       end
     end
+  end
+
+  # Trace buffer that stores application traces and
+  # can be safely used concurrently with CRuby.
+  #
+  # Under extreme concurrency scenarios, this class can exceed
+  # its +max_size+ by up to 4%.
+  #
+  # Because singular +Array+ operations are thread-safe in CRuby,
+  # we can implement the trace buffer without an explicit lock,
+  # while making the compromise of allowing the buffer to go
+  # over its maximum limit under extreme circumstances.
+  #
+  # On the following scenario:
+  # * 4.5 million spans/second.
+  # * Pushed into a single CRubyTraceBuffer from 1000 threads.
+  # The buffer can exceed its maximum size by no more than 4%.
+  #
+  # This implementation allocates less memory and is faster
+  # than {Datadog::ThreadSafeBuffer}.
+  #
+  # @see spec/ddtrace/benchmark/buffer_benchmark_spec.rb Buffer benchmarks
+  # @see https://github.com/ruby-concurrency/concurrent-ruby/blob/c1114a0c6891d9634f019f1f9fe58dcae8658964/lib/concurrent-ruby/concurrent/array.rb#L23-L27
+  class CRubyTraceBuffer < MeasuredBuffer
+    def initialize(max_size)
+      super()
+
+      @max_size = max_size
+
+      @traces = []
+      @closed = false
+    end
 
-    # Aggregate metrics:
-    # They reflect buffer activity since last #pop.
-    # These may not be as accurate or as granular, but they
-    # don't use as much network traffic as live stats.
+    # Add a new ``trace`` in the local queue. This method doesn't block the execution
+    # even if the buffer is full. In that case, a random trace is discarded.
+    def push(trace)
+      return if @closed
+      len = @traces.length
+      if len < @max_size || @max_size <= 0
+        @traces << trace
+      else
+        # we should replace a random trace with the new one
+        replace_index = rand(len)
+        replaced_trace = @traces.delete_at(replace_index)
+        @traces << trace
+
+        # Check if we deleted the element right when the buffer
+        # was popped. In that case we didn't actually delete anything,
+        # we just inserted into a newly cleared buffer instead.
+        measure_drop(replaced_trace) if replaced_trace
+      end
 
-    def measure_accept(trace)
-      @buffer_spans += trace.length
-      @buffer_accepted += 1
-      @buffer_accepted_lengths += trace.length
-    rescue StandardError => e
-      Datadog.logger.debug("Failed to measure queue accept. Cause: #{e.message} Source: #{e.backtrace.first}")
+      measure_accept(trace)
     end
 
-    def measure_drop(trace)
-      @buffer_dropped += 1
-      @buffer_spans -= trace.length
-      @buffer_accepted_lengths -= trace.length
-    rescue StandardError => e
-      Datadog.logger.debug("Failed to measure queue drop. Cause: #{e.message} Source: #{e.backtrace.first}")
+    # Return the current number of stored traces.
+    def length
+      @traces.length
     end
 
-    def measure_pop(traces)
-      # Accepted
-      Datadog.health_metrics.queue_accepted(@buffer_accepted)
-      Datadog.health_metrics.queue_accepted_lengths(@buffer_accepted_lengths)
+    # Return if the buffer is empty.
+    def empty?
+      @traces.empty?
+    end
 
-      # Dropped
-      Datadog.health_metrics.queue_dropped(@buffer_dropped)
+    # Return all traces stored and reset buffer.
+    def pop
+      traces = @traces.pop(VERY_LARGE_INTEGER)
 
-      # Queue gauges
-      Datadog.health_metrics.queue_max_length(@max_size)
-      Datadog.health_metrics.queue_spans(@buffer_spans)
-      Datadog.health_metrics.queue_length(traces.length)
+      measure_pop(traces)
 
-      # Reset aggregated metrics
-      @buffer_accepted = 0
-      @buffer_accepted_lengths = 0
-      @buffer_dropped = 0
-      @buffer_spans = 0
-    rescue StandardError => e
-      Datadog.logger.debug("Failed to measure queue. Cause: #{e.message} Source: #{e.backtrace.first}")
+      traces
+    end
+
+    # Very large value, to ensure that we drain the whole buffer.
+    # 1<<62-1 happens to be the largest integer that can be stored inline in CRuby.
+    VERY_LARGE_INTEGER = 1 << 62 - 1
+
+    def close
+      @closed = true
     end
   end
+
+  # Choose default TraceBuffer implementation for current platform.
+  BUFFER_IMPLEMENTATION = if Datadog::Ext::Runtime::RUBY_ENGINE == 'ruby'
+                            CRubyTraceBuffer
+                          else
+                            ThreadSafeBuffer
+                          end
+  private_constant :BUFFER_IMPLEMENTATION
+
+  # Trace buffer that stores application traces. The buffer has a maximum size and when
+  # the buffer is full, a random trace is discarded. This class is thread-safe and is used
+  # automatically by the ``Tracer`` instance when a ``Span`` is finished.
+  #
+  # TODO We should restructure this module, so that classes are not declared at top-level ::Datadog.
+  # TODO Making such a change is potentially breaking for users manually configuring the tracer.
+  class TraceBuffer < BUFFER_IMPLEMENTATION
+  end
 end

data/lib/ddtrace/configuration.rb

@@ -56,7 +56,10 @@ module Datadog
     end
 
     def shutdown!
-      components.shutdown! if instance_variable_defined?(:@components) && @components
+      if instance_variable_defined?(:@components) && @components
+        components.shutdown!
+        @components = nil
+      end
     end
 
     protected

data/lib/ddtrace/configuration/options.rb

@@ -26,7 +26,9 @@ module Datadog
           builder = OptionDefinition::Builder.new(name, meta, &block)
           options[name] = builder.to_definition.tap do
             # Resolve and define helper functions
-            helpers = default_helpers(name).merge(builder.helpers)
+            helpers = default_helpers(name)
+            # Prevent unnecessary creation of an identical copy of helpers if there's nothing to merge
+            helpers = helpers.merge(builder.helpers) unless builder.helpers.empty?
             define_helpers(helpers)
           end
         end

data/lib/ddtrace/configuration/settings.rb

@@ -40,6 +40,11 @@ module Datadog
         option :debug do |o|
           o.default { env_to_bool(Datadog::Ext::Diagnostics::DD_TRACE_DEBUG, false) }
           o.lazy
+          o.on_set do |enabled|
+            # Enable rich debug print statements.
+            # We do not need to unnecessarily load 'pp' unless in debugging mode.
+            require 'pp' if enabled
+          end
         end
 
         settings :health_metrics do
@@ -91,7 +96,6 @@ module Datadog
 
       settings :logger do
         option :instance do |o|
-          o.setter { |value, old_value| value.is_a?(::Logger) ? value : old_value }
           o.on_set { |value| set_option(:level, value.level) unless value.nil? }
         end
 

data/lib/ddtrace/contrib/active_record/events/sql.rb

@@ -1,3 +1,4 @@
+require 'ddtrace/ext/integration'
 require 'ddtrace/ext/net'
 require 'ddtrace/contrib/analytics'
 require 'ddtrace/contrib/active_record/ext'
@@ -39,6 +40,9 @@ module Datadog
             span.resource = payload.fetch(:sql)
             span.span_type = Datadog::Ext::SQL::TYPE
 
+            # Tag as an external peer service
+            span.set_tag(Datadog::Ext::Integration::TAG_PEER_SERVICE, span.service)
+
             # Set analytics sample rate
             if Contrib::Analytics.enabled?(configuration[:analytics_enabled])
               Contrib::Analytics.set_sample_rate(span, configuration[:analytics_sample_rate])

data/lib/ddtrace/contrib/active_support/notifications/subscription.rb

@@ -76,8 +76,8 @@ module Datadog
 
             # Start a trace
             tracer.trace(@span_name, @options.dup).tap do |span|
-              # Assign start time if provided
-              span.start_time = start unless start.nil?
+              # Start span if time is provided
+              span.start(start) unless start.nil?
               payload[:datadog_span] = span
             end
           end