ddtrace 0.41.0 → 0.42.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0bbda933f09b8f29d20edc57d6825f6491158161b7e5164252deb006931e0e7d
-  data.tar.gz: b3aaa6718e835fbfe97db64faa62f82b2f3ff76aefadb53cb27fb9b236b842e3
+  metadata.gz: 356d9acad651e21531cb323e2894fd187a02e91bd1c68973bf8030b2aa1512a3
+  data.tar.gz: e9997b324030189490fc43e67dc3a9e4ba4ec9c334e9fe7a08c14ea66b25145b
 SHA512:
-  metadata.gz: 60d239de8c63091ede6a810127ad580629428b96181982cf543e77f45a89087dfdd238b98418fe72f0e27ebefae6d5f730208c5f6149541dfe1d1c6923579826
-  data.tar.gz: 1d3657bc28458d0b50d0c2c68d3397b20ef6b2d4c2a0adf2ec834f701fd2c7701bcdac7f6fbce913743482e5eb2492a6409baf7c9c37c3467677ead705fd752e
+  metadata.gz: 2e276f69c9c8c39c63c6b5c8abc7572d18d635f549f0a152acd2fbbd9a3cfe4cb151112fb2a928f0785c054a416087d97fd6650ce057b199598123f89df43e57
+  data.tar.gz: edc8ad3cd84b445c276f8d26f76ab9d404ce234154ea916fac94c65122628ce2b6cd6f74df450af19e231fe1d20a7f21ed8ee8707c70a7647f58fee51ee16912

data/Appraisals

@@ -539,6 +539,16 @@ elsif Gem::Version.new('2.3.0') <= Gem::Version.new(RUBY_VERSION) \
       gem 'lograge'
     end
 
+    appraise 'resque2-redis3' do
+      gem 'redis', '< 4.0'
+      gem 'resque', '>= 2.0'
+    end
+
+    appraise 'resque2-redis4' do
+      gem 'redis', '>= 4.0'
+      gem 'resque', '>= 2.0'
+    end
+
     appraise 'contrib' do
       gem 'actionpack'
       gem 'actionview'
@@ -569,7 +579,7 @@ elsif Gem::Version.new('2.3.0') <= Gem::Version.new(RUBY_VERSION) \
       gem 'rake', '>= 12.3'
       gem 'redis', '< 4.0'
       gem 'rest-client'
-      gem 'resque', '< 2.0'
+      gem 'resque'
       gem 'ruby-kafka', '>= 0.7.10'
       gem 'sequel'
       gem 'shoryuken'
@@ -628,6 +638,16 @@ elsif Gem::Version.new('2.4.0') <= Gem::Version.new(RUBY_VERSION) \
       gem 'lograge'
     end
 
+    appraise 'resque2-redis3' do
+      gem 'redis', '< 4.0'
+      gem 'resque', '>= 2.0'
+    end
+
+    appraise 'resque2-redis4' do
+      gem 'redis', '>= 4.0'
+      gem 'resque', '>= 2.0'
+    end
+
     appraise 'contrib' do
       gem 'actionpack'
       gem 'actionview'
@@ -658,7 +678,7 @@ elsif Gem::Version.new('2.4.0') <= Gem::Version.new(RUBY_VERSION) \
       gem 'rake', '>= 12.3'
       gem 'redis', '< 4.0'
       gem 'rest-client'
-      gem 'resque', '< 2.0'
+      gem 'resque'
       gem 'ruby-kafka', '>= 0.7.10'
       gem 'sequel'
       gem 'shoryuken'
@@ -765,6 +785,16 @@ elsif Gem::Version.new('2.5.0') <= Gem::Version.new(RUBY_VERSION) \
       gem 'lograge'
     end
 
+    appraise 'resque2-redis3' do
+      gem 'redis', '< 4.0'
+      gem 'resque', '>= 2.0'
+    end
+
+    appraise 'resque2-redis4' do
+      gem 'redis', '>= 4.0'
+      gem 'resque', '>= 2.0'
+    end
+
     appraise 'contrib' do
       gem 'actionpack'
       gem 'actionview'
@@ -796,7 +826,7 @@ elsif Gem::Version.new('2.5.0') <= Gem::Version.new(RUBY_VERSION) \
       gem 'rake', '>= 12.3'
       gem 'redis', '< 4.0'
       gem 'rest-client'
-      gem 'resque', '< 2.0'
+      gem 'resque'
       gem 'ruby-kafka', '>= 0.7.10'
       gem 'sequel'
       gem 'shoryuken'
@@ -893,6 +923,16 @@ elsif Gem::Version.new('2.6.0') <= Gem::Version.new(RUBY_VERSION) \
       gem 'lograge'
     end
 
+    appraise 'resque2-redis3' do
+      gem 'redis', '< 4.0'
+      gem 'resque', '>= 2.0'
+    end
+
+    appraise 'resque2-redis4' do
+      gem 'redis', '>= 4.0'
+      gem 'resque', '>= 2.0'
+    end
+
     appraise 'contrib' do
       gem 'actionpack'
       gem 'actionview'
@@ -922,7 +962,7 @@ elsif Gem::Version.new('2.6.0') <= Gem::Version.new(RUBY_VERSION) \
       gem 'rake', '>= 12.3'
       gem 'redis', '< 4.0'
       gem 'rest-client'
-      gem 'resque', '< 2.0'
+      gem 'resque'
       gem 'ruby-kafka', '>= 0.7.10'
       gem 'sequel'
       gem 'shoryuken'
@@ -1023,6 +1063,16 @@ elsif Gem::Version.new('2.7.0') <= Gem::Version.new(RUBY_VERSION)
       gem 'lograge'
     end
 
+    appraise 'resque2-redis3' do
+      gem 'redis', '< 4.0'
+      gem 'resque', '>= 2.0'
+    end
+
+    appraise 'resque2-redis4' do
+      gem 'redis', '>= 4.0'
+      gem 'resque', '>= 2.0'
+    end
+
     appraise 'contrib' do
       gem 'actionpack'
       gem 'actionview'
@@ -1051,7 +1101,7 @@ elsif Gem::Version.new('2.7.0') <= Gem::Version.new(RUBY_VERSION)
       gem 'rake', '>= 12.3'
       gem 'redis', '< 4.0'
       gem 'rest-client'
-      gem 'resque', '< 2.0'
+      gem 'resque'
       gem 'ruby-kafka', '>= 0.7.10'
       gem 'sequel'
       gem 'shoryuken'

data/CHANGELOG.md

@@ -2,6 +2,36 @@
 
 ## [Unreleased]
 
+## [0.42.0] - 2020-10-21
+
+Release notes: https://github.com/DataDog/dd-trace-rb/releases/tag/v0.42.0
+
+Git diff: https://github.com/DataDog/dd-trace-rb/compare/v0.41.0...v0.42.0
+
+### Added
+
+- Increase Resque support to include 2.0  (#1213) (@erict-square)
+
+- Improve gRPC Propagator to support metadata array values (#1203) (@mdehoog)
+
+- Add CPU benchmarks, diagnostics to tests (#1188, #1198)
+
+- Access active correlation by Thread (#1200)
+
+- Improve delayed_job instrumentation (#1187) (@norbertnytko)
+
+### Changed
+
+### Fixed
+
+- Improve Rails `log_injection` option to support more Lograge formats (#1210) (@Supy)
+
+- Fix Changelog (#1199) (@y-yagi)
+
+### Refactored
+
+- Refactor Trace buffer into multiple components (#1195)
+
 ## [0.41.0] - 2020-09-30
 
 Release notes: https://github.com/DataDog/dd-trace-rb/releases/tag/v0.41.0
@@ -17,7 +47,7 @@ Git diff: https://github.com/DataDog/dd-trace-rb/compare/v0.40.0...v0.41.0
 - Add peer.service tag to external services and skip tagging external services with language tag for runtime metrics (#934, #935, #1180)
   - This helps support the way runtime metrics are associated with spans in the UI.
 - Faster TraceBuffer for CRuby (#1172)
-- Reduce memory usage during gem statup (#1090)
+- Reduce memory usage during gem startup (#1090)
 - Reduce memory usage of the HTTP transport (#1165)
 
 ### Fixed

data/CONTRIBUTING.md

@@ -56,7 +56,7 @@ We welcome code contributions to the library, which you can [submit as a pull re
 2. **Make any changes** for your patch.
 3. **Write tests** that demonstrate how the feature works or how the bug is fixed.
 4. **Update any documentation** such as `docs/GettingStarted.md`, especially for new features.
-5. **Submit the pull request** from your fork back to https://github.com/DataDog/dd-trace-rb. Bugs should merge back to `master`, and new features should merge back to the latest `dev` branch (e.g. `0.24-dev`).
+5. **Submit the pull request** from your fork back to the latest revision of the `master` branch on https://github.com/DataDog/dd-trace-rb.
 
 The pull request will be run through our CI pipeline, and a project member will review the changes with you. At a minimum, to be accepted and merged, pull requests must:
 

data/Rakefile

@@ -464,6 +464,10 @@ task :ci do
       declare 'bundle exec appraisal rails4-postgres rake spec:rails'
       declare 'bundle exec appraisal rails5-mysql2 rake spec:rails'
       declare 'bundle exec appraisal rails5-postgres rake spec:rails'
+
+      # explicitly test resque-2x compatability
+      declare 'bundle exec appraisal resque2-redis3 rake spec:resque'
+      declare 'bundle exec appraisal resque2-redis4 rake spec:resque'
     end
   elsif Gem::Version.new('2.4.0') <= Gem::Version.new(RUBY_VERSION) \
         && Gem::Version.new(RUBY_VERSION) < Gem::Version.new('2.5.0')
@@ -525,6 +529,10 @@ task :ci do
       # Rails specs
       declare 'bundle exec appraisal rails5-mysql2 rake spec:rails'
       declare 'bundle exec appraisal rails5-postgres rake spec:rails'
+
+      # explicitly test resque-2x compatability
+      declare 'bundle exec appraisal resque2-redis3 rake spec:resque'
+      declare 'bundle exec appraisal resque2-redis4 rake spec:resque'
     end
   elsif Gem::Version.new('2.5.0') <= Gem::Version.new(RUBY_VERSION) \
         && Gem::Version.new(RUBY_VERSION) < Gem::Version.new('2.6.0')
@@ -594,6 +602,10 @@ task :ci do
     # declare 'bundle exec appraisal rails6-mysql2 rake spec:action_cable' # TODO: Hangs CI jobs... fix and re-enable.
     declare 'bundle exec appraisal rails6-mysql2 rake spec:rails'
     declare 'bundle exec appraisal rails6-postgres rake spec:rails'
+
+    # explicitly test resque-2x compatability
+    declare 'bundle exec appraisal resque2-redis3 rake spec:resque'
+    declare 'bundle exec appraisal resque2-redis4 rake spec:resque'
   elsif Gem::Version.new('2.6.0') <= Gem::Version.new(RUBY_VERSION) \
       && Gem::Version.new(RUBY_VERSION) < Gem::Version.new('2.7.0')
     # Main library
@@ -664,6 +676,10 @@ task :ci do
       # declare 'bundle exec appraisal rails6-mysql2 rake spec:action_cable' # TODO: Hangs CI jobs... fix and re-enable.
       declare 'bundle exec appraisal rails6-mysql2 rake spec:rails'
       declare 'bundle exec appraisal rails6-postgres rake spec:rails'
+
+      # explicitly test resque-2x compatability
+      declare 'bundle exec appraisal resque2-redis3 rake spec:resque'
+      declare 'bundle exec appraisal resque2-redis4 rake spec:resque'
     end
   elsif Gem::Version.new('2.7.0') <= Gem::Version.new(RUBY_VERSION)
     # Main library
@@ -732,6 +748,10 @@ task :ci do
       declare 'bundle exec appraisal rails5-postgres rake spec:rails'
       declare 'bundle exec appraisal rails6-mysql2 rake spec:rails'
       declare 'bundle exec appraisal rails6-postgres rake spec:rails'
+
+      # explicitly test resque-2x compatability
+      declare 'bundle exec appraisal resque2-redis3 rake spec:resque'
+      declare 'bundle exec appraisal resque2-redis4 rake spec:resque'
     end
   end
 end

data/ddtrace.gemspec

@@ -46,6 +46,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'rubocop', '= 0.49.1' if RUBY_VERSION >= '2.1.0'
   spec.add_development_dependency 'rspec', '~> 3.0'
   spec.add_development_dependency 'rspec-collection_matchers', '~> 1.1'
+  spec.add_development_dependency 'ruby-prof', '~> 1.4' if RUBY_PLATFORM != 'java' && RUBY_VERSION >= '2.4.0'
   spec.add_development_dependency 'minitest', '= 5.10.1'
   spec.add_development_dependency 'minitest-around', '0.5.0'
   spec.add_development_dependency 'minitest-stub_any_instance', '1.0.2'

data/docs/DevelopmentGuide.md

@@ -124,6 +124,16 @@ The trace library uses Rubocop to enforce [code style](https://github.com/bbatso
 $ bundle exec rake rubocop
 ```
 
+### Running benchmarks
+
+If your changes can have a measurable performance impact, we recommend running our benchmark suite:
+
+```
+$ bundle exec rake spec:benchmark
+```
+
+Results are printed to STDOUT as well as written to the `./tmp/benchmark/` directory.
+
 ## Appendix
 
 ### Writing new integrations

data/docs/GettingStarted.md

@@ -359,7 +359,7 @@ For a list of available integrations, and their configuration options, please re
 | Rails                    | `rails`                    | `>= 3.0`                 | `>= 3.0`                  | *[Link](#rails)*                    | *[Link](https://github.com/rails/rails)*                                       |
 | Rake                     | `rake`                     | `>= 12.0`                | `>= 12.0`                 | *[Link](#rake)*                     | *[Link](https://github.com/ruby/rake)*                                         |
 | Redis                    | `redis`                    | `>= 3.2`                 | `>= 3.2`                  | *[Link](#redis)*                    | *[Link](https://github.com/redis/redis-rb)*                                    |
-| Resque                   | `resque`                   | `>= 1.0, < 2.0`          | `>= 1.0, < 2.0`           | *[Link](#resque)*                   | *[Link](https://github.com/resque/resque)*                                     |
+| Resque                   | `resque`                   | `>= 1.0`                 | `>= 1.0`                  | *[Link](#resque)*                   | *[Link](https://github.com/resque/resque)*                                     |
 | Rest Client              | `rest-client`              | `>= 1.8`                 | `>= 1.8`                  | *[Link](#rest-client)*              | *[Link](https://github.com/rest-client/rest-client)*                           |
 | Sequel                   | `sequel`                   | `>= 3.41`                | `>= 3.41`                 | *[Link](#sequel)*                   | *[Link](https://github.com/jeremyevans/sequel)*                                |
 | Shoryuken                | `shoryuken`                | `>= 3.2`                 | `>= 3.2`                  | *[Link](#shoryuken)*                | *[Link](https://github.com/phstc/shoryuken)*                                   |
@@ -617,7 +617,7 @@ Where `options` is an optional `Hash` that accepts the following parameters:
 
 ### DelayedJob
 
-The DelayedJob integration uses lifecycle hooks to trace the job executions.
+The DelayedJob integration uses lifecycle hooks to trace the job executions and enqueues.
 
 You can enable it through `Datadog.configure`:
 
@@ -635,6 +635,7 @@ Where `options` is an optional `Hash` that accepts the following parameters:
 | --- | ----------- | ------- |
 | `analytics_enabled` | Enable analytics for spans produced by this integration. `true` for on, `nil` to defer to global setting, `false` for off. | `false` |
 | `service_name` | Service name used for `DelayedJob` instrumentation | `'delayed_job'` |
+| `client_service_name` | Service name used for client-side `DelayedJob` instrumentation | `'delayed_job-client'` |
 
 ### Elasticsearch
 
@@ -2045,6 +2046,8 @@ Datadog.configure do |c|
 end
 ```
 
+_Note:_ For `lograge` users who have also defined `lograge.custom_options` in an `initializers/lograge.rb` configuration file, due to the order that Rails loads initializers (alphabetical), automatic trace correlation may not take effect, since `initializers/datadog.rb` would be overwritten by the `initializers/lograge.rb` initializer. To support automatic trace correlation with _existing_ `lograge.custom_options`, use the [Manual (Lograge)](#manual-lograge) configuration below.
+
 ##### Manual (Lograge)
 
 After [setting up Lograge in a Rails application](https://docs.datadoghq.com/logs/log_collection/ruby/), manually modify the `custom_options` block in your environment configuration file (e.g. `config/environments/production.rb`) to add the trace IDs. 

data/lib/ddtrace/buffer.rb

@@ -5,132 +5,175 @@ require 'ddtrace/runtime/object_space'
 # Trace buffer that accumulates traces for a consumer.
 # Consumption can happen from a different thread.
 module Datadog
-  # Aggregate metrics:
-  # They reflect buffer activity since last #pop.
-  # These may not be as accurate or as granular, but they
-  # don't use as much network traffic as live stats.
-  class MeasuredBuffer
-    def initialize
-      @buffer_accepted = 0
-      @buffer_accepted_lengths = 0
-      @buffer_dropped = 0
-      @buffer_spans = 0
+  # Buffer that stores objects. The buffer has a maximum size and when
+  # the buffer is full, a random object is discarded.
+  class Buffer
+    def initialize(max_size)
+      @max_size = max_size
+      @items = []
+      @closed = false
     end
 
-    def measure_accept(trace)
-      @buffer_accepted += 1
-      @buffer_accepted_lengths += trace.length
+    # Add a new ``item`` in the local queue. This method doesn't block the execution
+    # even if the buffer is full. In that case, a random item is discarded.
+    def push(item)
+      return if closed?
+      full? ? replace!(item) : add!(item)
+      item
+    end
 
-      @buffer_spans += trace.length
-    rescue StandardError => e
-      Datadog.logger.debug("Failed to measure queue accept. Cause: #{e.message} Source: #{e.backtrace.first}")
+    # A bulk push alternative to +#push+. Use this method if
+    # pushing more than one item for efficiency.
+    def concat(items)
+      return if closed?
+
+      # Segment items into underflow and overflow
+      underflow, overflow = overflow_segments(items)
+
+      # Concatenate items do not exceed capacity.
+      add_all!(underflow) unless underflow.nil?
+
+      # Iteratively replace items, to ensure pseudo-random replacement.
+      overflow.each { |item| replace!(item) } unless overflow.nil?
     end
 
-    def measure_drop(trace)
-      @buffer_dropped += 1
+    # Stored items are returned and the local buffer is reset.
+    def pop
+      drain!
+    end
 
-      @buffer_spans -= trace.length
-    rescue StandardError => e
-      Datadog.logger.debug("Failed to measure queue drop. Cause: #{e.message} Source: #{e.backtrace.first}")
+    # Return the current number of stored traces.
+    def length
+      @items.length
     end
 
-    def measure_pop(traces)
-      # Accepted, cumulative totals
-      Datadog.health_metrics.queue_accepted(@buffer_accepted)
-      Datadog.health_metrics.queue_accepted_lengths(@buffer_accepted_lengths)
+    # Return if the buffer is empty.
+    def empty?
+      @items.empty?
+    end
 
-      # Dropped, cumulative totals
-      Datadog.health_metrics.queue_dropped(@buffer_dropped)
-      # TODO: are we missing a +queue_dropped_lengths+ metric?
+    # Closes this buffer, preventing further pushing.
+    # Draining is still allowed.
+    def close
+      @closed = true
+    end
 
-      # Queue gauges, current values
-      Datadog.health_metrics.queue_max_length(@max_size)
-      Datadog.health_metrics.queue_spans(@buffer_spans)
-      Datadog.health_metrics.queue_length(traces.length)
+    def closed?
+      @closed
+    end
 
-      # Reset aggregated metrics
-      @buffer_accepted = 0
-      @buffer_accepted_lengths = 0
-      @buffer_dropped = 0
-      @buffer_spans = 0
-    rescue StandardError => e
-      Datadog.logger.debug("Failed to measure queue. Cause: #{e.message} Source: #{e.backtrace.first}")
+    protected
+
+    # Segment items into two distinct segments: underflow and overflow.
+    # Underflow are items that will fit into buffer.
+    # Overflow are items that will exceed capacity, after underflow is added.
+    # Returns each array, and nil if there is no underflow/overflow.
+    def overflow_segments(items)
+      underflow = nil
+      overflow = nil
+
+      overflow_size = @max_size > 0 ? (@items.length + items.length) - @max_size : 0
+
+      if overflow_size > 0
+        # Items will overflow
+        if overflow_size < items.length
+          # Partial overflow
+          underflow_end_index = items.length - overflow_size - 1
+          underflow = items[0..underflow_end_index]
+          overflow = items[(underflow_end_index + 1)..-1]
+        else
+          # Total overflow
+          overflow = items
+        end
+      else
+        # Items do not exceed capacity.
+        underflow = items
+      end
+
+      [underflow, overflow]
+    end
+
+    def full?
+      @max_size > 0 && @items.length >= @max_size
+    end
+
+    def add_all!(items)
+      @items.concat(items)
+    end
+
+    def add!(item)
+      @items << item
+    end
+
+    def replace!(item)
+      # Choose random item to be replaced
+      replace_index = rand(@items.length)
+
+      # Replace random item
+      discarded_item = @items[replace_index]
+      @items[replace_index] = item
+
+      # Return discarded item
+      discarded_item
+    end
+
+    def drain!
+      items = @items
+      @items = []
+      items
     end
   end
 
-  # Trace buffer that stores application traces and
+  # Buffer that stores objects, has a maximum size, and
   # can be safely used concurrently on any environment.
   #
   # This implementation uses a {Mutex} around public methods, incurring
-  # overhead in order to ensure full thread-safety.
+  # overhead in order to ensure thread-safety.
   #
   # This is implementation is recommended for non-CRuby environments.
-  # If using CRuby, {Datadog::CRubyTraceBuffer} is a faster implementation with minimal compromise.
-  class ThreadSafeBuffer < MeasuredBuffer
+  # If using CRuby, {Datadog::CRubyBuffer} is a faster implementation with minimal compromise.
+  class ThreadSafeBuffer < Buffer
     def initialize(max_size)
-      super()
-
-      @max_size = max_size
+      super
 
-      @mutex = Mutex.new()
-      @traces = []
-      @closed = false
+      @mutex = Mutex.new
     end
 
-    # Add a new ``trace`` in the local queue. This method doesn't block the execution
-    # even if the buffer is full. In that case, a random trace is discarded.
-    def push(trace)
-      @mutex.synchronize do
-        return if @closed
-        len = @traces.length
-        if len < @max_size || @max_size <= 0
-          @traces << trace
-        else
-          # we should replace a random trace with the new one
-          replace_index = rand(len)
-          replaced_trace = @traces[replace_index]
-          @traces[replace_index] = trace
-          measure_drop(replaced_trace)
-        end
+    # Add a new ``item`` in the local queue. This method doesn't block the execution
+    # even if the buffer is full. In that case, a random item is discarded.
+    def push(item)
+      synchronize { super }
+    end
 
-        measure_accept(trace)
-      end
+    def concat(items)
+      synchronize { super }
     end
 
     # Return the current number of stored traces.
     def length
-      @mutex.synchronize do
-        return @traces.length
-      end
+      synchronize { super }
     end
 
     # Return if the buffer is empty.
     def empty?
-      @mutex.synchronize do
-        return @traces.empty?
-      end
+      synchronize { super }
     end
 
     # Stored traces are returned and the local buffer is reset.
     def pop
-      @mutex.synchronize do
-        traces = @traces
-        @traces = []
-
-        measure_pop(traces)
-
-        return traces
-      end
+      synchronize { super }
     end
 
     def close
-      @mutex.synchronize do
-        @closed = true
-      end
+      synchronize { super }
+    end
+
+    def synchronize
+      @mutex.synchronize { yield }
     end
   end
 
-  # Trace buffer that stores application traces and
+  # Buffer that stores objects, has a maximum size, and
   # can be safely used concurrently with CRuby.
   #
   # Under extreme concurrency scenarios, this class can exceed
@@ -151,80 +194,133 @@ module Datadog
   #
   # @see spec/ddtrace/benchmark/buffer_benchmark_spec.rb Buffer benchmarks
   # @see https://github.com/ruby-concurrency/concurrent-ruby/blob/c1114a0c6891d9634f019f1f9fe58dcae8658964/lib/concurrent-ruby/concurrent/array.rb#L23-L27
-  class CRubyTraceBuffer < MeasuredBuffer
-    def initialize(max_size)
-      super()
+  class CRubyBuffer < Buffer
+    # Add a new ``trace`` in the local queue. This method doesn't block the execution
+    # even if the buffer is full. In that case, a random trace is discarded.
+    def replace!(item)
+      # we should replace a random trace with the new one
+      replace_index = rand(@items.size)
+      replaced_trace = @items.delete_at(replace_index)
+      @items << item
+
+      # We might have deleted an element right when the buffer
+      # was drained, thus +replaced_trace+ will be +nil+.
+      # In that case, nothing was replaced, and this method
+      # performed a simple insertion into the buffer.
+      replaced_trace
+    end
+  end
 
-      @max_size = max_size
+  # Health metrics for trace buffers.
+  module MeasuredBuffer
+    def initialize(*_)
+      super
 
-      @traces = []
-      @closed = false
+      @buffer_accepted = 0
+      @buffer_accepted_lengths = 0
+      @buffer_dropped = 0
+      @buffer_spans = 0
     end
 
-    # Add a new ``trace`` in the local queue. This method doesn't block the execution
-    # even if the buffer is full. In that case, a random trace is discarded.
-    def push(trace)
-      return if @closed
-      len = @traces.length
-      if len < @max_size || @max_size <= 0
-        @traces << trace
-      else
-        # we should replace a random trace with the new one
-        replace_index = rand(len)
-        replaced_trace = @traces.delete_at(replace_index)
-        @traces << trace
-
-        # Check if we deleted the element right when the buffer
-        # was popped. In that case we didn't actually delete anything,
-        # we just inserted into a newly cleared buffer instead.
-        measure_drop(replaced_trace) if replaced_trace
-      end
+    def add!(trace)
+      super
 
+      # Emit health metrics
       measure_accept(trace)
     end
 
-    # Return the current number of stored traces.
-    def length
-      @traces.length
-    end
+    def add_all!(traces)
+      super
 
-    # Return if the buffer is empty.
-    def empty?
-      @traces.empty?
+      # Emit health metrics
+      traces.each { |trace| measure_accept(trace) }
     end
 
-    # Return all traces stored and reset buffer.
-    def pop
-      traces = @traces.pop(VERY_LARGE_INTEGER)
+    def replace!(trace)
+      discarded_trace = super
 
-      measure_pop(traces)
+      # Emit health metrics
+      measure_accept(trace)
+      measure_drop(discarded_trace) if discarded_trace
+
+      discarded_trace
+    end
 
+    # Stored traces are returned and the local buffer is reset.
+    def drain!
+      traces = super
+      measure_pop(traces)
       traces
     end
 
-    # Very large value, to ensure that we drain the whole buffer.
-    # 1<<62-1 happens to be the largest integer that can be stored inline in CRuby.
-    VERY_LARGE_INTEGER = 1 << 62 - 1
+    def measure_accept(trace)
+      @buffer_accepted += 1
+      @buffer_accepted_lengths += trace.length
 
-    def close
-      @closed = true
+      @buffer_spans += trace.length
+    rescue StandardError => e
+      Datadog.logger.debug("Failed to measure queue accept. Cause: #{e.message} Source: #{e.backtrace.first}")
     end
+
+    def measure_drop(trace)
+      @buffer_dropped += 1
+
+      @buffer_spans -= trace.length
+    rescue StandardError => e
+      Datadog.logger.debug("Failed to measure queue drop. Cause: #{e.message} Source: #{e.backtrace.first}")
+    end
+
+    def measure_pop(traces)
+      # Accepted, cumulative totals
+      Datadog.health_metrics.queue_accepted(@buffer_accepted)
+      Datadog.health_metrics.queue_accepted_lengths(@buffer_accepted_lengths)
+
+      # Dropped, cumulative totals
+      Datadog.health_metrics.queue_dropped(@buffer_dropped)
+      # TODO: are we missing a +queue_dropped_lengths+ metric?
+
+      # Queue gauges, current values
+      Datadog.health_metrics.queue_max_length(@max_size)
+      Datadog.health_metrics.queue_spans(@buffer_spans)
+      Datadog.health_metrics.queue_length(traces.length)
+
+      # Reset aggregated metrics
+      @buffer_accepted = 0
+      @buffer_accepted_lengths = 0
+      @buffer_dropped = 0
+      @buffer_spans = 0
+    rescue StandardError => e
+      Datadog.logger.debug("Failed to measure queue. Cause: #{e.message} Source: #{e.backtrace.first}")
+    end
+  end
+
+  # Trace buffer that stores application traces, has a maximum size, and
+  # can be safely used concurrently on any environment.
+  #
+  # @see {Datadog::ThreadSafeBuffer}
+  class ThreadSafeTraceBuffer < ThreadSafeBuffer
+    prepend MeasuredBuffer
   end
 
-  # Choose default TraceBuffer implementation for current platform.
-  BUFFER_IMPLEMENTATION = if Datadog::Ext::Runtime::RUBY_ENGINE == 'ruby'
-                            CRubyTraceBuffer
-                          else
-                            ThreadSafeBuffer
-                          end
-  private_constant :BUFFER_IMPLEMENTATION
+  # Trace buffer that stores application traces, has a maximum size, and
+  # can be safely used concurrently with CRuby.
+  #
+  # @see {Datadog::CRubyBuffer}
+  class CRubyTraceBuffer < CRubyBuffer
+    prepend MeasuredBuffer
+  end
 
   # Trace buffer that stores application traces. The buffer has a maximum size and when
   # the buffer is full, a random trace is discarded. This class is thread-safe and is used
   # automatically by the ``Tracer`` instance when a ``Span`` is finished.
   #
+  # We choose the default TraceBuffer implementation for current platform dynamically here.
+  #
   # TODO We should restructure this module, so that classes are not declared at top-level ::Datadog.
   # TODO Making such a change is potentially breaking for users manually configuring the tracer.
-  class TraceBuffer < BUFFER_IMPLEMENTATION
-  end
+  TraceBuffer = if Datadog::Ext::Runtime::RUBY_ENGINE == 'ruby' # rubocop:disable Style/ConstantName
+                  CRubyTraceBuffer
+                else
+                  ThreadSafeTraceBuffer
+                end
 end

data/lib/ddtrace/context_provider.rb

@@ -13,9 +13,9 @@ module Datadog
       @context.local = ctx
     end
 
-    # Return the current context.
-    def context
-      @context.local
+    # Return the local context.
+    def context(key = nil)
+      key.nil? ? @context.local : @context.local(key)
     end
   end
 
@@ -43,8 +43,9 @@ module Datadog
     end
 
     # Return the thread-local context.
-    def local
-      Thread.current[@key] ||= Datadog::Context.new
+    def local(thread = Thread.current)
+      raise ArgumentError, '\'thread\' must be a Thread.' unless thread.is_a?(Thread)
+      thread[@key] ||= Datadog::Context.new
     end
   end
 end

data/lib/ddtrace/contrib/delayed_job/configuration/settings.rb

@@ -23,6 +23,7 @@ module Datadog
           end
 
           option :service_name, default: Ext::SERVICE_NAME
+          option :client_service_name, default: Ext::CLIENT_SERVICE_NAME
         end
       end
     end

data/lib/ddtrace/contrib/delayed_job/ext.rb

@@ -10,7 +10,9 @@ module Datadog
         ENV_ANALYTICS_SAMPLE_RATE = 'DD_TRACE_DELAYED_JOB_ANALYTICS_SAMPLE_RATE'.freeze
         ENV_ANALYTICS_SAMPLE_RATE_OLD = 'DD_DELAYED_JOB_ANALYTICS_SAMPLE_RATE'.freeze
         SERVICE_NAME = 'delayed_job'.freeze
+        CLIENT_SERVICE_NAME = 'delayed_job-client'.freeze
         SPAN_JOB = 'delayed_job'.freeze
+        SPAN_ENQUEUE = 'delayed_job.enqueue'.freeze
         TAG_ATTEMPTS = 'delayed_job.attempts'.freeze
         TAG_ID = 'delayed_job.id'.freeze
         TAG_PRIORITY = 'delayed_job.priority'.freeze

data/lib/ddtrace/contrib/delayed_job/plugin.rb

@@ -7,22 +7,11 @@ module Datadog
     module DelayedJob
       # DelayedJob plugin that instruments invoke_job hook
       class Plugin < Delayed::Plugin
-        def self.instrument(job, &block)
+        def self.instrument_invoke(job, &block)
           return block.call(job) unless tracer && tracer.enabled
 
-          # When DelayedJob is used through ActiveJob, we need to parse the payload differentely
-          # to get the actual job name
-          job_name = if job.payload_object.respond_to?(:job_data)
-                       job.payload_object.job_data['job_class']
-                     else
-                       job.name
-                     end
-
-          tracer.trace(Ext::SPAN_JOB, service: configuration[:service_name], resource: job_name) do |span|
-            # Set analytics sample rate
-            if Contrib::Analytics.enabled?(configuration[:analytics_enabled])
-              Contrib::Analytics.set_sample_rate(span, configuration[:analytics_sample_rate])
-            end
+          tracer.trace(Ext::SPAN_JOB, service: configuration[:service_name], resource: job_name(job)) do |span|
+            set_sample_rate(span)
 
             # Measure service stats
             Contrib::Analytics.set_measured(span)
@@ -37,6 +26,23 @@ module Datadog
           end
         end
 
+        def self.instrument_enqueue(job, &block)
+          return block.call(job) unless tracer && tracer.enabled
+
+          tracer.trace(Ext::SPAN_ENQUEUE, service: configuration[:client_service_name], resource: job_name(job)) do |span|
+            set_sample_rate(span)
+
+            # Measure service stats
+            Contrib::Analytics.set_measured(span)
+
+            span.set_tag(Ext::TAG_QUEUE, job.queue) if job.queue
+            span.set_tag(Ext::TAG_PRIORITY, job.priority)
+            span.span_type = Datadog::Ext::AppTypes::WORKER
+
+            yield job
+          end
+        end
+
         def self.flush(worker, &block)
           yield worker
 
@@ -51,8 +57,24 @@ module Datadog
           configuration[:tracer]
         end
 
+        def self.job_name(job)
+          # When DelayedJob is used through ActiveJob, we need to parse the payload differentely
+          # to get the actual job name
+          return job.payload_object.job_data['job_class'] if job.payload_object.respond_to?(:job_data)
+
+          job.name
+        end
+
+        def self.set_sample_rate(span)
+          # Set analytics sample rate
+          if Contrib::Analytics.enabled?(configuration[:analytics_enabled])
+            Contrib::Analytics.set_sample_rate(span, configuration[:analytics_sample_rate])
+          end
+        end
+
         callbacks do |lifecycle|
-          lifecycle.around(:invoke_job, &method(:instrument))
+          lifecycle.around(:invoke_job, &method(:instrument_invoke))
+          lifecycle.around(:enqueue, &method(:instrument_enqueue))
           lifecycle.around(:execute, &method(:flush))
         end
       end

data/lib/ddtrace/contrib/grpc/datadog_interceptor/client.rb

@@ -8,7 +8,7 @@ module Datadog
     module GRPC
       module DatadogInterceptor
         # The DatadogInterceptor::Client implements the tracing strategy
-        # for gRPC client-side endpoitns. This middleware compoent will
+        # for gRPC client-side endpoints. This middleware component will
         # inject trace context information into gRPC metadata prior to
         # sending the request to the server.
         class Client < Base

data/lib/ddtrace/contrib/rails/patcher.rb

@@ -54,14 +54,25 @@ module Datadog
 
         def add_logger(app)
           # check if lograge key exists
-          if app.config.respond_to?(:lograge) && app.config.lograge.enabled
+          # Note: Rails executes initializers sequentially based on alphabetical order,
+          # and lograge config could occur after dd config.
+          # Checking for `app.config.lograge.enabled` may yield a false negative.
+          # Instead we should naively add custom options if `config.lograge` exists from the lograge Railtie,
+          # since the custom options get ignored without lograge explicitly being enabled.
+          # See: https://github.com/roidrage/lograge/blob/1729eab7956bb95c5992e4adab251e4f93ff9280/lib/lograge/railtie.rb#L7-L12
+          if app.config.respond_to?(:lograge)
             Datadog::Contrib::Rails::LogInjection.add_lograge_logger(app)
-          # if lograge isn't set, check if tagged logged is enabed.
+          end
+
+          # if lograge isn't set, check if tagged logged is enabled.
           # if so, add proc that injects trace identifiers for tagged logging.
-          elsif (logger = app.config.logger) && logger.is_a?(::ActiveSupport::TaggedLogging)
+          if (logger = app.config.logger) &&
+             defined?(::ActiveSupport::TaggedLogging) &&
+             logger.is_a?(::ActiveSupport::TaggedLogging)
+
             Datadog::Contrib::Rails::LogInjection.add_as_tagged_logging_logger(app)
           else
-            Datadog.logger.warn("Unabe to enable Datadog Trace context, Logger #{logger} is not supported")
+            Datadog.logger.warn("Unable to enable Datadog Trace context, Logger #{logger} is not supported")
           end
         end
 

data/lib/ddtrace/contrib/resque/integration.rb

@@ -11,7 +11,7 @@ module Datadog
 
         MINIMUM_VERSION = Gem::Version.new('1.0')
         # Maximum is first version it's NOT compatible with (not inclusive)
-        MAXIMUM_VERSION = Gem::Version.new('2.0')
+        MAXIMUM_VERSION = Gem::Version.new('3.0')
 
         register_as :resque, auto_patch: true
 

data/lib/ddtrace/propagation/grpc_propagator.rb

@@ -38,24 +38,36 @@ module Datadog
       end
 
       def trace_id
-        value = @metadata[GRPC_METADATA_TRACE_ID].to_i
+        value = metadata_for_key(GRPC_METADATA_TRACE_ID).to_i
         value if (1..Span::EXTERNAL_MAX_ID).cover? value
       end
 
       def parent_id
-        value = @metadata[GRPC_METADATA_PARENT_ID].to_i
+        value = metadata_for_key(GRPC_METADATA_PARENT_ID).to_i
         value if (1..Span::EXTERNAL_MAX_ID).cover? value
       end
 
       def sampling_priority
-        value = @metadata[GRPC_METADATA_SAMPLING_PRIORITY]
+        value = metadata_for_key(GRPC_METADATA_SAMPLING_PRIORITY)
         value && value.to_i
       end
 
       def origin
-        value = @metadata[GRPC_METADATA_ORIGIN]
+        value = metadata_for_key(GRPC_METADATA_ORIGIN)
         value if value != ''
       end
+
+      private
+
+      def metadata_for_key(key)
+        # metadata values can be arrays (multiple headers with the same key)
+        value = @metadata[key]
+        if value.is_a?(Array)
+          value.first
+        else
+          value
+        end
+      end
     end
   end
 end

data/lib/ddtrace/tracer.rb

@@ -59,8 +59,8 @@ module Datadog
     #
     # This method makes use of a \ContextProvider that is automatically set during the tracer
     # initialization, or while using a library instrumentation.
-    def call_context
-      @provider.context
+    def call_context(key = nil)
+      @provider.context(key)
     end
 
     # Initialize a new \Tracer used to create, sample and submit spans that measure the
@@ -324,18 +324,18 @@ module Datadog
     end
 
     # Return the current active span or +nil+.
-    def active_span
-      call_context.current_span
+    def active_span(key = nil)
+      call_context(key).current_span
     end
 
     # Return the current active root span or +nil+.
-    def active_root_span
-      call_context.current_root_span
+    def active_root_span(key = nil)
+      call_context(key).current_root_span
     end
 
     # Return a CorrelationIdentifier for active span
-    def active_correlation
-      Datadog::Correlation.identifier_from_context(call_context)
+    def active_correlation(key = nil)
+      Datadog::Correlation.identifier_from_context(call_context(key))
     end
 
     # Send the trace to the writer to enqueue the spans list in the agent

data/lib/ddtrace/version.rb

@@ -1,7 +1,7 @@
 module Datadog
   module VERSION
     MAJOR = 0
-    MINOR = 41
+    MINOR = 42
     PATCH = 0
     PRE = nil
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ddtrace
 version: !ruby/object:Gem::Version
-  version: 0.41.0
+  version: 0.42.0
 platform: ruby
 authors:
 - Datadog, Inc.
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-09-30 00:00:00.000000000 Z
+date: 2020-10-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
@@ -122,6 +122,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.1'
+- !ruby/object:Gem::Dependency
+  name: ruby-prof
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.4'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.4'
 - !ruby/object:Gem::Dependency
   name: minitest
   requirement: !ruby/object:Gem::Requirement