ddtrace 0.53.0 → 1.9.0

data/lib/datadog/ci/extensions.rb

@@ -1,17 +1,18 @@
 # typed: true
-require 'ddtrace/configuration/settings'
-require 'ddtrace/configuration/components'
 
-require 'datadog/ci/configuration/settings'
-require 'datadog/ci/configuration/components'
+require_relative '../core/configuration/settings'
+require_relative '../core/configuration/components'
+
+require_relative 'configuration/settings'
+require_relative 'configuration/components'
 
 module Datadog
   module CI
     # Extends Datadog tracing with CI features
     module Extensions
       def self.activate!
-        Datadog::Configuration::Settings.extend(CI::Configuration::Settings)
-        Datadog::Configuration::Components.prepend(CI::Configuration::Components)
+        Core::Configuration::Settings.extend(CI::Configuration::Settings)
+        Core::Configuration::Components.prepend(CI::Configuration::Components)
       end
     end
   end

data/lib/datadog/ci/flush.rb

@@ -0,0 +1,38 @@
+# typed: true
+
+require_relative '../tracing/metadata/ext'
+require_relative '../tracing/flush'
+
+module Datadog
+  module CI
+    module Flush
+      # Common behavior for CI flushing
+      module Tagging
+        # Decorate a trace with CI tags
+        def get_trace(trace_op)
+          trace = trace_op.flush!
+
+          # Origin tag is required on every span
+          trace.spans.each do |span|
+            span.set_tag(
+              Tracing::Metadata::Ext::Distributed::TAG_ORIGIN,
+              trace.origin
+            )
+          end
+
+          trace
+        end
+      end
+
+      # Consumes only completed traces (where all spans have finished)
+      class Finished < Tracing::Flush::Finished
+        prepend Tagging
+      end
+
+      # Performs partial trace flushing to avoid large traces residing in memory for too long
+      class Partial < Tracing::Flush::Partial
+        prepend Tagging
+      end
+    end
+  end
+end

data/lib/datadog/ci/test.rb

@@ -1,10 +1,11 @@
-# typed: true
 # frozen_string_literal: true
 
-require 'ddtrace/contrib/analytics'
+# typed: false
 
-require 'datadog/ci/ext/app_types'
-require 'datadog/ci/ext/test'
+require_relative '../tracing/contrib/analytics'
+
+require_relative 'ext/app_types'
+require_relative 'ext/test'
 
 require 'rbconfig'
 
@@ -13,35 +14,40 @@ module Datadog
     # Common behavior for CI tests
     module Test
       # Creates a new span for a CI test
-      def self.trace(tracer, span_name, options = {})
+      def self.trace(span_name, options = {})
         span_options = {
-          span_type: Ext::AppTypes::TEST
+          span_type: Ext::AppTypes::TYPE_TEST
         }.merge(options[:span_options] || {})
 
         if block_given?
-          tracer.trace(span_name, span_options) do |span|
-            set_tags!(span, options)
-            yield(span)
+          Tracing.trace(span_name, **span_options) do |span, trace|
+            set_tags!(trace, span, options)
+            yield(span, trace)
           end
         else
-          span = tracer.trace(span_name, span_options)
-          set_tags!(span, options)
+          span = Tracing.trace(span_name, **span_options)
+          trace = Tracing.active_trace
+          set_tags!(trace, span, options)
           span
         end
       end
 
       # Adds tags to a CI test span.
-      def self.set_tags!(span, tags = {})
+      def self.set_tags!(trace, span, tags = {})
         tags ||= {}
 
         # Set default tags
-        span.context.origin = Ext::Test::CONTEXT_ORIGIN if span.context
-        Datadog::Contrib::Analytics.set_measured(span)
-        span.set_tag(Ext::Test::TAG_SPAN_KIND, Ext::AppTypes::TEST)
-        Ext::Environment.tags(ENV).each { |k, v| span.set_tag(k, v) }
+        trace.origin = Ext::Test::CONTEXT_ORIGIN if trace
+        Datadog::Tracing::Contrib::Analytics.set_measured(span)
+        span.set_tag(Ext::Test::TAG_SPAN_KIND, Ext::AppTypes::TYPE_TEST)
+
+        # Set environment tags
+        @environment_tags ||= Ext::Environment.tags(ENV)
+        @environment_tags.each { |k, v| span.set_tag(k, v) }
 
         # Set contextual tags
         span.set_tag(Ext::Test::TAG_FRAMEWORK, tags[:framework]) if tags[:framework]
+        span.set_tag(Ext::Test::TAG_FRAMEWORK_VERSION, tags[:framework_version]) if tags[:framework_version]
         span.set_tag(Ext::Test::TAG_NAME, tags[:test_name]) if tags[:test_name]
         span.set_tag(Ext::Test::TAG_SUITE, tags[:test_suite]) if tags[:test_suite]
         span.set_tag(Ext::Test::TAG_TYPE, tags[:test_type]) if tags[:test_type]

data/lib/datadog/ci.rb

@@ -1,5 +1,8 @@
 # typed: strict
-require 'ddtrace'
+
+require_relative 'core'
+require_relative 'tracing'
+require_relative 'tracing/contrib'
 
 module Datadog
   # Namespace for Datadog CI instrumentation:
@@ -9,9 +12,9 @@ module Datadog
 end
 
 # Integrations
-require 'datadog/ci/contrib/cucumber/integration'
-require 'datadog/ci/contrib/rspec/integration'
+require_relative 'ci/contrib/cucumber/integration'
+require_relative 'ci/contrib/rspec/integration'
 
 # Extensions
-require 'datadog/ci/extensions'
+require_relative 'ci/extensions'
 Datadog::CI::Extensions.activate!

data/lib/datadog/core/buffer/cruby.rb

@@ -0,0 +1,55 @@
+# typed: true
+
+require_relative 'random'
+
+module Datadog
+  module Core
+    module Buffer
+      # Buffer that stores objects, has a maximum size, and
+      # can be safely used concurrently with CRuby.
+      #
+      # Because singular +Array+ operations are thread-safe in CRuby,
+      # we can implement the buffer without an explicit lock,
+      # while making the compromise of allowing the buffer to go
+      # over its maximum limit under extreme circumstances.
+      #
+      # On the following scenario:
+      # * 4.5 million spans/second.
+      # * Pushed into a single CRubyTraceBuffer from 1000 threads.
+      #
+      # This implementation allocates less memory and is faster
+      # than {Datadog::Core::Buffer::ThreadSafe}.
+      #
+      # @see spec/ddtrace/benchmark/buffer_benchmark_spec.rb Buffer benchmarks
+      # @see https://github.com/ruby-concurrency/concurrent-ruby/blob/c1114a0c6891d9634f019f1f9fe58dcae8658964/lib/concurrent-ruby/concurrent/array.rb#L23-L27
+      class CRuby < Random
+        # A very large number to allow us to effectively
+        # drop all items when invoking `slice!(i, FIXNUM_MAX)`.
+        FIXNUM_MAX = (1 << 62) - 1
+
+        # Add a new ``item`` in the local queue. This method doesn't block the execution
+        # even if the buffer is full. In that case, a random item is discarded.
+        def replace!(item)
+          # Ensure buffer stays within +max_size+ items.
+          # This can happen when there's concurrent modification
+          # between a call the check in `full?` and the `add!` call in
+          # `full? ? replace!(item) : add!(item)`.
+          #
+          # We can still have `@items.size > @max_size` for a short period of
+          # time, but we will always try to correct it here.
+          #
+          # `slice!` is performed before `delete_at` & `<<` to avoid always
+          # removing the item that was just inserted.
+          #
+          # DEV: `slice!` with two integer arguments is ~10% faster than
+          # `slice!` with a {Range} argument.
+          @items.slice!(@max_size, FIXNUM_MAX)
+
+          # We should replace a random item with the new one
+          replace_index = rand(@max_size)
+          @items[replace_index] = item
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/buffer/random.rb

@@ -0,0 +1,134 @@
+# typed: true
+
+module Datadog
+  module Core
+    module Buffer
+      # Buffer that accumulates items for a consumer.
+      # Consumption can happen from a different thread.
+
+      # Buffer that stores objects. The buffer has a maximum size and when
+      # the buffer is full, a random object is discarded.
+      class Random
+        def initialize(max_size)
+          @max_size = max_size
+          @items = []
+          @closed = false
+        end
+
+        # Add a new ``item`` in the local queue. This method doesn't block the execution
+        # even if the buffer is full.
+        #
+        # When the buffer is full, we try to ensure that we are fairly choosing newly
+        # pushed items by randomly inserting them into the buffer slots. This discards
+        # old items randomly while trying to ensure that recent items are still captured.
+        def push(item)
+          return if closed?
+
+          full? ? replace!(item) : add!(item)
+          item
+        end
+
+        # A bulk push alternative to +#push+. Use this method if
+        # pushing more than one item for efficiency.
+        def concat(items)
+          return if closed?
+
+          # Segment items into underflow and overflow
+          underflow, overflow = overflow_segments(items)
+
+          # Concatenate items do not exceed capacity.
+          add_all!(underflow) unless underflow.nil?
+
+          # Iteratively replace items, to ensure pseudo-random replacement.
+          overflow.each { |item| replace!(item) } unless overflow.nil?
+        end
+
+        # Stored items are returned and the local buffer is reset.
+        def pop
+          drain!
+        end
+
+        # Return the current number of stored items.
+        def length
+          @items.length
+        end
+
+        # Return if the buffer is empty.
+        def empty?
+          @items.empty?
+        end
+
+        # Closes this buffer, preventing further pushing.
+        # Draining is still allowed.
+        def close
+          @closed = true
+        end
+
+        def closed?
+          @closed
+        end
+
+        protected
+
+        # Segment items into two segments: underflow and overflow.
+        # Underflow are items that will fit into buffer.
+        # Overflow are items that will exceed capacity, after underflow is added.
+        # Returns each array, and nil if there is no underflow/overflow.
+        def overflow_segments(items)
+          underflow = nil
+          overflow = nil
+
+          overflow_size = @max_size > 0 ? (@items.length + items.length) - @max_size : 0
+
+          if overflow_size > 0
+            # Items will overflow
+            if overflow_size < items.length
+              # Partial overflow
+              underflow_end_index = items.length - overflow_size - 1
+              underflow = items[0..underflow_end_index]
+              overflow = items[(underflow_end_index + 1)..-1]
+            else
+              # Total overflow
+              overflow = items
+            end
+          else
+            # Items do not exceed capacity.
+            underflow = items
+          end
+
+          [underflow, overflow]
+        end
+
+        def full?
+          @max_size > 0 && @items.length >= @max_size
+        end
+
+        def add_all!(items)
+          @items.concat(items)
+        end
+
+        def add!(item)
+          @items << item
+        end
+
+        def replace!(item)
+          # Choose random item to be replaced
+          replace_index = rand(@items.length)
+
+          # Replace random item
+          discarded_item = @items[replace_index]
+          @items[replace_index] = item
+
+          # Return discarded item
+          discarded_item
+        end
+
+        def drain!
+          items = @items
+          @items = []
+          items
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/buffer/thread_safe.rb

@@ -0,0 +1,58 @@
+# typed: true
+
+require_relative 'random'
+
+module Datadog
+  module Core
+    module Buffer
+      # Buffer that stores objects, has a maximum size, and
+      # can be safely used concurrently on any environment.
+      #
+      # This implementation uses a {Mutex} around public methods, incurring
+      # overhead in order to ensure thread-safety.
+      #
+      # This is implementation is recommended for non-CRuby environments.
+      # If using CRuby, {Datadog::Core::Buffer::CRuby} is a faster implementation with minimal compromise.
+      class ThreadSafe < Random
+        def initialize(max_size)
+          super
+
+          @mutex = Mutex.new
+        end
+
+        # Add a new ``item`` in the local queue. This method doesn't block the execution
+        # even if the buffer is full. In that case, a random item is discarded.
+        def push(item)
+          synchronize { super }
+        end
+
+        def concat(items)
+          synchronize { super }
+        end
+
+        # Return the current number of stored items.
+        def length
+          synchronize { super }
+        end
+
+        # Return if the buffer is empty.
+        def empty?
+          synchronize { super }
+        end
+
+        # Stored items are returned and the local buffer is reset.
+        def pop
+          synchronize { super }
+        end
+
+        def close
+          synchronize { super }
+        end
+
+        def synchronize(&block)
+          @mutex.synchronize(&block)
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/chunker.rb

@@ -0,0 +1,35 @@
+# typed: false
+
+module Datadog
+  module Core
+    # Chunks list of elements into batches
+    module Chunker
+      module_function
+
+      # Chunks a list into batches of at most +max_chunk_size+ elements each.
+      #
+      # An exception can occur if a single element is too large. That single
+      # element will be returned in its own chunk. You have to verify by yourself
+      # when such elements are returned.
+      #
+      # @param list [Enumerable] list of elements
+      # @param max_chunk_size [Numeric] maximum acceptable chunk size
+      # @return [Enumerable] lazy list of chunks
+      def chunk_by_size(list, max_chunk_size)
+        chunk_agg = 0
+        list.slice_before do |elem|
+          size = elem.size
+          chunk_agg += size
+          if chunk_agg > max_chunk_size
+            # Can't fit element in current chunk, start a new one.
+            chunk_agg = size
+            true
+          else
+            # Add to current chunk
+            false
+          end
+        end
+      end
+    end
+  end
+end