ddtrace 0.13.0 → 0.47.0

data/lib/ddtrace/transport/statistics.rb

@@ -0,0 +1,72 @@
+require 'ddtrace/diagnostics/health'
+
+module Datadog
+  module Transport
+    # Tracks statistics for transports
+    module Statistics
+      def stats
+        @stats ||= Counts.new
+      end
+
+      def update_stats_from_response!(response)
+        if response.ok?
+          stats.success += 1
+          stats.consecutive_errors = 0
+        else
+          stats.client_error += 1 if response.client_error?
+          stats.server_error += 1 if response.server_error?
+          stats.internal_error += 1 if response.internal_error?
+          stats.consecutive_errors += 1
+        end
+
+        # Send health metrics
+        Datadog.health_metrics.send_metrics(
+          metrics_for_response(response).values
+        )
+      end
+
+      def metrics_for_response(response)
+        {}.tap do |metrics|
+          metrics[:api_errors] = Metrics::Metric.new(:api_errors, nil, 1) if response.internal_error?
+          metrics[:api_responses] = Metrics::Metric.new(:api_responses, nil, 1) unless response.internal_error?
+        end
+      end
+
+      def update_stats_from_exception!(exception)
+        stats.internal_error += 1
+        stats.consecutive_errors += 1
+
+        # Send health metrics
+        Datadog.health_metrics.send_metrics(
+          metrics_for_exception(exception).values
+        )
+      end
+
+      def metrics_for_exception(_exception)
+        { api_errors: Metrics::Metric.new(:api_errors, nil, 1) }
+      end
+
+      # Stat counts
+      class Counts
+        attr_accessor \
+          :success,
+          :client_error,
+          :server_error,
+          :internal_error,
+          :consecutive_errors
+
+        def initialize
+          reset!
+        end
+
+        def reset!
+          @success = 0
+          @client_error = 0
+          @server_error = 0
+          @internal_error = 0
+          @consecutive_errors = 0
+        end
+      end
+    end
+  end
+end

data/lib/ddtrace/transport/traces.rb

@@ -0,0 +1,205 @@
+require 'ddtrace/transport/parcel'
+require 'ddtrace/transport/request'
+require 'ddtrace/chunker'
+
+module Datadog
+  module Transport
+    module Traces
+      # Data transfer object for encoded traces
+      class EncodedParcel
+        include Transport::Parcel
+
+        attr_reader :trace_count
+
+        def initialize(data, trace_count)
+          super(data)
+          @trace_count = trace_count
+        end
+
+        def count
+          data.length
+        end
+      end
+
+      # Traces request
+      class Request < Transport::Request
+      end
+
+      # Traces response
+      module Response
+        attr_reader :service_rates, :trace_count
+      end
+
+      # Traces chunker
+      class Chunker
+        # Trace agent limit payload size of 10 MiB (since agent v5.11.0):
+        # https://github.com/DataDog/datadog-agent/blob/6.14.1/pkg/trace/api/api.go#L46
+        #
+        # We set the value to a conservative 5 MiB, in case network speed is slow.
+        DEFAULT_MAX_PAYLOAD_SIZE = 5 * 1024 * 1024
+
+        attr_reader :encoder, :max_size
+
+        #
+        # Single traces larger than +max_size+ will be discarded.
+        #
+        # @param encoder [Datadog::Encoding::Encoder]
+        # @param max_size [String] maximum acceptable payload size
+        def initialize(encoder, max_size: DEFAULT_MAX_PAYLOAD_SIZE)
+          @encoder = encoder
+          @max_size = max_size
+        end
+
+        # Encodes a list of traces in chunks.
+        # Before serializing, all traces are normalized. Trace nesting is not changed.
+        #
+        # @param traces [Enumerable<Trace>] list of traces
+        # @return [Enumerable[Array[Bytes,Integer]]] list of encoded chunks: each containing a byte array and
+        #   number of traces
+        def encode_in_chunks(traces)
+          encoded_traces = if traces.respond_to?(:filter_map)
+                             # DEV Supported since Ruby 2.7, saves an intermediate object creation
+                             traces.filter_map { |t| encode_one(t) }
+                           else
+                             traces.map { |t| encode_one(t) }.reject(&:nil?)
+                           end
+
+          Datadog::Chunker.chunk_by_size(encoded_traces, max_size).map do |chunk|
+            [encoder.join(chunk), chunk.size]
+          end
+        end
+
+        private
+
+        def encode_one(trace)
+          encoded = Encoder.encode_trace(encoder, trace)
+
+          if encoded.size > max_size
+            # This single trace is too large, we can't flush it
+            Datadog.logger.debug { "Dropping trace. Payload too large: '#{trace.map(&:to_hash)}'" }
+            Datadog.health_metrics.transport_trace_too_large(1)
+
+            return nil
+          end
+
+          encoded
+        end
+      end
+
+      # Encodes traces using {Datadog::Encoding::Encoder} instances.
+      module Encoder
+        module_function
+
+        def encode_trace(encoder, trace)
+          encoder.encode(trace)
+        end
+      end
+
+      # Sends traces based on transport API configuration.
+      #
+      # This class initializes the HTTP client, breaks down large
+      # batches of traces into smaller chunks and handles
+      # API version downgrade handshake.
+      class Transport
+        attr_reader :client, :apis, :default_api, :current_api_id
+
+        def initialize(apis, default_api)
+          @apis = apis
+          @default_api = default_api
+
+          change_api!(default_api)
+        end
+
+        def send_traces(traces)
+          encoder = current_api.encoder
+          chunker = Datadog::Transport::Traces::Chunker.new(encoder)
+
+          responses = chunker.encode_in_chunks(traces.lazy).map do |encoded_traces, trace_count|
+            request = Request.new(EncodedParcel.new(encoded_traces, trace_count))
+
+            client.send_payload(request).tap do |response|
+              if downgrade?(response)
+                downgrade!
+                return send_traces(traces)
+              end
+            end
+          end
+
+          # Force resolution of lazy enumerator.
+          #
+          # The "correct" method to call here would be `#force`,
+          # as this method was created to force the eager loading
+          # of a lazy enumerator.
+          #
+          # Unfortunately, JRuby < 9.2.9.0 erroneously eagerly loads
+          # the lazy Enumerator during intermediate steps.
+          # This forces us to use `#to_a`, as this method works for both
+          # lazy and regular Enumerators.
+          # Using `#to_a` can mask the fact that we expect a lazy
+          # Enumerator.
+          responses = responses.to_a
+
+          Datadog.health_metrics.transport_chunked(responses.size)
+
+          responses
+        end
+
+        def stats
+          @client.stats
+        end
+
+        def current_api
+          apis[@current_api_id]
+        end
+
+        private
+
+        def downgrade?(response)
+          return false unless apis.fallbacks.key?(@current_api_id)
+
+          response.not_found? || response.unsupported?
+        end
+
+        def downgrade!
+          downgrade_api_id = apis.fallbacks[@current_api_id]
+          raise NoDowngradeAvailableError, @current_api_id if downgrade_api_id.nil?
+
+          change_api!(downgrade_api_id)
+        end
+
+        def change_api!(api_id)
+          raise UnknownApiVersionError, api_id unless apis.key?(api_id)
+
+          @current_api_id = api_id
+          @client = HTTP::Client.new(current_api)
+        end
+
+        # Raised when configured with an unknown API version
+        class UnknownApiVersionError < StandardError
+          attr_reader :version
+
+          def initialize(version)
+            @version = version
+          end
+
+          def message
+            "No matching transport API for version #{version}!"
+          end
+        end
+
+        # Raised when configured with an unknown API version
+        class NoDowngradeAvailableError < StandardError
+          attr_reader :version
+
+          def initialize(version)
+            @version = version
+          end
+
+          def message
+            "No downgrade from transport API version #{version} is available!"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ddtrace/utils.rb

@@ -1,32 +1,31 @@
 require 'ddtrace/utils/database'
+require 'ddtrace/utils/forking'
 
 module Datadog
   # Utils contains low-level utilities, typically to provide pseudo-random trace IDs.
   module Utils
-    STRING_PLACEHOLDER = ''.encode(::Encoding::UTF_8).freeze
+    extend Utils::Forking
+
+    EMPTY_STRING = ''.encode(::Encoding::UTF_8).freeze
     # We use a custom random number generator because we want no interference
     # with the default one. Using the default prng, we could break code that
     # would rely on srand/rand sequences.
 
     # Return a span id
     def self.next_id
-      reset! if was_forked?
-
-      @rnd.rand(Datadog::Span::MAX_ID)
+      after_fork! { reset! }
+      id_rng.rand(Datadog::Span::RUBY_MAX_ID)
     end
 
-    def self.reset!
-      @pid = Process.pid
-      @rnd = Random.new
+    def self.id_rng
+      @id_rng ||= Random.new
     end
 
-    def self.was_forked?
-      Process.pid != @pid
+    def self.reset!
+      @id_rng = Random.new
     end
 
-    private_class_method :reset!, :was_forked?
-
-    reset!
+    private_class_method :id_rng, :reset!
 
     def self.truncate(value, size, omission = '...'.freeze)
       string = value.to_s
@@ -53,13 +52,17 @@ module Datadog
         str.encode('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: '')
       elsif str.encoding == ::Encoding::UTF_8
         str
+      elsif str.empty?
+        # DEV Optimization as `nil.to_s` is a very common source for an empty string,
+        # DEV but it comes encoded as US_ASCII.
+        EMPTY_STRING
       else
         str.encode(::Encoding::UTF_8)
       end
     rescue => e
-      Tracer.log.debug("Error encoding string in UTF-8: #{e}")
+      Datadog.logger.debug("Error encoding string in UTF-8: #{e}")
 
-      options.fetch(:placeholder, STRING_PLACEHOLDER)
+      options.fetch(:placeholder, EMPTY_STRING)
     end
   end
 end

data/lib/ddtrace/utils/compression.rb

@@ -0,0 +1,27 @@
+require 'zlib'
+
+module Datadog
+  module Utils
+    # Common database-related utility functions.
+    module Compression
+      module_function
+
+      def gzip(string, level: nil, strategy: nil)
+        sio = StringIO.new
+        sio.binmode
+        gz = Zlib::GzipWriter.new(sio, level, strategy)
+        gz.write(string)
+        gz.close
+        sio.string
+      end
+
+      def gunzip(string, encoding = ::Encoding::ASCII_8BIT)
+        sio = StringIO.new(string)
+        gz = Zlib::GzipReader.new(sio, encoding: encoding)
+        gz.read
+      ensure
+        gz && gz.close
+      end
+    end
+  end
+end

data/lib/ddtrace/utils/database.rb

@@ -2,16 +2,20 @@ module Datadog
   module Utils
     # Common database-related utility functions.
     module Database
+      VENDOR_DEFAULT = 'defaultdb'.freeze
+      VENDOR_POSTGRES = 'postgres'.freeze
+      VENDOR_SQLITE = 'sqlite'.freeze
+
       module_function
 
       def normalize_vendor(vendor)
         case vendor
         when nil
-          'defaultdb'
+          VENDOR_DEFAULT
         when 'postgresql'
-          'postgres'
+          VENDOR_POSTGRES
         when 'sqlite3'
-          'sqlite'
+          VENDOR_SQLITE
         else
           vendor
         end

data/lib/ddtrace/utils/forking.rb

@@ -0,0 +1,52 @@
+module Datadog
+  module Utils
+    # Helper methods for managing forking behavior
+    module Forking
+      def self.included(base)
+        base.send(:prepend, ClassExtensions) if base.is_a?(Class)
+      end
+
+      def self.extended(base)
+        # Explicitly update PID here because there's a case where
+        # the code path that lazily updates the PID may not be exercised
+        # until after a fork occurs, thus causing the event to be missed.
+        # By eagerly setting this, we avoid this scenario.
+        base.update_fork_pid!
+      end
+
+      def after_fork!
+        if forked?
+          yield
+          update_fork_pid!
+          true
+        else
+          false
+        end
+      end
+
+      def forked?
+        Process.pid != fork_pid
+      end
+
+      def update_fork_pid!
+        @fork_pid = Process.pid
+      end
+
+      def fork_pid
+        @fork_pid ||= Process.pid
+      end
+
+      # Adds additional functionality for Classes that implement Forking
+      module ClassExtensions
+        # Addresses an edge case where forking before invoking #update_fork_pid! on the
+        # object will cause forking to not be detected in the fork when it should have.
+        #
+        # This wrapper prevents this by initializing the fork PID when the object is created.
+        def initialize(*args, &block)
+          super
+          update_fork_pid!
+        end
+      end
+    end
+  end
+end