ddtrace 0.34.2 → 0.35.0

data/lib/ddtrace/transport/http.rb

@@ -19,7 +19,7 @@ module Datadog
 
       # Builds a new Transport::HTTP::Client
       def new(&block)
-        Builder.new(&block).to_client
+        Builder.new(&block).to_transport
       end
 
       # Builds a new Transport::HTTP::Client with default settings

data/lib/ddtrace/transport/http/api/instance.rb

@@ -15,6 +15,10 @@ module Datadog
             @headers = options.fetch(:headers, {})
           end
 
+          def encoder
+            spec.encoder
+          end
+
           def call(env)
             # Add headers to request env, unless empty.
             env.headers.merge!(headers) unless headers.empty?

data/lib/ddtrace/transport/http/builder.rb

@@ -70,13 +70,11 @@ module Datadog
           @default_api = key
         end
 
-        def to_client
+        def to_transport
           raise NoDefaultApiError if @default_api.nil?
 
-          @client ||= Client.new(
-            to_api_instances,
-            @default_api
-          )
+          # DEV: Should not be specific to traces
+          Transport::Traces::Transport.new(to_api_instances, @default_api)
         end
 
         def to_api_instances

data/lib/ddtrace/transport/http/client.rb

@@ -8,42 +8,31 @@ module Datadog
       class Client
         include Transport::HTTP::Statistics
 
-        attr_reader \
-          :apis,
-          :current_api_id
+        attr_reader :api
 
-        def initialize(apis, current_api_id)
-          @apis = apis
-
-          # Activate initial API
-          change_api!(current_api_id)
+        def initialize(api)
+          @api = api
         end
 
         def send_request(request, &block)
           # Build request into env
           env = build_env(request)
 
-          # Get response from API
-          response = yield(current_api, env)
+          # Get responses from API
+          response = yield(api, env)
 
           # Update statistics
           update_stats_from_response!(response)
 
-          # If API should be downgraded, downgrade and try again.
-          if downgrade?(response)
-            downgrade!
-            response = send_request(request, &block)
-          end
-
           response
         rescue StandardError => e
           message = "Internal error during HTTP transport request. Cause: #{e.message} Location: #{e.backtrace.first}"
 
           # Log error
           if stats.consecutive_errors > 0
-            Datadog::Logger.log.debug(message)
+            Datadog.logger.debug(message)
           else
-            Datadog::Logger.log.error(message)
+            Datadog.logger.error(message)
           end
 
           # Update statistics
@@ -55,52 +44,6 @@ module Datadog
         def build_env(request)
           Env.new(request)
         end
-
-        def downgrade?(response)
-          return false unless apis.fallbacks.key?(current_api_id)
-          response.not_found? || response.unsupported?
-        end
-
-        def current_api
-          apis[current_api_id]
-        end
-
-        def change_api!(api_id)
-          raise UnknownApiVersionError, api_id unless apis.key?(api_id)
-          @current_api_id = api_id
-        end
-
-        def downgrade!
-          downgrade_api_id = apis.fallbacks[current_api_id]
-          raise NoDowngradeAvailableError, current_api_id if downgrade_api_id.nil?
-          change_api!(downgrade_api_id)
-        end
-
-        # Raised when configured with an unknown API version
-        class UnknownApiVersionError < StandardError
-          attr_reader :version
-
-          def initialize(version)
-            @version = version
-          end
-
-          def message
-            "No matching transport API for version #{version}!"
-          end
-        end
-
-        # Raised when configured with an unknown API version
-        class NoDowngradeAvailableError < StandardError
-          attr_reader :version
-
-          def initialize(version)
-            @version = version
-          end
-
-          def message
-            "No downgrade from transport API version #{version} is available!"
-          end
-        end
       end
     end
   end

data/lib/ddtrace/transport/http/response.rb

@@ -15,7 +15,7 @@ module Datadog
           @http_response = http_response
         end
 
-        def_delegators :@http_response, *Transport::Response.instance_methods
+        def_delegators :@http_response, *Datadog::Transport::Response.instance_methods
 
         def code
           @http_response.respond_to?(:code) ? @http_response.code : nil

data/lib/ddtrace/transport/http/statistics.rb

@@ -6,7 +6,7 @@ module Datadog
       # Tracks statistics for HTTP transports
       module Statistics
         def self.included(base)
-          base.send(:include, Transport::Statistics)
+          base.send(:include, Datadog::Transport::Statistics)
           base.send(:include, InstanceMethods)
         end
 

data/lib/ddtrace/transport/http/traces.rb

@@ -12,19 +12,18 @@ module Datadog
         # Response from HTTP transport for traces
         class Response
           include HTTP::Response
-          include Transport::Traces::Response
+          include Datadog::Transport::Traces::Response
 
           def initialize(http_response, options = {})
             super(http_response)
             @service_rates = options.fetch(:service_rates, nil)
+            @trace_count = options.fetch(:trace_count, 0)
           end
         end
 
         # Extensions for HTTP client
         module Client
-          def send_traces(traces)
-            request = Transport::Traces::Request.new(traces)
-
+          def send_payload(request)
             send_request(request) do |api, env|
               api.send_traces(env)
             end
@@ -45,6 +44,10 @@ module Datadog
               traces.call(env, &block)
             end
 
+            def encoder
+              traces.encoder
+            end
+
             # Raised when traces sent but no traces endpoint is defined
             class NoTraceEndpointDefinedError < StandardError
               attr_reader :spec
@@ -104,17 +107,17 @@ module Datadog
 
             def call(env, &block)
               # Add trace count header
-              env.headers[HEADER_TRACE_COUNT] = env.request.parcel.count.to_s
+              env.headers[HEADER_TRACE_COUNT] = env.request.parcel.trace_count.to_s
 
               # Encode body & type
               env.headers[HEADER_CONTENT_TYPE] = encoder.content_type
-              env.body = env.request.parcel.encode_with(encoder)
+              env.body = env.request.parcel.data
 
               # Query for response
               http_response = super(env, &block)
 
               # Process the response
-              response_options = {}.tap do |options|
+              response_options = { trace_count: env.request.parcel.trace_count }.tap do |options|
                 # Parse service rates, if configured to do so.
                 if service_rates? && !http_response.payload.to_s.empty?
                   body = JSON.parse(http_response.payload)

data/lib/ddtrace/transport/io.rb

@@ -18,7 +18,7 @@ module Datadog
       def default(options = {})
         new(
           options.fetch(:out, STDOUT),
-          options.fetch(:encoder, Encoding::JSONEncoder::V2)
+          options.fetch(:encoder, Encoding::JSONEncoder)
         )
       end
     end

data/lib/ddtrace/transport/io/client.rb

@@ -37,9 +37,9 @@ module Datadog
 
           # Log error
           if stats.consecutive_errors > 0
-            Datadog::Logger.log.debug(message)
+            Datadog.logger.debug(message)
           else
-            Datadog::Logger.log.error(message)
+            Datadog.logger.error(message)
           end
 
           # Update statistics

data/lib/ddtrace/transport/io/response.rb

@@ -6,12 +6,14 @@ module Datadog
       # Response from HTTP transport for traces
       class Response
         include Transport::Response
+        include Transport::Traces::Response
 
         attr_reader \
           :result
 
-        def initialize(result)
+        def initialize(result, trace_count = 1)
           @result = result
+          @trace_count = trace_count
         end
 
         def ok?

data/lib/ddtrace/transport/io/traces.rb

@@ -10,16 +10,15 @@ module Datadog
       module Traces
         # Response from HTTP transport for traces
         class Response < IO::Response
-          include Transport::Traces::Response
         end
 
         # Extensions for HTTP client
         module Client
           def send_traces(traces)
             # Build a request
-            req = Transport::Traces::Request.new(traces)
+            req = Transport::Traces::Request.new(Parcel.new(traces))
 
-            send_request(req) do |out, request|
+            [send_request(req) do |out, request|
               # Encode trace data
               data = encode_data(encoder, request)
 
@@ -32,7 +31,55 @@ module Datadog
 
               # Generate response
               Traces::Response.new(result)
+            end]
+          end
+        end
+
+        # Encoder for IO-specific trace encoding
+        # API compliant when used with {JSONEncoder}.
+        module Encoder
+          ENCODED_IDS = [
+            :trace_id,
+            :span_id,
+            :parent_id
+          ].freeze
+
+          # Encodes a list of traces
+          def encode_traces(encoder, traces)
+            trace_hashes = traces.map do |trace|
+              encode_trace(trace)
             end
+
+            # Wrap traces & encode them
+            encoder.encode(traces: trace_hashes)
+          end
+
+          private
+
+          def encode_trace(trace)
+            # Convert each trace to hash
+            trace.map(&:to_hash).tap do |spans|
+              # Convert IDs to hexadecimal
+              spans.each do |span|
+                ENCODED_IDS.each do |id|
+                  span[id] = span[id].to_s(16) if span.key?(id)
+                end
+              end
+            end
+          end
+        end
+
+        # Transfer object for list of traces
+        class Parcel
+          include Transport::Parcel
+          include Encoder
+
+          def count
+            data.length
+          end
+
+          def encode_with(encoder)
+            encode_traces(encoder, data)
           end
         end
 

data/lib/ddtrace/transport/parcel.rb

@@ -8,10 +8,6 @@ module Datadog
       def initialize(data)
         @data = data
       end
-
-      def encode_with(encoder)
-        encoder.encode(data)
-      end
     end
   end
 end

data/lib/ddtrace/transport/statistics.rb

@@ -20,7 +20,7 @@ module Datadog
         end
 
         # Send health metrics
-        Diagnostics::Health.metrics.send_metrics(
+        Datadog.health_metrics.send_metrics(
           metrics_for_response(response).values
         )
       end
@@ -37,7 +37,7 @@ module Datadog
         stats.consecutive_errors += 1
 
         # Send health metrics
-        Diagnostics::Health.metrics.send_metrics(
+        Datadog.health_metrics.send_metrics(
           metrics_for_exception(exception).values
         )
       end

data/lib/ddtrace/transport/traces.rb

@@ -1,32 +1,182 @@
 require 'ddtrace/transport/parcel'
 require 'ddtrace/transport/request'
+require 'ddtrace/chunker'
 
 module Datadog
   module Transport
     module Traces
-      # Data transfer object for trace data
-      class Parcel
+      # Data transfer object for encoded traces
+      class EncodedParcel
         include Transport::Parcel
 
-        def count
-          data.length
+        attr_reader :trace_count
+
+        def initialize(data, trace_count)
+          super(data)
+          @trace_count = trace_count
         end
 
-        def encode_with(encoder)
-          encoder.encode_traces(data)
+        def count
+          data.length
         end
       end
 
       # Traces request
       class Request < Transport::Request
-        def initialize(traces)
-          super(Parcel.new(traces))
-        end
       end
 
       # Traces response
       module Response
-        attr_reader :service_rates
+        attr_reader :service_rates, :trace_count
+      end
+
+      # Traces chunker
+      class Chunker
+        # Trace agent limit payload size of 10 MiB (since agent v5.11.0):
+        # https://github.com/DataDog/datadog-agent/blob/6.14.1/pkg/trace/api/api.go#L46
+        #
+        # We set the value to a conservative 5 MiB, in case network speed is slow.
+        DEFAULT_MAX_PAYLOAD_SIZE = 5 * 1024 * 1024
+
+        attr_reader :encoder, :max_size
+
+        #
+        # Single traces larger than +max_size+ will be discarded.
+        #
+        # @param encoder [Datadog::Encoding::Encoder]
+        # @param max_size [String] maximum acceptable payload size
+        def initialize(encoder, max_size: DEFAULT_MAX_PAYLOAD_SIZE)
+          @encoder = encoder
+          @max_size = max_size
+        end
+
+        # Encodes a list of traces in chunks.
+        # Before serializing, all traces are normalized. Trace nesting is not changed.
+        #
+        # @param traces [Enumerable<Trace>] list of traces
+        # @return [Enumerable[Array[Bytes,Integer]]] list of encoded chunks: each containing a byte array and
+        #   number of traces
+        def encode_in_chunks(traces)
+          encoded_traces = traces.map { |t| encode_one(t) }.reject(&:nil?)
+
+          Datadog::Chunker.chunk_by_size(encoded_traces, max_size).map do |chunk|
+            [encoder.join(chunk), chunk.size]
+          end
+        end
+
+        private
+
+        def encode_one(trace)
+          encoded = Encoder.encode_trace(encoder, trace)
+
+          if encoded.size > max_size
+            # This single trace is too large, we can't flush it
+            Datadog.logger.debug { "Dropping trace. Payload too large: '#{trace.map(&:to_hash)}'" }
+            Datadog.health_metrics.transport_trace_too_large(1)
+
+            return nil
+          end
+
+          encoded
+        end
+      end
+
+      # Encodes traces using {Datadog::Encoding::Encoder} instances.
+      module Encoder
+        module_function
+
+        def encode_trace(encoder, trace)
+          encoder.encode(trace.map(&:to_hash))
+        end
+      end
+
+      # Sends traces based on transport API configuration.
+      #
+      # This class initializes the HTTP client, breaks down large
+      # batches of traces into smaller chunks and handles
+      # API version downgrade handshake.
+      class Transport
+        attr_reader :client, :apis, :default_api, :current_api_id
+
+        def initialize(apis, default_api)
+          @apis = apis
+          @default_api = default_api
+
+          change_api!(default_api)
+        end
+
+        def send_traces(traces)
+          encoder = current_api.encoder
+          chunker = Datadog::Transport::Traces::Chunker.new(encoder)
+
+          responses = chunker.encode_in_chunks(traces.lazy).map do |encoded_traces, trace_count|
+            request = Request.new(EncodedParcel.new(encoded_traces, trace_count))
+
+            client.send_payload(request).tap do |response|
+              if downgrade?(response)
+                downgrade!
+                return send_traces(traces)
+              end
+            end
+          end.force
+
+          Datadog.health_metrics.transport_chunked(responses.size)
+
+          responses
+        end
+
+        def stats
+          @client.stats
+        end
+
+        def current_api
+          apis[@current_api_id]
+        end
+
+        private
+
+        def downgrade?(response)
+          return false unless apis.fallbacks.key?(@current_api_id)
+          response.not_found? || response.unsupported?
+        end
+
+        def downgrade!
+          downgrade_api_id = apis.fallbacks[@current_api_id]
+          raise NoDowngradeAvailableError, @current_api_id if downgrade_api_id.nil?
+          change_api!(downgrade_api_id)
+        end
+
+        def change_api!(api_id)
+          raise UnknownApiVersionError, api_id unless apis.key?(api_id)
+          @current_api_id = api_id
+          @client = HTTP::Client.new(current_api)
+        end
+
+        # Raised when configured with an unknown API version
+        class UnknownApiVersionError < StandardError
+          attr_reader :version
+
+          def initialize(version)
+            @version = version
+          end
+
+          def message
+            "No matching transport API for version #{version}!"
+          end
+        end
+
+        # Raised when configured with an unknown API version
+        class NoDowngradeAvailableError < StandardError
+          attr_reader :version
+
+          def initialize(version)
+            @version = version
+          end
+
+          def message
+            "No downgrade from transport API version #{version} is available!"
+          end
+        end
       end
     end
   end