ddtrace 0.38.0

data/lib/ddtrace/transport/io.rb

@@ -0,0 +1,26 @@
+require 'ddtrace/encoding'
+require 'ddtrace/transport/io/client'
+require 'ddtrace/transport/io/traces'
+
+module Datadog
+  module Transport
+    # Namespace for IO transport components
+    module IO
+      module_function
+
+      # Builds a new Transport::IO::Client
+      def new(out, encoder)
+        Client.new(out, encoder)
+      end
+
+      # Builds a new Transport::IO::Client with default settings
+      # Pass options to override any settings.
+      def default(options = {})
+        new(
+          options.fetch(:out, STDOUT),
+          options.fetch(:encoder, Encoding::JSONEncoder)
+        )
+      end
+    end
+  end
+end

data/lib/ddtrace/transport/io/client.rb

@@ -0,0 +1,76 @@
+require 'ddtrace/transport/statistics'
+require 'ddtrace/transport/io/response'
+
+module Datadog
+  module Transport
+    module IO
+      # Encodes and writes tracer data to IO
+      class Client
+        include Transport::Statistics
+
+        attr_reader \
+          :encoder,
+          :out
+
+        def initialize(out, encoder)
+          @out = out
+          @encoder = encoder
+        end
+
+        def send_request(request)
+          # Write data to IO
+          # If block is given, allow it to handle writing
+          # Otherwise use default encoding.
+          response = if block_given?
+                       yield(out, request)
+                     else
+                       send_default_request(out, request)
+                     end
+
+          # Update statistics
+          update_stats_from_response!(response)
+
+          # Return response
+          response
+        rescue StandardError => e
+          message = "Internal error during IO transport request. Cause: #{e.message} Location: #{e.backtrace.first}"
+
+          # Log error
+          if stats.consecutive_errors > 0
+            Datadog.logger.debug(message)
+          else
+            Datadog.logger.error(message)
+          end
+
+          # Update statistics
+          update_stats_from_exception!(e)
+
+          InternalErrorResponse.new(e)
+        end
+
+        protected
+
+        def encode_data(encoder, request)
+          request.parcel.encode_with(encoder)
+        end
+
+        def write_data(out, data)
+          out.puts(data)
+        end
+
+        private
+
+        def send_default_request(out, request)
+          # Encode data
+          data = encode_data(encoder, request)
+
+          # Write to IO
+          result = write_data(out, data)
+
+          # Generate a response
+          IO::Response.new(result)
+        end
+      end
+    end
+  end
+end

data/lib/ddtrace/transport/io/response.rb

@@ -0,0 +1,25 @@
+require 'ddtrace/transport/response'
+
+module Datadog
+  module Transport
+    module IO
+      # Response from HTTP transport for traces
+      class Response
+        include Transport::Response
+        include Transport::Traces::Response
+
+        attr_reader \
+          :result
+
+        def initialize(result, trace_count = 1)
+          @result = result
+          @trace_count = trace_count
+        end
+
+        def ok?
+          true
+        end
+      end
+    end
+  end
+end

data/lib/ddtrace/transport/io/traces.rb

@@ -0,0 +1,91 @@
+require 'ddtrace/transport/traces'
+
+require 'ddtrace/transport/io/response'
+require 'ddtrace/transport/io/client'
+
+module Datadog
+  module Transport
+    module IO
+      # IO transport behavior for traces
+      module Traces
+        # Response from HTTP transport for traces
+        class Response < IO::Response
+        end
+
+        # Extensions for HTTP client
+        module Client
+          def send_traces(traces)
+            # Build a request
+            req = Transport::Traces::Request.new(Parcel.new(traces))
+
+            [send_request(req) do |out, request|
+              # Encode trace data
+              data = encode_data(encoder, request)
+
+              # Write to IO
+              result = if block_given?
+                         yield(out, data)
+                       else
+                         write_data(out, data)
+                       end
+
+              # Generate response
+              Traces::Response.new(result)
+            end]
+          end
+        end
+
+        # Encoder for IO-specific trace encoding
+        # API compliant when used with {JSONEncoder}.
+        module Encoder
+          ENCODED_IDS = [
+            :trace_id,
+            :span_id,
+            :parent_id
+          ].freeze
+
+          # Encodes a list of traces
+          def encode_traces(encoder, traces)
+            trace_hashes = traces.map do |trace|
+              encode_trace(trace)
+            end
+
+            # Wrap traces & encode them
+            encoder.encode(traces: trace_hashes)
+          end
+
+          private
+
+          def encode_trace(trace)
+            # Convert each trace to hash
+            trace.map(&:to_hash).tap do |spans|
+              # Convert IDs to hexadecimal
+              spans.each do |span|
+                ENCODED_IDS.each do |id|
+                  span[id] = span[id].to_s(16) if span.key?(id)
+                end
+              end
+            end
+          end
+        end
+
+        # Transfer object for list of traces
+        class Parcel
+          include Transport::Parcel
+          include Encoder
+
+          def count
+            data.length
+          end
+
+          def encode_with(encoder)
+            encode_traces(encoder, data)
+          end
+        end
+
+        # Add traces behavior to transport components
+        IO::Client.send(:include, Traces::Client)
+      end
+    end
+  end
+end

data/lib/ddtrace/transport/parcel.rb

@@ -0,0 +1,13 @@
+module Datadog
+  module Transport
+    # Data transfer object for generic data
+    module Parcel
+      attr_reader \
+        :data
+
+      def initialize(data)
+        @data = data
+      end
+    end
+  end
+end

data/lib/ddtrace/transport/request.rb

@@ -0,0 +1,13 @@
+module Datadog
+  module Transport
+    # Defines request for transport operations
+    class Request
+      attr_reader \
+        :parcel
+
+      def initialize(parcel)
+        @parcel = parcel
+      end
+    end
+  end
+end

data/lib/ddtrace/transport/response.rb

@@ -0,0 +1,60 @@
+module Datadog
+  module Transport
+    # Defines abstract response for transport operations
+    module Response
+      def payload
+        nil
+      end
+
+      def ok?
+        nil
+      end
+
+      def unsupported?
+        nil
+      end
+
+      def not_found?
+        nil
+      end
+
+      def client_error?
+        nil
+      end
+
+      def server_error?
+        nil
+      end
+
+      def internal_error?
+        nil
+      end
+
+      def inspect
+        "#{self.class} ok?:#{ok?} unsupported?:#{unsupported?}, " \
+        "not_found?:#{not_found?}, client_error?:#{client_error?}, " \
+        "server_error?:#{server_error?}, internal_error?:#{internal_error?}, " \
+        "payload:#{payload}"
+      end
+    end
+
+    # A generic error response for internal errors
+    class InternalErrorResponse
+      include Response
+
+      attr_reader :error
+
+      def initialize(error)
+        @error = error
+      end
+
+      def internal_error?
+        true
+      end
+
+      def inspect
+        "#{super}, error_type:#{error.class} error:#{error}"
+      end
+    end
+  end
+end

data/lib/ddtrace/transport/statistics.rb

@@ -0,0 +1,72 @@
+require 'ddtrace/diagnostics/health'
+
+module Datadog
+  module Transport
+    # Tracks statistics for transports
+    module Statistics
+      def stats
+        @stats ||= Counts.new
+      end
+
+      def update_stats_from_response!(response)
+        if response.ok?
+          stats.success += 1
+          stats.consecutive_errors = 0
+        else
+          stats.client_error += 1 if response.client_error?
+          stats.server_error += 1 if response.server_error?
+          stats.internal_error += 1 if response.internal_error?
+          stats.consecutive_errors += 1
+        end
+
+        # Send health metrics
+        Datadog.health_metrics.send_metrics(
+          metrics_for_response(response).values
+        )
+      end
+
+      def metrics_for_response(response)
+        {}.tap do |metrics|
+          metrics[:api_errors] = Metrics::Metric.new(:api_errors, nil, 1) if response.internal_error?
+          metrics[:api_responses] = Metrics::Metric.new(:api_responses, nil, 1) unless response.internal_error?
+        end
+      end
+
+      def update_stats_from_exception!(exception)
+        stats.internal_error += 1
+        stats.consecutive_errors += 1
+
+        # Send health metrics
+        Datadog.health_metrics.send_metrics(
+          metrics_for_exception(exception).values
+        )
+      end
+
+      def metrics_for_exception(_exception)
+        { api_errors: Metrics::Metric.new(:api_errors, nil, 1) }
+      end
+
+      # Stat counts
+      class Counts
+        attr_accessor \
+          :success,
+          :client_error,
+          :server_error,
+          :internal_error,
+          :consecutive_errors
+
+        def initialize
+          reset!
+        end
+
+        def reset!
+          @success = 0
+          @client_error = 0
+          @server_error = 0
+          @internal_error = 0
+          @consecutive_errors = 0
+        end
+      end
+    end
+  end
+end

data/lib/ddtrace/transport/traces.rb

@@ -0,0 +1,183 @@
+require 'ddtrace/transport/parcel'
+require 'ddtrace/transport/request'
+require 'ddtrace/chunker'
+
+module Datadog
+  module Transport
+    module Traces
+      # Data transfer object for encoded traces
+      class EncodedParcel
+        include Transport::Parcel
+
+        attr_reader :trace_count
+
+        def initialize(data, trace_count)
+          super(data)
+          @trace_count = trace_count
+        end
+
+        def count
+          data.length
+        end
+      end
+
+      # Traces request
+      class Request < Transport::Request
+      end
+
+      # Traces response
+      module Response
+        attr_reader :service_rates, :trace_count
+      end
+
+      # Traces chunker
+      class Chunker
+        # Trace agent limit payload size of 10 MiB (since agent v5.11.0):
+        # https://github.com/DataDog/datadog-agent/blob/6.14.1/pkg/trace/api/api.go#L46
+        #
+        # We set the value to a conservative 5 MiB, in case network speed is slow.
+        DEFAULT_MAX_PAYLOAD_SIZE = 5 * 1024 * 1024
+
+        attr_reader :encoder, :max_size
+
+        #
+        # Single traces larger than +max_size+ will be discarded.
+        #
+        # @param encoder [Datadog::Encoding::Encoder]
+        # @param max_size [String] maximum acceptable payload size
+        def initialize(encoder, max_size: DEFAULT_MAX_PAYLOAD_SIZE)
+          @encoder = encoder
+          @max_size = max_size
+        end
+
+        # Encodes a list of traces in chunks.
+        # Before serializing, all traces are normalized. Trace nesting is not changed.
+        #
+        # @param traces [Enumerable<Trace>] list of traces
+        # @return [Enumerable[Array[Bytes,Integer]]] list of encoded chunks: each containing a byte array and
+        #   number of traces
+        def encode_in_chunks(traces)
+          encoded_traces = traces.map { |t| encode_one(t) }.reject(&:nil?)
+
+          Datadog::Chunker.chunk_by_size(encoded_traces, max_size).map do |chunk|
+            [encoder.join(chunk), chunk.size]
+          end
+        end
+
+        private
+
+        def encode_one(trace)
+          encoded = Encoder.encode_trace(encoder, trace)
+
+          if encoded.size > max_size
+            # This single trace is too large, we can't flush it
+            Datadog.logger.debug { "Dropping trace. Payload too large: '#{trace.map(&:to_hash)}'" }
+            Datadog.health_metrics.transport_trace_too_large(1)
+
+            return nil
+          end
+
+          encoded
+        end
+      end
+
+      # Encodes traces using {Datadog::Encoding::Encoder} instances.
+      module Encoder
+        module_function
+
+        def encode_trace(encoder, trace)
+          encoder.encode(trace.map(&:to_hash))
+        end
+      end
+
+      # Sends traces based on transport API configuration.
+      #
+      # This class initializes the HTTP client, breaks down large
+      # batches of traces into smaller chunks and handles
+      # API version downgrade handshake.
+      class Transport
+        attr_reader :client, :apis, :default_api, :current_api_id
+
+        def initialize(apis, default_api)
+          @apis = apis
+          @default_api = default_api
+
+          change_api!(default_api)
+        end
+
+        def send_traces(traces)
+          encoder = current_api.encoder
+          chunker = Datadog::Transport::Traces::Chunker.new(encoder)
+
+          responses = chunker.encode_in_chunks(traces.lazy).map do |encoded_traces, trace_count|
+            request = Request.new(EncodedParcel.new(encoded_traces, trace_count))
+
+            client.send_payload(request).tap do |response|
+              if downgrade?(response)
+                downgrade!
+                return send_traces(traces)
+              end
+            end
+          end.force
+
+          Datadog.health_metrics.transport_chunked(responses.size)
+
+          responses
+        end
+
+        def stats
+          @client.stats
+        end
+
+        def current_api
+          apis[@current_api_id]
+        end
+
+        private
+
+        def downgrade?(response)
+          return false unless apis.fallbacks.key?(@current_api_id)
+          response.not_found? || response.unsupported?
+        end
+
+        def downgrade!
+          downgrade_api_id = apis.fallbacks[@current_api_id]
+          raise NoDowngradeAvailableError, @current_api_id if downgrade_api_id.nil?
+          change_api!(downgrade_api_id)
+        end
+
+        def change_api!(api_id)
+          raise UnknownApiVersionError, api_id unless apis.key?(api_id)
+          @current_api_id = api_id
+          @client = HTTP::Client.new(current_api)
+        end
+
+        # Raised when configured with an unknown API version
+        class UnknownApiVersionError < StandardError
+          attr_reader :version
+
+          def initialize(version)
+            @version = version
+          end
+
+          def message
+            "No matching transport API for version #{version}!"
+          end
+        end
+
+        # Raised when configured with an unknown API version
+        class NoDowngradeAvailableError < StandardError
+          attr_reader :version
+
+          def initialize(version)
+            @version = version
+          end
+
+          def message
+            "No downgrade from transport API version #{version} is available!"
+          end
+        end
+      end
+    end
+  end
+end