ddtrace 1.14.0 → 1.16.2

data/lib/datadog/tracing/transport/serializable_trace.rb

@@ -0,0 +1,126 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'msgpack'
+require 'datadog/tracing/utils'
+
+module Datadog
+  module Tracing
+    module Transport
+      # Adds serialization functions to a {Datadog::TraceSegment}
+      class SerializableTrace
+        attr_reader \
+          :trace
+
+        def initialize(trace)
+          @trace = trace
+        end
+
+        # MessagePack serializer interface. Making this object
+        # respond to `#to_msgpack` allows it to be automatically
+        # serialized by MessagePack.
+        #
+        # This is more efficient than doing +MessagePack.pack(span.to_hash)+
+        # as we don't have to create an intermediate Hash.
+        #
+        # @param packer [MessagePack::Packer] serialization buffer, can be +nil+ with JRuby
+        def to_msgpack(packer = nil)
+          # As of 1.3.3, JRuby implementation doesn't pass an existing packer
+          trace.spans.map { |s| SerializableSpan.new(s) }.to_msgpack(packer)
+        end
+
+        # JSON serializer interface.
+        # Used by older version of the transport.
+        def to_json(*args)
+          trace.spans.map { |s| SerializableSpan.new(s).to_hash }.to_json(*args)
+        end
+      end
+
+      # Adds serialization functions to a {Datadog::Span}
+      class SerializableSpan
+        attr_reader \
+          :span
+
+        def initialize(span)
+          @span = span
+          @trace_id = Tracing::Utils::TraceId.to_low_order(span.trace_id)
+        end
+
+        # MessagePack serializer interface. Making this object
+        # respond to `#to_msgpack` allows it to be automatically
+        # serialized by MessagePack.
+        #
+        # This is more efficient than doing +MessagePack.pack(span.to_hash)+
+        # as we don't have to create an intermediate Hash.
+        #
+        # @param packer [MessagePack::Packer] serialization buffer, can be +nil+ with JRuby
+        # rubocop:disable Metrics/AbcSize
+        def to_msgpack(packer = nil)
+          packer ||= MessagePack::Packer.new
+
+          number_of_elements_to_write = 10
+
+          if span.stopped?
+            packer.write_map_header(number_of_elements_to_write + 2) # Set header with how many elements in the map
+
+            packer.write('start')
+            packer.write(time_nano(span.start_time))
+
+            packer.write('duration')
+            packer.write(duration_nano(span.duration))
+          else
+            packer.write_map_header(number_of_elements_to_write) # Set header with how many elements in the map
+          end
+
+          # DEV: We use strings as keys here, instead of symbols, as
+          # DEV: MessagePack will ultimately convert them to strings.
+          # DEV: By providing strings directly, we skip this indirection operation.
+          packer.write('span_id')
+          packer.write(span.id)
+          packer.write('parent_id')
+          packer.write(span.parent_id)
+          packer.write('trace_id')
+          packer.write(@trace_id)
+          packer.write('name')
+          packer.write(span.name)
+          packer.write('service')
+          packer.write(span.service)
+          packer.write('resource')
+          packer.write(span.resource)
+          packer.write('type')
+          packer.write(span.type)
+          packer.write('meta')
+          packer.write(span.meta)
+          packer.write('metrics')
+          packer.write(span.metrics)
+          packer.write('error')
+          packer.write(span.status)
+          packer
+        end
+        # rubocop:enable Metrics/AbcSize
+
+        # JSON serializer interface.
+        # Used by older version of the transport.
+        def to_json(*args)
+          to_hash.to_json(*args)
+        end
+
+        # Used for serialization
+        # @return [Integer] in nanoseconds since Epoch
+        def time_nano(time)
+          time.to_i * 1000000000 + time.nsec
+        end
+
+        def to_hash
+          span.to_hash.merge(trace_id: @trace_id)
+        end
+
+        # Used for serialization
+        # @return [Integer] in nanoseconds since Epoch
+        def duration_nano(duration)
+          (duration * 1e9).to_i
+        end
+      end
+    end
+  end
+end

data/lib/datadog/tracing/transport/statistics.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+require_relative '../../core/metrics/metric'
+require_relative '../../core/diagnostics/health'
+
+module Datadog
+  module Tracing
+    module Transport
+      # Tracks statistics for transports
+      module Statistics
+        def stats
+          @stats ||= Counts.new
+        end
+
+        def update_stats_from_response!(response)
+          if response.ok?
+            stats.success += 1
+            stats.consecutive_errors = 0
+          else
+            stats.client_error += 1 if response.client_error?
+            stats.server_error += 1 if response.server_error?
+            stats.internal_error += 1 if response.internal_error?
+            stats.consecutive_errors += 1
+          end
+
+          # Send health metrics
+          Datadog.health_metrics.send_metrics(
+            metrics_for_response(response).values
+          )
+        end
+
+        def metrics_for_response(response)
+          {}.tap do |metrics|
+            metrics[:api_errors] = Core::Metrics::Metric.new(:api_errors, nil, 1) if response.internal_error?
+            metrics[:api_responses] = Core::Metrics::Metric.new(:api_responses, nil, 1) unless response.internal_error?
+          end
+        end
+
+        def update_stats_from_exception!(exception)
+          stats.internal_error += 1
+          stats.consecutive_errors += 1
+
+          # Send health metrics
+          Datadog.health_metrics.send_metrics(
+            metrics_for_exception(exception).values
+          )
+        end
+
+        def metrics_for_exception(_exception)
+          { api_errors: Core::Metrics::Metric.new(:api_errors, nil, 1) }
+        end
+
+        # Stat counts
+        class Counts
+          attr_accessor \
+            :success,
+            :client_error,
+            :server_error,
+            :internal_error,
+            :consecutive_errors
+
+          def initialize
+            reset!
+          end
+
+          def reset!
+            @success = 0
+            @client_error = 0
+            @server_error = 0
+            @internal_error = 0
+            @consecutive_errors = 0
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/tracing/transport/trace_formatter.rb

@@ -0,0 +1,209 @@
+# frozen_string_literal: true
+
+require_relative '../../core/environment/identity'
+require_relative '../../core/environment/socket'
+require_relative '../../core/runtime/ext'
+require_relative '../metadata/ext'
+require_relative '../trace_segment'
+
+module Datadog
+  module Tracing
+    module Transport
+      # Prepares traces for transport
+      class TraceFormatter
+        attr_reader \
+          :root_span,
+          :trace
+
+        def self.format!(trace)
+          new(trace).format!
+        end
+
+        def initialize(trace)
+          @trace = trace
+          @root_span = find_root_span(trace)
+        end
+
+        # Modifies a trace so suitable for transport
+        def format!
+          return unless trace
+          return trace unless root_span
+
+          # Because the trace API does not support
+          # trace metadata, we must put our trace
+          # metadata on the root span. This "root span"
+          # is needed by the agent/API to ingest the trace.
+
+          # Apply generic trace tags. Any more specific value will be overridden
+          # by the subsequent calls below.
+          set_trace_tags!
+
+          set_resource!
+
+          tag_agent_sample_rate!
+          tag_hostname!
+          tag_lang!
+          tag_origin!
+          tag_process_id!
+          tag_rule_sample_rate!
+          tag_runtime_id!
+          tag_rate_limiter_rate!
+          tag_sample_rate!
+          tag_sampling_decision_maker!
+          tag_high_order_trace_id!
+          tag_sampling_priority!
+          tag_profiling_enabled!
+
+          trace
+        end
+
+        protected
+
+        def set_resource!
+          # If the trace resource is undefined, or the root span wasn't
+          # specified, don't set this. We don't want to overwrite the
+          # resource of a span that is in the middle of the trace.
+          return if trace.resource.nil? || partial?
+
+          root_span.resource = trace.resource
+        end
+
+        def set_trace_tags!
+          # If the root span wasn't specified, don't set this. We don't want to
+          # misset or overwrite the tags of a span that is in the middle of the
+          # trace.
+          return if partial?
+
+          root_span.set_tags(trace.send(:meta))
+          root_span.set_tags(trace.send(:metrics))
+        end
+
+        def tag_agent_sample_rate!
+          return unless trace.agent_sample_rate
+
+          root_span.set_tag(
+            Tracing::Metadata::Ext::Sampling::TAG_AGENT_RATE,
+            trace.agent_sample_rate
+          )
+        end
+
+        def tag_hostname!
+          return unless trace.hostname
+
+          root_span.set_tag(
+            Tracing::Metadata::Ext::NET::TAG_HOSTNAME,
+            trace.hostname
+          )
+        end
+
+        def tag_lang!
+          return if trace.lang.nil?
+
+          root_span.set_tag(
+            Core::Runtime::Ext::TAG_LANG,
+            trace.lang
+          )
+        end
+
+        def tag_origin!
+          return unless trace.origin
+
+          root_span.set_tag(
+            Tracing::Metadata::Ext::Distributed::TAG_ORIGIN,
+            trace.origin
+          )
+        end
+
+        def tag_process_id!
+          return unless trace.process_id
+
+          root_span.set_tag(Core::Runtime::Ext::TAG_PROCESS_ID, trace.process_id)
+        end
+
+        def tag_rate_limiter_rate!
+          return unless trace.rate_limiter_rate
+
+          root_span.set_tag(
+            Tracing::Metadata::Ext::Sampling::TAG_RATE_LIMITER_RATE,
+            trace.rate_limiter_rate
+          )
+        end
+
+        def tag_rule_sample_rate!
+          return unless trace.rule_sample_rate
+
+          root_span.set_tag(
+            Tracing::Metadata::Ext::Sampling::TAG_RULE_SAMPLE_RATE,
+            trace.rule_sample_rate
+          )
+        end
+
+        def tag_runtime_id!
+          return unless trace.runtime_id
+
+          root_span.set_tag(
+            Core::Runtime::Ext::TAG_ID,
+            trace.runtime_id
+          )
+        end
+
+        def tag_sample_rate!
+          return unless trace.sample_rate
+
+          root_span.set_tag(
+            Tracing::Metadata::Ext::Sampling::TAG_SAMPLE_RATE,
+            trace.sample_rate
+          )
+        end
+
+        def tag_sampling_decision_maker!
+          return unless (decision = trace.sampling_decision_maker)
+
+          root_span.set_tag(Tracing::Metadata::Ext::Distributed::TAG_DECISION_MAKER, decision)
+        end
+
+        def tag_sampling_priority!
+          return unless trace.sampling_priority
+
+          root_span.set_metric(
+            Tracing::Metadata::Ext::Distributed::TAG_SAMPLING_PRIORITY,
+            trace.sampling_priority
+          )
+        end
+
+        def tag_high_order_trace_id!
+          return unless (high_order_tid = trace.high_order_tid)
+
+          root_span.set_tag(Tracing::Metadata::Ext::Distributed::TAG_TID, high_order_tid)
+        end
+
+        def tag_profiling_enabled!
+          return if trace.profiling_enabled.nil?
+
+          root_span.set_tag(
+            Tracing::Metadata::Ext::TAG_PROFILING_ENABLED, trace.profiling_enabled ? 1 : 0
+          )
+        end
+
+        private
+
+        def partial?
+          !@found_root_span
+        end
+
+        def find_root_span(trace)
+          # TODO: Should we memoize this?
+          #       Would be safe, but `spans` is mutable, so if
+          #       the root span were removed, it would be a stale reference.
+          #       Figure out a better way to deal with this.
+          root_span_id = trace.send(:root_span_id)
+          root_span = trace.spans.find { |s| s.id == root_span_id } if root_span_id
+          @found_root_span = !root_span.nil?
+
+          # when root span is not found, fall back to last span (partial flush)
+          root_span || trace.spans.last
+        end
+      end
+    end
+  end
+end

data/lib/datadog/tracing/transport/traces.rb

@@ -0,0 +1,224 @@
+# frozen_string_literal: true
+
+require_relative '../../core/chunker'
+require_relative '../../core/transport/parcel'
+require_relative '../../core/transport/request'
+require_relative 'serializable_trace'
+require_relative 'trace_formatter'
+
+module Datadog
+  module Tracing
+    module Transport
+      module Traces
+        # Data transfer object for encoded traces
+        class EncodedParcel
+          include Datadog::Core::Transport::Parcel
+
+          attr_reader :trace_count
+
+          def initialize(data, trace_count)
+            super(data)
+            @trace_count = trace_count
+          end
+
+          def count
+            data.length
+          end
+        end
+
+        # Traces request
+        class Request < Datadog::Core::Transport::Request
+        end
+
+        # Traces response
+        module Response
+          attr_reader :service_rates, :trace_count
+        end
+
+        # Traces chunker
+        class Chunker
+          # Trace agent limit payload size of 10 MiB (since agent v5.11.0):
+          # https://github.com/DataDog/datadog-agent/blob/6.14.1/pkg/trace/api/api.go#L46
+          #
+          # We set the value to a conservative 5 MiB, in case network speed is slow.
+          DEFAULT_MAX_PAYLOAD_SIZE = 5 * 1024 * 1024
+
+          attr_reader :encoder, :max_size
+
+          #
+          # Single traces larger than +max_size+ will be discarded.
+          #
+          # @param encoder [Datadog::Core::Encoding::Encoder]
+          # @param max_size [String] maximum acceptable payload size
+          def initialize(encoder, max_size: DEFAULT_MAX_PAYLOAD_SIZE)
+            @encoder = encoder
+            @max_size = max_size
+          end
+
+          # Encodes a list of traces in chunks.
+          # Before serializing, all traces are normalized. Trace nesting is not changed.
+          #
+          # @param traces [Enumerable<Trace>] list of traces
+          # @return [Enumerable[Array[Bytes,Integer]]] list of encoded chunks: each containing a byte array and
+          #   number of traces
+          def encode_in_chunks(traces)
+            encoded_traces = if traces.respond_to?(:filter_map)
+                               # DEV Supported since Ruby 2.7, saves an intermediate object creation
+                               traces.filter_map { |t| encode_one(t) }
+                             else
+                               traces.map { |t| encode_one(t) }.reject(&:nil?)
+                             end
+
+            Datadog::Core::Chunker.chunk_by_size(encoded_traces, max_size).map do |chunk|
+              [encoder.join(chunk), chunk.size]
+            end
+          end
+
+          private
+
+          def encode_one(trace)
+            encoded = Encoder.encode_trace(encoder, trace)
+
+            if encoded.size > max_size
+              # This single trace is too large, we can't flush it
+              Datadog.logger.debug { "Dropping trace. Payload too large: '#{trace.inspect}'" }
+              Datadog.health_metrics.transport_trace_too_large(1)
+
+              return nil
+            end
+
+            encoded
+          end
+        end
+
+        # Encodes traces using {Datadog::Core::Encoding::Encoder} instances.
+        module Encoder
+          module_function
+
+          def encode_trace(encoder, trace)
+            # Format the trace for transport
+            TraceFormatter.format!(trace)
+
+            # Make the trace serializable
+            serializable_trace = SerializableTrace.new(trace)
+
+            Datadog.logger.debug { "Flushing trace: #{JSON.dump(serializable_trace)}" }
+
+            # Encode the trace
+            encoder.encode(serializable_trace)
+          end
+        end
+
+        # Sends traces based on transport API configuration.
+        #
+        # This class initializes the HTTP client, breaks down large
+        # batches of traces into smaller chunks and handles
+        # API version downgrade handshake.
+        class Transport
+          attr_reader :client, :apis, :default_api, :current_api_id
+
+          def initialize(apis, default_api)
+            @apis = apis
+            @default_api = default_api
+
+            change_api!(default_api)
+          end
+
+          def send_traces(traces)
+            encoder = current_api.encoder
+            chunker = Datadog::Tracing::Transport::Traces::Chunker.new(encoder)
+
+            responses = chunker.encode_in_chunks(traces.lazy).map do |encoded_traces, trace_count|
+              request = Request.new(EncodedParcel.new(encoded_traces, trace_count))
+
+              client.send_traces_payload(request).tap do |response|
+                if downgrade?(response)
+                  downgrade!
+                  return send_traces(traces)
+                end
+              end
+            end
+
+            # Force resolution of lazy enumerator.
+            #
+            # The "correct" method to call here would be `#force`,
+            # as this method was created to force the eager loading
+            # of a lazy enumerator.
+            #
+            # Unfortunately, JRuby < 9.2.9.0 erroneously eagerly loads
+            # the lazy Enumerator during intermediate steps.
+            # This forces us to use `#to_a`, as this method works for both
+            # lazy and regular Enumerators.
+            # Using `#to_a` can mask the fact that we expect a lazy
+            # Enumerator.
+            responses = responses.to_a
+
+            Datadog.health_metrics.transport_chunked(responses.size)
+
+            responses
+          end
+
+          def stats
+            @client.stats
+          end
+
+          def current_api
+            apis[@current_api_id]
+          end
+
+          private
+
+          def downgrade?(response)
+            return false unless apis.fallbacks.key?(@current_api_id)
+
+            response.not_found? || response.unsupported?
+          end
+
+          def downgrade!
+            downgrade_api_id = apis.fallbacks[@current_api_id]
+            raise NoDowngradeAvailableError, @current_api_id if downgrade_api_id.nil?
+
+            change_api!(downgrade_api_id)
+          end
+
+          def change_api!(api_id)
+            raise UnknownApiVersionError, api_id unless apis.key?(api_id)
+
+            @current_api_id = api_id
+            @client = HTTP::Client.new(current_api)
+          end
+
+          # Raised when configured with an unknown API version
+          class UnknownApiVersionError < StandardError
+            attr_reader :version
+
+            def initialize(version)
+              super
+
+              @version = version
+            end
+
+            def message
+              "No matching transport API for version #{version}!"
+            end
+          end
+
+          # Raised when configured with an unknown API version
+          class NoDowngradeAvailableError < StandardError
+            attr_reader :version
+
+            def initialize(version)
+              super
+
+              @version = version
+            end
+
+            def message
+              "No downgrade from transport API version #{version} is available!"
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/tracing/workers/trace_writer.rb

@@ -7,7 +7,7 @@ require_relative '../buffer'
 require_relative '../pipeline'
 require_relative '../event'
 
-require_relative '../../../ddtrace/transport/http'
+require_relative '../transport/http'
 
 module Datadog
   module Tracing
@@ -24,7 +24,7 @@ module Datadog
           transport_options[:agent_settings] = options[:agent_settings] if options.key?(:agent_settings)
 
           @transport = options.fetch(:transport) do
-            Transport::HTTP.default(**transport_options)
+            Datadog::Tracing::Transport::HTTP.default(**transport_options)
           end
         end
         # rubocop:enable Lint/MissingSuper
@@ -104,6 +104,8 @@ module Datadog
           # Workers::Queue settings
           @buffer_size = options.fetch(:buffer_size, DEFAULT_BUFFER_MAX_SIZE)
           self.buffer = TraceBuffer.new(@buffer_size)
+
+          @shutdown_timeout = options.fetch(:shutdown_timeout, Core::Workers::Polling::DEFAULT_SHUTDOWN_TIMEOUT)
         end
 
         # NOTE: #perform is wrapped by other modules:
@@ -119,7 +121,7 @@ module Datadog
           nil
         end
 
-        def stop(*args)
+        def stop(force_stop = false, timeout = @shutdown_timeout)
           buffer.close if running?
           super
         end

data/lib/datadog/tracing/workers.rb

@@ -14,7 +14,7 @@ module Datadog
         DEFAULT_TIMEOUT = 5
         BACK_OFF_RATIO = 1.2
         BACK_OFF_MAX = 5
-        SHUTDOWN_TIMEOUT = 1
+        DEFAULT_SHUTDOWN_TIMEOUT = 1
 
         attr_reader \
           :trace_buffer
@@ -36,6 +36,7 @@ module Datadog
 
           # Threading
           @shutdown = ConditionVariable.new
+          @shutdown_timeout = options.fetch(:shutdown_timeout, DEFAULT_SHUTDOWN_TIMEOUT)
           @mutex = Mutex.new
           @worker = nil
           @run = false
@@ -89,7 +90,7 @@ module Datadog
 
         # Block until executor shutdown is complete or until timeout seconds have passed.
         def join
-          @worker.join(SHUTDOWN_TIMEOUT)
+          @worker.join(@shutdown_timeout)
         end
 
         # Enqueue an item in the trace internal buffer. This operation is thread-safe