cw-datadog 2.23.0.2

data/lib/datadog/tracing/transport/traces.rb

@@ -0,0 +1,258 @@
+# frozen_string_literal: true
+
+require_relative '../../core/chunker'
+require_relative '../../core/transport/parcel'
+require_relative '../../core/transport/request'
+require_relative 'serializable_trace'
+require_relative 'trace_formatter'
+
+module Datadog
+  module Tracing
+    module Transport
+      module Traces
+        # Data transfer object for encoded traces
+        class EncodedParcel
+          include Datadog::Core::Transport::Parcel
+
+          attr_reader :trace_count
+
+          def initialize(data, trace_count)
+            super(data)
+            @trace_count = trace_count
+          end
+
+          def count
+            data.length
+          end
+        end
+
+        # Traces request
+        class Request < Datadog::Core::Transport::Request
+        end
+
+        # Traces response
+        module Response
+          attr_reader :service_rates, :trace_count
+        end
+
+        # Traces chunker
+        class Chunker
+          # Trace agent limit payload size of 10 MiB (since agent v5.11.0):
+          # https://github.com/DataDog/datadog-agent/blob/6.14.1/pkg/trace/api/api.go#L46
+          #
+          # We set the value to a conservative 5 MiB, in case network speed is slow.
+          DEFAULT_MAX_PAYLOAD_SIZE = 5 * 1024 * 1024
+
+          attr_reader :encoder, :max_size, :logger
+
+          #
+          # Single traces larger than +max_size+ will be discarded.
+          #
+          # @param encoder [Datadog::Core::Encoding::Encoder]
+          # @param logger [Datadog::Core::Logger]
+          # @param max_size [String] maximum acceptable payload size
+          def initialize(encoder, logger:, native_events_supported:, max_size: DEFAULT_MAX_PAYLOAD_SIZE)
+            @encoder = encoder
+            @logger = logger
+            @native_events_supported = native_events_supported
+            @max_size = max_size
+          end
+
+          # Encodes a list of traces in chunks.
+          # Before serializing, all traces are normalized. Trace nesting is not changed.
+          #
+          # @param traces [Enumerable<Trace>] list of traces
+          # @return [Enumerable[Array[Bytes,Integer]]] list of encoded chunks: each containing a byte array and
+          #   number of traces
+          def encode_in_chunks(traces)
+            encoded_traces = if traces.respond_to?(:filter_map)
+              # DEV Supported since Ruby 2.7, saves an intermediate object creation
+              traces.filter_map { |t| encode_one(t) }
+            else
+              traces.map { |t| encode_one(t) }.reject(&:nil?)
+            end
+
+            Datadog::Core::Chunker.chunk_by_size(encoded_traces, max_size).map do |chunk|
+              [encoder.join(chunk), chunk.size]
+            end
+          end
+
+          private
+
+          def encode_one(trace)
+            encoded = Encoder.encode_trace(
+              encoder,
+              trace,
+              logger: logger,
+              native_events_supported: @native_events_supported
+            )
+
+            if encoded.size > max_size
+              # This single trace is too large, we can't flush it
+              logger.debug { "Dropping trace. Payload too large: '#{trace.inspect}'" }
+              Datadog.health_metrics.transport_trace_too_large(1)
+
+              return nil
+            end
+
+            encoded
+          end
+        end
+
+        # Encodes traces using {Datadog::Core::Encoding::Encoder} instances.
+        module Encoder
+          module_function
+
+          def encode_trace(encoder, trace, logger:, native_events_supported:)
+            # Format the trace for transport
+            TraceFormatter.format!(trace)
+
+            # Make the trace serializable
+            serializable_trace = SerializableTrace.new(trace, native_events_supported: native_events_supported)
+
+            # Encode the trace
+            encoder.encode(serializable_trace).tap do |encoded|
+              # Print the actual serialized trace, since the encoder can change make non-trivial changes
+              logger.debug { "Flushing trace: #{encoder.decode(encoded)}" }
+            end
+          end
+        end
+
+        # Sends traces based on transport API configuration.
+        #
+        # This class initializes the HTTP client, breaks down large
+        # batches of traces into smaller chunks and handles
+        # API version downgrade handshake.
+        class Transport
+          attr_reader :client, :apis, :default_api, :current_api_id, :logger
+
+          def initialize(apis, default_api, logger: Datadog.logger)
+            @apis = apis
+            @default_api = default_api
+            @logger = logger
+
+            change_api!(default_api)
+          end
+
+          def send_traces(traces)
+            encoder = current_api.encoder
+            chunker = Datadog::Tracing::Transport::Traces::Chunker.new(
+              encoder,
+              logger: logger,
+              native_events_supported: native_events_supported?
+            )
+
+            responses = chunker.encode_in_chunks(traces.lazy).map do |encoded_traces, trace_count|
+              request = Request.new(EncodedParcel.new(encoded_traces, trace_count))
+
+              client.send_traces_payload(request).tap do |response|
+                if downgrade?(response)
+                  downgrade!
+                  return send_traces(traces)
+                end
+              end
+            end
+
+            # Force resolution of lazy enumerator.
+            #
+            # The "correct" method to call here would be `#force`,
+            # as this method was created to force the eager loading
+            # of a lazy enumerator.
+            #
+            # Unfortunately, JRuby < 9.2.9.0 erroneously eagerly loads
+            # the lazy Enumerator during intermediate steps.
+            # This forces us to use `#to_a`, as this method works for both
+            # lazy and regular Enumerators.
+            # Using `#to_a` can mask the fact that we expect a lazy
+            # Enumerator.
+            responses = responses.to_a
+
+            Datadog.health_metrics.transport_chunked(responses.size)
+
+            responses
+          end
+
+          def stats
+            @client.stats
+          end
+
+          def current_api
+            apis[@current_api_id]
+          end
+
+          private
+
+          def downgrade?(response)
+            return false unless apis.fallbacks.key?(@current_api_id)
+
+            response.not_found? || response.unsupported?
+          end
+
+          def downgrade!
+            downgrade_api_id = apis.fallbacks[@current_api_id]
+            raise NoDowngradeAvailableError, @current_api_id if downgrade_api_id.nil?
+
+            change_api!(downgrade_api_id)
+          end
+
+          def change_api!(api_id)
+            raise UnknownApiVersionError, api_id unless apis.key?(api_id)
+
+            @current_api_id = api_id
+            @client = HTTP::Client.new(current_api, logger: logger)
+          end
+
+          # Queries the agent for native span events serialization support.
+          # This changes how the serialization of span events performed.
+          def native_events_supported?
+            return @native_events_supported if defined?(@native_events_supported)
+
+            # Check for an explicit override
+            option = Datadog.configuration.tracing.native_span_events
+            unless option.nil?
+              @native_events_supported = option
+              return option
+            end
+
+            # Otherwise, check for agent support, to ensure a configuration-less setup.
+            if (res = Datadog.send(:components).agent_info.fetch)
+              @native_events_supported = res.span_events == true
+            else
+              false
+            end
+          end
+
+          # Raised when configured with an unknown API version
+          class UnknownApiVersionError < StandardError
+            attr_reader :version
+
+            def initialize(version)
+              super
+
+              @version = version
+            end
+
+            def message
+              "No matching transport API for version #{version}!"
+            end
+          end
+
+          # Raised when configured with an unknown API version
+          class NoDowngradeAvailableError < StandardError
+            attr_reader :version
+
+            def initialize(version)
+              super
+
+              @version = version
+            end
+
+            def message
+              "No downgrade from transport API version #{version} is available!"
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/tracing/utils.rb

@@ -0,0 +1,99 @@
+# frozen_string_literal: true
+
+require_relative '../core/utils/forking'
+require_relative '../core/utils/time'
+
+module Datadog
+  module Tracing
+    # Utils contains low-level tracing utility functions.
+    # @public_api
+    module Utils
+      extend Datadog::Core::Utils::Forking
+
+      # The max value for a {Datadog::Tracing::Span} identifier.
+      # Span and trace identifiers should be strictly positive and strictly inferior to this limit.
+      #
+      # Limited to +2<<62-1+ positive integers, as Ruby is able to represent such numbers "inline",
+      # inside a +VALUE+ scalar, thus not requiring memory allocation.
+      #
+      # The range of IDs also has to consider portability across different languages and platforms.
+      RUBY_MAX_ID = (1 << 62) - 1
+
+      # Excludes zero from possible values
+      RUBY_ID_RANGE = (1..RUBY_MAX_ID).freeze
+
+      # While we only generate 63-bit integers due to limitations in other languages, we support
+      # parsing 64-bit integers for distributed tracing since an upstream system may generate one
+      EXTERNAL_MAX_ID = (1 << 64) - 1
+
+      # We use a custom random number generator because we want no interference
+      # with the default one. Using the default prng, we could break code that
+      # would rely on srand/rand sequences.
+
+      # Return a randomly generated integer, valid as a Span ID or Trace ID.
+      # This method is thread-safe and fork-safe.
+      def self.next_id
+        after_fork! { reset! }
+        id_rng.rand(RUBY_ID_RANGE)
+      end
+
+      def self.id_rng
+        @id_rng ||= Random.new
+      end
+
+      def self.reset!
+        @id_rng = Random.new
+      end
+
+      # Serialize values into Datadog span tags and metrics.
+      # Notably, arrays are exploded into many keys, each with
+      # a numeric suffix representing the array index, for example:
+      # `'foo' => ['a','b']` becomes `'foo.0' => 'a', 'foo.1' => 'b'`
+      def self.serialize_attribute(key, value)
+        if value.is_a?(Array)
+          value.flat_map.with_index do |v, idx|
+            serialize_attribute("#{key}.#{idx}", v)
+          end
+        elsif value.is_a?(TrueClass) || value.is_a?(FalseClass)
+          [[key, value.to_s]]
+        else
+          [[key, value]]
+        end
+      end
+
+      private_class_method :id_rng, :reset!
+
+      # The module handles bitwise operation for trace id
+      module TraceId
+        MAX = (1 << 128) - 1
+
+        module_function
+
+        # Format for generating 128 bits trace id =>
+        # - 32-bits : seconds since Epoch
+        # - 32-bits : set to zero,
+        # - 64 bits : random 64-bits
+        def next_id
+          return Utils.next_id unless Datadog.configuration.tracing.trace_id_128_bit_generation_enabled
+
+          concatenate(
+            Core::Utils::Time.now.to_i << 32,
+            Utils.next_id
+          )
+        end
+
+        def to_high_order(trace_id)
+          trace_id >> 64
+        end
+
+        def to_low_order(trace_id)
+          trace_id & 0xFFFFFFFFFFFFFFFF
+        end
+
+        def concatenate(high_order, low_order)
+          high_order << 64 | low_order
+        end
+      end
+    end
+  end
+end

data/lib/datadog/tracing/workers/trace_writer.rb

@@ -0,0 +1,199 @@
+# frozen_string_literal: true
+
+require_relative '../../core/worker'
+require_relative '../../core/workers/async'
+require_relative '../../core/workers/polling'
+require_relative '../../core/workers/queue'
+
+require_relative '../buffer'
+require_relative '../pipeline'
+require_relative '../event'
+
+require_relative '../transport/http'
+
+module Datadog
+  module Tracing
+    module Workers
+      # Writes traces to transport synchronously
+      class TraceWriter < Core::Worker
+        attr_reader \
+          :logger,
+          :transport,
+          :agent_settings
+
+        # rubocop:disable Lint/MissingSuper
+        def initialize(options = {})
+          @logger = options[:logger] || Datadog.logger
+
+          transport_options = options.fetch(:transport_options, {})
+          @agent_settings = options[:agent_settings]
+
+          @transport = options.fetch(:transport) do
+            Datadog::Tracing::Transport::HTTP.default(agent_settings: agent_settings, logger: logger, **transport_options)
+          end
+        end
+        # rubocop:enable Lint/MissingSuper
+
+        def perform(traces)
+          write_traces(traces)
+        end
+
+        def write(trace)
+          write_traces([trace])
+        end
+
+        def write_traces(traces)
+          traces = process_traces(traces)
+          flush_traces(traces)
+        rescue => e
+          logger.warn(
+            "Error while writing traces: dropped #{traces.length} items. Cause: #{e} Location: #{Array(e.backtrace).first}"
+          )
+        end
+
+        def process_traces(traces)
+          # Run traces through the processing pipeline
+          Pipeline.process!(traces)
+        end
+
+        def flush_traces(traces)
+          transport.send_traces(traces).tap do |response|
+            flush_completed.publish(response)
+          end
+        end
+
+        # TODO: Register `Datadog::Tracing::Diagnostics::EnvironmentLogger.collect_and_log!`
+        # TODO: as a flush_completed subscriber when the `TraceWriter`
+        # TODO: instantiation code is implemented.
+        def flush_completed
+          @flush_completed ||= FlushCompleted.new
+        end
+
+        # Flush completed event for worker
+        class FlushCompleted < Event
+          def initialize
+            super(:flush_completed)
+          end
+        end
+      end
+
+      # Writes traces to transport asynchronously,
+      # using a thread & buffer.
+      class AsyncTraceWriter < TraceWriter
+        include Core::Workers::Queue
+        include Core::Workers::Polling
+
+        DEFAULT_BUFFER_MAX_SIZE = 1000
+        FORK_POLICY_ASYNC = :async
+        FORK_POLICY_SYNC = :sync
+
+        attr_writer \
+          :async
+
+        def initialize(options = {})
+          # Workers::TraceWriter settings
+          super
+
+          # Workers::Polling settings
+          self.enabled = options.fetch(:enabled, true)
+
+          # Workers::Async::Thread settings
+          @async = true
+          self.fork_policy = options.fetch(:fork_policy, FORK_POLICY_ASYNC)
+
+          # Workers::IntervalLoop settings
+          self.loop_base_interval = options[:interval] if options.key?(:interval)
+          self.loop_back_off_ratio = options[:back_off_ratio] if options.key?(:back_off_ratio)
+          self.loop_back_off_max = options[:back_off_max] if options.key?(:back_off_max)
+
+          # Workers::Queue settings
+          @buffer_size = options.fetch(:buffer_size, DEFAULT_BUFFER_MAX_SIZE)
+          self.buffer = TraceBuffer.new(@buffer_size)
+
+          @shutdown_timeout = options.fetch(:shutdown_timeout, Core::Workers::Polling::DEFAULT_SHUTDOWN_TIMEOUT)
+        end
+
+        # NOTE: #perform is wrapped by other modules:
+        #       Polling --> Async --> IntervalLoop --> AsyncTraceWriter --> TraceWriter
+        #
+        # WARNING: This method breaks the Liskov Substitution Principle -- TraceWriter#perform is spec'd to return the
+        # result from the writer, whereas this method always returns nil.
+        def perform(traces)
+          super.tap do |responses|
+            loop_back_off! if responses.find(&:server_error?)
+          end
+
+          nil
+        end
+
+        def stop(force_stop = false, timeout = @shutdown_timeout)
+          buffer.close if running?
+          super
+        end
+
+        def enqueue(trace)
+          buffer.push(trace)
+        end
+
+        def dequeue
+          # Wrap results in Array because they are
+          # splatted as args against TraceWriter#perform.
+          [buffer.pop]
+        end
+
+        # Are there more traces to be processed next?
+        def work_pending?
+          !buffer.empty?
+        end
+
+        def async?
+          @async == true
+        end
+
+        def fork_policy=(policy)
+          # Translate to Workers::Async::Thread policy
+          thread_fork_policy = case policy
+          when Core::Workers::Async::Thread::FORK_POLICY_STOP
+            policy
+          when FORK_POLICY_SYNC
+            # Stop the async thread because the writer
+            # will bypass and run synchronously.
+            Core::Workers::Async::Thread::FORK_POLICY_STOP
+          else
+            Core::Workers::Async::Thread::FORK_POLICY_RESTART
+          end
+
+          # Update thread fork policy
+          super(thread_fork_policy)
+
+          # Update local policy
+          @writer_fork_policy = policy
+        end
+
+        def after_fork
+          # In multiprocess environments, forks will share the same buffer until its written to.
+          # A.K.A. copy-on-write. We don't want forks to write traces generated from another process.
+          # Instead, we reset it after the fork. (Make sure any enqueue operations happen after this.)
+          self.buffer = TraceBuffer.new(@buffer_size)
+
+          # Switch to synchronous mode if configured to do so.
+          # In some cases synchronous writing is preferred because the fork will be short lived.
+          @async = false if @writer_fork_policy == FORK_POLICY_SYNC
+        end
+
+        # WARNING: This method breaks the Liskov Substitution Principle -- TraceWriter#write is spec'd to return the
+        # result from the writer, whereas this method returns something else when running in async mode.
+        def write(trace)
+          # Start worker thread. If the process has forked, it will trigger #after_fork to
+          # reconfigure the worker accordingly.
+          # NOTE: It's important we do this before queuing or it will drop the current trace,
+          #       because #after_fork resets the buffer.
+          perform
+
+          # Queue the trace if running asynchronously, otherwise short-circuit and write it directly.
+          async? ? enqueue(trace) : write_traces([trace])
+        end
+      end
+    end
+  end
+end

data/lib/datadog/tracing/workers.rb

@@ -0,0 +1,126 @@
+# frozen_string_literal: true
+
+require_relative 'buffer'
+require_relative 'pipeline'
+
+module Datadog
+  module Tracing
+    module Workers
+      # Asynchronous worker that executes a +Send()+ operation after given
+      # seconds. Under the hood, it uses +Concurrent::TimerTask+ so that the thread
+      # will perform a task at regular intervals. The thread can be stopped
+      # with the +stop()+ method and can start with the +start()+ method.
+      class AsyncTransport
+        DEFAULT_BUFFER_MAX_SIZE = 1000
+        DEFAULT_FLUSH_INTERVAL = 1
+        DEFAULT_TIMEOUT = 5
+        BACK_OFF_RATIO = 1.2
+        BACK_OFF_MAX = 5
+        DEFAULT_SHUTDOWN_TIMEOUT = 1
+
+        attr_reader :trace_buffer, :logger
+
+        def initialize(options = {})
+          @transport = options[:transport]
+
+          # Callbacks
+          @trace_task = options[:on_trace]
+
+          # Intervals
+          interval = options.fetch(:interval, DEFAULT_FLUSH_INTERVAL)
+          @flush_interval = interval
+          @back_off = interval
+
+          # Buffers
+          buffer_size = options.fetch(:buffer_size, DEFAULT_BUFFER_MAX_SIZE)
+          @trace_buffer = TraceBuffer.new(buffer_size)
+
+          # Threading
+          @shutdown = ConditionVariable.new
+          @shutdown_timeout = options.fetch(:shutdown_timeout, DEFAULT_SHUTDOWN_TIMEOUT)
+          @mutex = Mutex.new
+          @worker = nil
+          @run = false
+
+          @logger = options.fetch(:logger)
+        end
+
+        # Callback function that process traces and executes the +send_traces()+ method.
+        def callback_traces
+          return true if @trace_buffer.empty?
+
+          begin
+            traces = @trace_buffer.pop
+            traces = Pipeline.process!(traces)
+            @trace_task.call(traces, @transport) unless @trace_task.nil? || traces.empty?
+          rescue => e
+            # ensures that the thread will not die because of an exception.
+            # TODO[manu]: findout the reason and reschedule the send if it's not
+            # a fatal exception
+            logger.warn(
+              "Error during traces flush: dropped #{traces.length} items. Cause: #{e} Location: #{Array(e.backtrace).first}"
+            )
+          end
+        end
+
+        # Start the timer execution.
+        def start
+          @mutex.synchronize do
+            return if @run
+
+            @run = true
+            logger.debug { "Starting thread for: #{self}" }
+            @worker = Thread.new { perform }
+            @worker.name = self.class.name
+            @worker.thread_variable_set(:fork_safe, true)
+
+            nil
+          end
+        end
+
+        # Closes all available queues and waits for the trace buffer to flush
+        def stop
+          @mutex.synchronize do
+            return unless @run
+
+            @trace_buffer.close
+            @run = false
+            @shutdown.signal
+          end
+
+          join
+          true
+        end
+
+        # Block until executor shutdown is complete or until timeout seconds have passed.
+        def join
+          @worker.join(@shutdown_timeout)
+        end
+
+        # Enqueue an item in the trace internal buffer. This operation is thread-safe
+        # because uses the +TraceBuffer+ data structure.
+        def enqueue_trace(trace)
+          return unless trace && !trace.empty?
+
+          @trace_buffer.push(trace)
+        end
+
+        private
+
+        alias_method :flush_data, :callback_traces
+
+        def perform
+          loop do
+            @back_off = flush_data ? @flush_interval : [@back_off * BACK_OFF_RATIO, BACK_OFF_MAX].min
+
+            @mutex.synchronize do
+              return if !@run && @trace_buffer.empty?
+
+              @shutdown.wait(@mutex, @back_off) if @run # do not wait when shutting down
+            end
+          end
+        end
+      end
+    end
+  end
+end