ddtrace 0.54.0 → 1.1.0

data/lib/ddtrace/workers/trace_writer.rb

@@ -1,200 +0,0 @@
-# typed: false
-require 'ddtrace/logger'
-require 'ddtrace/transport/http'
-
-require 'ddtrace/event'
-require 'ddtrace/worker'
-require 'ddtrace/workers/polling'
-require 'ddtrace/workers/queue'
-
-module Datadog
-  module Workers
-    # Writes traces to transport synchronously
-    class TraceWriter < Worker
-      attr_reader \
-        :transport
-
-      def initialize(options = {})
-        transport_options = options.fetch(:transport_options, {})
-
-        transport_options[:agent_settings] = options[:agent_settings] if options.key?(:agent_settings)
-
-        @transport = options.fetch(:transport) do
-          Transport::HTTP.default(**transport_options)
-        end
-      end
-
-      def perform(traces)
-        write_traces(traces)
-      end
-
-      def write(trace)
-        write_traces([trace])
-      end
-
-      def write_traces(traces)
-        traces = process_traces(traces)
-        flush_traces(traces)
-      rescue StandardError => e
-        Datadog.logger.error(
-          "Error while writing traces: dropped #{traces.length} items. Cause: #{e} Location: #{Array(e.backtrace).first}"
-        )
-      end
-
-      def process_traces(traces)
-        # Run traces through the processing pipeline
-        traces = Pipeline.process!(traces)
-
-        # Inject hostname if configured to do so
-        inject_hostname!(traces) if Datadog.configuration.report_hostname
-
-        traces
-      end
-
-      def flush_traces(traces)
-        transport.send_traces(traces).tap do |response|
-          flush_completed.publish(response)
-        end
-      end
-
-      def inject_hostname!(traces)
-        traces.each do |trace|
-          next if trace.first.nil?
-
-          hostname = Datadog::Core::Environment::Socket.hostname
-          trace.first.set_tag(Ext::NET::TAG_HOSTNAME, hostname) unless hostname.nil? || hostname.empty?
-        end
-      end
-
-      # TODO: Register `Datadog::Diagnostics::EnvironmentLogger.log!`
-      # TODO: as a flush_completed subscriber when the `TraceWriter`
-      # TODO: instantiation code is implemented.
-      def flush_completed
-        @flush_completed ||= FlushCompleted.new
-      end
-
-      # Flush completed event for worker
-      class FlushCompleted < Event
-        def initialize
-          super(:flush_completed)
-        end
-      end
-    end
-
-    # Writes traces to transport asynchronously,
-    # using a thread & buffer.
-    class AsyncTraceWriter < TraceWriter
-      include Workers::Queue
-      include Workers::Polling
-
-      DEFAULT_BUFFER_MAX_SIZE = 1000
-      FORK_POLICY_ASYNC = :async
-      FORK_POLICY_SYNC = :sync
-
-      attr_writer \
-        :async
-
-      def initialize(options = {})
-        # Workers::TraceWriter settings
-        super
-
-        # Workers::Polling settings
-        self.enabled = options.fetch(:enabled, true)
-
-        # Workers::Async::Thread settings
-        @async = true
-        self.fork_policy = options.fetch(:fork_policy, FORK_POLICY_ASYNC)
-
-        # Workers::IntervalLoop settings
-        self.loop_base_interval = options[:interval] if options.key?(:interval)
-        self.loop_back_off_ratio = options[:back_off_ratio] if options.key?(:back_off_ratio)
-        self.loop_back_off_max = options[:back_off_max] if options.key?(:back_off_max)
-
-        # Workers::Queue settings
-        @buffer_size = options.fetch(:buffer_size, DEFAULT_BUFFER_MAX_SIZE)
-        self.buffer = TraceBuffer.new(@buffer_size)
-      end
-
-      # NOTE: #perform is wrapped by other modules:
-      #       Polling --> Async --> IntervalLoop --> AsyncTraceWriter --> TraceWriter
-      #
-      # WARNING: This method breaks the Liskov Substitution Principle -- TraceWriter#perform is spec'd to return the
-      # result from the writer, whereas this method always returns nil.
-      def perform(traces)
-        super(traces).tap do |responses|
-          loop_back_off! if responses.find(&:server_error?)
-        end
-
-        nil
-      end
-
-      def stop(*args)
-        buffer.close if running?
-        super
-      end
-
-      def enqueue(trace)
-        buffer.push(trace)
-      end
-
-      def dequeue
-        # Wrap results in Array because they are
-        # splatted as args against TraceWriter#perform.
-        [buffer.pop]
-      end
-
-      # Are there more traces to be processed next?
-      def work_pending?
-        !buffer.empty?
-      end
-
-      def async?
-        @async == true
-      end
-
-      def fork_policy=(policy)
-        # Translate to Workers::Async::Thread policy
-        thread_fork_policy = case policy
-                             when Workers::Async::Thread::FORK_POLICY_STOP
-                               policy
-                             when FORK_POLICY_SYNC
-                               # Stop the async thread because the writer
-                               # will bypass and run synchronously.
-                               Workers::Async::Thread::FORK_POLICY_STOP
-                             else
-                               Workers::Async::Thread::FORK_POLICY_RESTART
-                             end
-
-        # Update thread fork policy
-        super(thread_fork_policy)
-
-        # Update local policy
-        @writer_fork_policy = policy
-      end
-
-      def after_fork
-        # In multiprocess environments, forks will share the same buffer until its written to.
-        # A.K.A. copy-on-write. We don't want forks to write traces generated from another process.
-        # Instead, we reset it after the fork. (Make sure any enqueue operations happen after this.)
-        self.buffer = TraceBuffer.new(@buffer_size)
-
-        # Switch to synchronous mode if configured to do so.
-        # In some cases synchronous writing is preferred because the fork will be short lived.
-        @async = false if @writer_fork_policy == FORK_POLICY_SYNC
-      end
-
-      # WARNING: This method breaks the Liskov Substitution Principle -- TraceWriter#write is spec'd to return the
-      # result from the writer, whereas this method returns something else when running in async mode.
-      def write(trace)
-        # Start worker thread. If the process has forked, it will trigger #after_fork to
-        # reconfigure the worker accordingly.
-        # NOTE: It's important we do this before queuing or it will drop the current trace,
-        #       because #after_fork resets the buffer.
-        perform
-
-        # Queue the trace if running asynchronously, otherwise short-circuit and write it directly.
-        async? ? enqueue(trace) : write_traces([trace])
-      end
-    end
-  end
-end

data/lib/ddtrace/workers.rb

@@ -1,123 +0,0 @@
-# typed: true
-require 'time'
-
-require 'ddtrace/workers/trace_writer'
-require 'ddtrace/workers/runtime_metrics'
-
-require 'ddtrace/buffer'
-require 'ddtrace/runtime/metrics'
-
-module Datadog
-  module Workers
-    # Asynchronous worker that executes a +Send()+ operation after given
-    # seconds. Under the hood, it uses +Concurrent::TimerTask+ so that the thread
-    # will perform a task at regular intervals. The thread can be stopped
-    # with the +stop()+ method and can start with the +start()+ method.
-    class AsyncTransport
-      DEFAULT_BUFFER_MAX_SIZE = 1000
-      DEFAULT_FLUSH_INTERVAL = 1
-      DEFAULT_TIMEOUT = 5
-      BACK_OFF_RATIO = 1.2
-      BACK_OFF_MAX = 5
-      SHUTDOWN_TIMEOUT = 1
-
-      attr_reader \
-        :trace_buffer
-
-      def initialize(options = {})
-        @transport = options[:transport]
-
-        # Callbacks
-        @trace_task = options[:on_trace]
-
-        # Intervals
-        interval = options.fetch(:interval, DEFAULT_FLUSH_INTERVAL)
-        @flush_interval = interval
-        @back_off = interval
-
-        # Buffers
-        buffer_size = options.fetch(:buffer_size, DEFAULT_BUFFER_MAX_SIZE)
-        @trace_buffer = TraceBuffer.new(buffer_size)
-
-        # Threading
-        @shutdown = ConditionVariable.new
-        @mutex = Mutex.new
-        @worker = nil
-        @run = false
-      end
-
-      # Callback function that process traces and executes the +send_traces()+ method.
-      def callback_traces
-        return true if @trace_buffer.empty?
-
-        begin
-          traces = @trace_buffer.pop
-          traces = Pipeline.process!(traces)
-          @trace_task.call(traces, @transport) unless @trace_task.nil?
-        rescue StandardError => e
-          # ensures that the thread will not die because of an exception.
-          # TODO[manu]: findout the reason and reschedule the send if it's not
-          # a fatal exception
-          Datadog.logger.error(
-            "Error during traces flush: dropped #{traces.length} items. Cause: #{e} Location: #{Array(e.backtrace).first}"
-          )
-        end
-      end
-
-      # Start the timer execution.
-      def start
-        @mutex.synchronize do
-          return if @run
-
-          @run = true
-          Datadog.logger.debug { "Starting thread for: #{self}" }
-          @worker = Thread.new { perform }
-          @worker.name = self.class.name unless Gem::Version.new(RUBY_VERSION) < Gem::Version.new('2.3')
-
-          nil
-        end
-      end
-
-      # Closes all available queues and waits for the trace buffer to flush
-      def stop
-        @mutex.synchronize do
-          return unless @run
-
-          @trace_buffer.close
-          @run = false
-          @shutdown.signal
-        end
-
-        join
-        true
-      end
-
-      # Block until executor shutdown is complete or until timeout seconds have passed.
-      def join
-        @worker.join(SHUTDOWN_TIMEOUT)
-      end
-
-      # Enqueue an item in the trace internal buffer. This operation is thread-safe
-      # because uses the +TraceBuffer+ data structure.
-      def enqueue_trace(trace)
-        @trace_buffer.push(trace)
-      end
-
-      private
-
-      alias flush_data callback_traces
-
-      def perform
-        loop do
-          @back_off = flush_data ? @flush_interval : [@back_off * BACK_OFF_RATIO, BACK_OFF_MAX].min
-
-          @mutex.synchronize do
-            return if !@run && @trace_buffer.empty?
-
-            @shutdown.wait(@mutex, @back_off) if @run # do not wait when shutting down
-          end
-        end
-      end
-    end
-  end
-end

data/lib/ddtrace/writer.rb

@@ -1,200 +0,0 @@
-# typed: true
-require 'json'
-
-require 'ddtrace/ext/net'
-require 'datadog/core/environment/socket'
-
-require 'ddtrace/configuration/agent_settings_resolver'
-require 'ddtrace/transport/http'
-require 'ddtrace/transport/io'
-require 'ddtrace/encoding'
-require 'ddtrace/workers'
-require 'ddtrace/diagnostics/environment_logger'
-require 'ddtrace/utils/only_once'
-
-module Datadog
-  # Processor that sends traces and metadata to the agent
-  class Writer
-    DEPRECATION_WARN_ONLY_ONCE = Datadog::Utils::OnlyOnce.new
-
-    attr_reader \
-      :priority_sampler,
-      :transport,
-      :worker
-
-    def initialize(options = {})
-      # writer and transport parameters
-      @buff_size = options.fetch(:buffer_size, Workers::AsyncTransport::DEFAULT_BUFFER_MAX_SIZE)
-      @flush_interval = options.fetch(:flush_interval, Workers::AsyncTransport::DEFAULT_FLUSH_INTERVAL)
-      transport_options = options.fetch(:transport_options, {})
-
-      transport_options[:agent_settings] = options[:agent_settings] if options.key?(:agent_settings)
-
-      # priority sampling
-      if options[:priority_sampler]
-        @priority_sampler = options[:priority_sampler]
-        transport_options[:api_version] ||= Transport::HTTP::API::V4
-      end
-
-      # transport and buffers
-      @transport = options.fetch(:transport) do
-        Transport::HTTP.default(**transport_options)
-      end
-
-      # handles the thread creation after an eventual fork
-      @mutex_after_fork = Mutex.new
-      @pid = nil
-
-      @traces_flushed = 0
-
-      # one worker for traces
-      @worker = nil
-
-      # Once stopped, this writer instance cannot be restarted.
-      # This allow for graceful shutdown, while preventing
-      # the host application from inadvertently start new
-      # threads during shutdown.
-      @stopped = false
-    end
-
-    def start
-      @mutex_after_fork.synchronize do
-        return false if @stopped
-
-        pid = Process.pid
-        return if @worker && pid == @pid
-
-        @pid = pid
-
-        start_worker
-        true
-      end
-    end
-
-    # spawns a worker for spans; they share the same transport which is thread-safe
-    def start_worker
-      @trace_handler = ->(items, transport) { send_spans(items, transport) }
-      @worker = Datadog::Workers::AsyncTransport.new(
-        transport: @transport,
-        buffer_size: @buff_size,
-        on_trace: @trace_handler,
-        interval: @flush_interval
-      )
-
-      @worker.start
-    end
-
-    # Gracefully shuts down this writer.
-    #
-    # Once stopped methods calls won't fail, but
-    # no internal work will be performed.
-    #
-    # It is not possible to restart a stopped writer instance.
-    def stop
-      @mutex_after_fork.synchronize { stop_worker }
-    end
-
-    def stop_worker
-      @stopped = true
-
-      return if @worker.nil?
-
-      @worker.stop
-      @worker = nil
-
-      true
-    end
-
-    private :start_worker, :stop_worker
-
-    # flush spans to the trace-agent, handles spans only
-    def send_spans(traces, transport)
-      return true if traces.empty?
-
-      # Inject hostname if configured to do so
-      inject_hostname!(traces) if Datadog.configuration.report_hostname
-
-      # Send traces and get responses
-      responses = transport.send_traces(traces)
-
-      # Tally up successful flushes
-      responses.reject { |x| x.internal_error? || x.server_error? }.each do |response|
-        @traces_flushed += response.trace_count
-      end
-
-      # Update priority sampler
-      update_priority_sampler(responses.last)
-
-      record_environment_information!(responses)
-
-      # Return if server error occurred.
-      !responses.find(&:server_error?)
-    end
-
-    # enqueue the trace for submission to the API
-    def write(trace, services = nil)
-      unless services.nil?
-        DEPRECATION_WARN_ONLY_ONCE.run do
-          Datadog.logger.warn(%(
-            write: Writing services has been deprecated and no longer need to be provided.
-            write(traces, services) can be updated to write(traces)
-          ))
-        end
-      end
-
-      # In multiprocess environments, the main process initializes the +Writer+ instance and if
-      # the process forks (i.e. a web server like Unicorn or Puma with multiple workers) the new
-      # processes will share the same +Writer+ until the first write (COW). Because of that,
-      # each process owns a different copy of the +@buffer+ after each write and so the
-      # +AsyncTransport+ will not send data to the trace agent.
-      #
-      # This check ensures that if a process doesn't own the current +Writer+, async workers
-      # will be initialized again (but only once for each process).
-      start if @worker.nil? || @pid != Process.pid
-
-      # TODO: Remove this, and have the tracer pump traces directly to runtime metrics
-      #       instead of working through the trace writer.
-      # Associate root span with runtime metrics
-      if Datadog.configuration.runtime_metrics.enabled && !trace.empty?
-        Datadog.runtime_metrics.associate_with_span(trace.first)
-      end
-
-      worker_local = @worker
-
-      if worker_local
-        worker_local.enqueue_trace(trace)
-      elsif !@stopped
-        Datadog.logger.debug('Writer either failed to start or was stopped before #write could complete')
-      end
-    end
-
-    # stats returns a dictionary of stats about the writer.
-    def stats
-      {
-        traces_flushed: @traces_flushed,
-        transport: @transport.stats
-      }
-    end
-
-    private
-
-    def inject_hostname!(traces)
-      traces.each do |trace|
-        next if trace.first.nil?
-
-        hostname = Datadog::Core::Environment::Socket.hostname
-        trace.first.set_tag(Ext::NET::TAG_HOSTNAME, hostname) unless hostname.nil? || hostname.empty?
-      end
-    end
-
-    def update_priority_sampler(response)
-      return unless response && !response.internal_error? && priority_sampler && response.service_rates
-
-      priority_sampler.update(response.service_rates)
-    end
-
-    def record_environment_information!(responses)
-      Diagnostics::EnvironmentLogger.log!(responses)
-    end
-  end
-end