ddtrace 0.13.0 → 0.47.0

data/lib/ddtrace/workers/async.rb

@@ -0,0 +1,173 @@
+require 'ddtrace/logger'
+
+module Datadog
+  module Workers
+    module Async
+      # Adds threading behavior to workers
+      # to run tasks asynchronously.
+      # rubocop:disable Metrics/ModuleLength
+      module Thread
+        FORK_POLICY_STOP = :stop
+        FORK_POLICY_RESTART = :restart
+        SHUTDOWN_TIMEOUT = 1
+
+        def self.included(base)
+          base.send(:prepend, PrependedMethods)
+        end
+
+        # Methods that must be prepended
+        module PrependedMethods
+          def perform(*args)
+            start_async { self.result = super(*args) } unless started?
+          end
+        end
+
+        attr_reader \
+          :error,
+          :result
+
+        attr_writer \
+          :fork_policy
+
+        def join(timeout = nil)
+          return true unless running?
+
+          !worker.join(timeout).nil?
+        end
+
+        def terminate
+          return false unless running?
+
+          @run_async = false
+          worker.terminate
+          true
+        end
+
+        def run_async?
+          return false unless instance_variable_defined?(:@run_async)
+
+          @run_async == true
+        end
+
+        def started?
+          !(worker.nil? || forked?)
+        end
+
+        def running?
+          !worker.nil? && worker.alive?
+        end
+
+        def error?
+          return false unless instance_variable_defined?(:@error)
+
+          !@error.nil?
+        end
+
+        def completed?
+          !worker.nil? && worker.status == false && !error?
+        end
+
+        def failed?
+          !worker.nil? && worker.status.nil?
+        end
+
+        def forked?
+          !pid.nil? && pid != Process.pid
+        end
+
+        def fork_policy
+          @fork_policy ||= FORK_POLICY_STOP
+        end
+
+        protected
+
+        attr_writer \
+          :result
+
+        def mutex
+          @mutex ||= Mutex.new
+        end
+
+        def after_fork
+          # Do nothing by default
+        end
+
+        private
+
+        attr_reader \
+          :pid
+
+        def mutex_after_fork
+          @mutex_after_fork ||= Mutex.new
+        end
+
+        def worker
+          @worker ||= nil
+        end
+
+        def start_async(&block)
+          mutex.synchronize do
+            return if running?
+
+            if forked?
+              case fork_policy
+              when FORK_POLICY_STOP
+                stop_fork
+              when FORK_POLICY_RESTART
+                restart_after_fork(&block)
+              end
+            elsif !run_async?
+              start_worker(&block)
+            end
+          end
+        end
+
+        def start_worker
+          @run_async = true
+          @pid = Process.pid
+          @error = nil
+          Datadog.logger.debug("Starting thread in the process: #{Process.pid} for: #{self}")
+
+          @worker = ::Thread.new do
+            begin
+              yield
+            # rubocop:disable Lint/RescueException
+            rescue Exception => e
+              @error = e
+              Datadog.logger.debug("Worker thread error. Cause #{e.message} Location: #{e.backtrace.first}")
+              raise
+            end
+          end
+          @worker.name = self.class.name unless Gem::Version.new(RUBY_VERSION) < Gem::Version.new('2.3')
+
+          nil
+        end
+
+        def stop_fork
+          mutex_after_fork.synchronize do
+            if forked?
+              # Trigger callback to allow workers to reset themselves accordingly
+              after_fork
+
+              # Reset and turn off
+              @pid = Process.pid
+              @run_async = false
+            end
+          end
+        end
+
+        def restart_after_fork(&block)
+          mutex_after_fork.synchronize do
+            if forked?
+              # Trigger callback to allow workers to reset themselves accordingly
+              after_fork
+
+              # Start worker
+              start_worker(&block)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ddtrace/workers/loop.rb

@@ -0,0 +1,119 @@
+module Datadog
+  module Workers
+    # Adds looping behavior to workers, with a sleep
+    # interval between each loop.
+    module IntervalLoop
+      BACK_OFF_RATIO = 1.2
+      BACK_OFF_MAX = 5
+      BASE_INTERVAL = 1
+
+      def self.included(base)
+        base.send(:prepend, PrependedMethods)
+      end
+
+      # Methods that must be prepended
+      module PrependedMethods
+        def perform(*args)
+          perform_loop { super(*args) }
+        end
+      end
+
+      def stop_loop
+        mutex.synchronize do
+          return false unless run_loop?
+
+          @run_loop = false
+          shutdown.signal
+        end
+
+        true
+      end
+
+      def work_pending?
+        run_loop?
+      end
+
+      def run_loop?
+        return false unless instance_variable_defined?(:@run_loop)
+
+        @run_loop == true
+      end
+
+      def loop_base_interval
+        @loop_base_interval ||= BASE_INTERVAL
+      end
+
+      def loop_back_off_ratio
+        @loop_back_off_ratio ||= BACK_OFF_RATIO
+      end
+
+      def loop_back_off_max
+        @loop_back_off_max ||= BACK_OFF_MAX
+      end
+
+      def loop_wait_time
+        @loop_wait_time ||= loop_base_interval
+      end
+
+      def loop_wait_time=(value)
+        @loop_wait_time = value
+      end
+
+      def reset_loop_wait_time
+        self.loop_wait_time = loop_base_interval
+      end
+
+      # Should the loop "back off" when there's no work?
+      def loop_back_off?
+        false
+      end
+
+      def loop_back_off!
+        self.loop_wait_time = [loop_wait_time * BACK_OFF_RATIO, BACK_OFF_MAX].min
+      end
+
+      protected
+
+      attr_writer \
+        :loop_back_off_max,
+        :loop_back_off_ratio,
+        :loop_base_interval
+
+      def mutex
+        @mutex ||= Mutex.new
+      end
+
+      private
+
+      def perform_loop
+        @run_loop = true
+
+        loop do
+          if work_pending?
+            # There's work to do...
+            # Run the task
+            yield
+
+            # Reset the wait interval
+            reset_loop_wait_time if loop_back_off?
+          elsif loop_back_off?
+            # There's no work to do...
+            # Back off the wait interval a bit
+            loop_back_off!
+          end
+
+          # Wait for an interval, unless shutdown has been signaled.
+          mutex.synchronize do
+            return unless run_loop? || work_pending?
+
+            shutdown.wait(mutex, loop_wait_time) if run_loop?
+          end
+        end
+      end
+
+      def shutdown
+        @shutdown ||= ConditionVariable.new
+      end
+    end
+  end
+end

data/lib/ddtrace/workers/polling.rb

@@ -0,0 +1,49 @@
+require 'ddtrace/workers/async'
+require 'ddtrace/workers/loop'
+
+module Datadog
+  module Workers
+    # Adds polling (async looping) behavior to workers
+    module Polling
+      SHUTDOWN_TIMEOUT = 1
+
+      def self.included(base)
+        base.send(:include, Workers::IntervalLoop)
+        base.send(:include, Workers::Async::Thread)
+        base.send(:prepend, PrependedMethods)
+      end
+
+      # Methods that must be prepended
+      module PrependedMethods
+        def perform(*args)
+          super if enabled?
+        end
+      end
+
+      def stop(force_stop = false, timeout = SHUTDOWN_TIMEOUT)
+        if running?
+          # Attempt graceful stop and wait
+          stop_loop
+          graceful = join(timeout)
+
+          # If timeout and force stop...
+          !graceful && force_stop ? terminate : graceful
+        else
+          false
+        end
+      end
+
+      def enabled?
+        return true unless instance_variable_defined?(:@enabled)
+
+        @enabled
+      end
+
+      # Allow worker to be started
+      def enabled=(value)
+        # Coerce to boolean
+        @enabled = (value == true)
+      end
+    end
+  end
+end

data/lib/ddtrace/workers/queue.rb

@@ -0,0 +1,40 @@
+module Datadog
+  module Workers
+    # Adds queue behavior to workers, with a buffer
+    # to which items can be queued then dequeued.
+    module Queue
+      def self.included(base)
+        base.send(:prepend, PrependedMethods)
+      end
+
+      # Methods that must be prepended
+      module PrependedMethods
+        def perform(*args)
+          super(*dequeue) if work_pending?
+        end
+      end
+
+      def buffer
+        @buffer ||= []
+      end
+
+      def enqueue(*args)
+        buffer.push(args)
+      end
+
+      def dequeue
+        buffer.shift
+      end
+
+      # Are there more items to be processed next?
+      def work_pending?
+        !buffer.empty?
+      end
+
+      protected
+
+      attr_writer \
+        :buffer
+    end
+  end
+end

data/lib/ddtrace/workers/runtime_metrics.rb

@@ -0,0 +1,51 @@
+require 'forwardable'
+
+require 'ddtrace/runtime/metrics'
+
+require 'ddtrace/worker'
+require 'ddtrace/workers/polling'
+
+module Datadog
+  module Workers
+    # Emits runtime metrics asynchronously on a timed loop
+    class RuntimeMetrics < Worker
+      extend Forwardable
+      include Workers::Polling
+
+      # In seconds
+      DEFAULT_FLUSH_INTERVAL = 10
+      DEFAULT_BACK_OFF_MAX = 30
+
+      attr_reader \
+        :metrics
+
+      def initialize(options = {})
+        @metrics = options.fetch(:metrics, Runtime::Metrics.new)
+
+        # Workers::Async::Thread settings
+        self.fork_policy = options.fetch(:fork_policy, Workers::Async::Thread::FORK_POLICY_STOP)
+
+        # Workers::IntervalLoop settings
+        self.loop_base_interval = options.fetch(:interval, DEFAULT_FLUSH_INTERVAL)
+        self.loop_back_off_ratio = options[:back_off_ratio] if options.key?(:back_off_ratio)
+        self.loop_back_off_max = options.fetch(:back_off_max, DEFAULT_BACK_OFF_MAX)
+
+        self.enabled = options.fetch(:enabled, false)
+      end
+
+      def perform
+        metrics.flush
+        true
+      end
+
+      def associate_with_span(*args)
+        # Start the worker
+        metrics.associate_with_span(*args).tap { perform }
+      end
+
+      def_delegators \
+        :metrics,
+        :register_service
+    end
+  end
+end

data/lib/ddtrace/workers/trace_writer.rb

@@ -0,0 +1,202 @@
+require 'ddtrace/logger'
+require 'ddtrace/transport/http'
+
+require 'ddtrace/event'
+require 'ddtrace/worker'
+require 'ddtrace/workers/polling'
+require 'ddtrace/workers/queue'
+
+module Datadog
+  module Workers
+    # Writes traces to transport synchronously
+    class TraceWriter < Worker
+      attr_reader \
+        :transport
+
+      def initialize(options = {})
+        transport_options = options.fetch(:transport_options, {})
+
+        transport_options = { on_build: transport_options } if transport_options.is_a?(Proc)
+
+        transport_options[:hostname] = options[:hostname] if options.key?(:hostname)
+        transport_options[:port] = options[:port] if options.key?(:port)
+
+        @transport = options.fetch(:transport) do
+          Transport::HTTP.default(transport_options)
+        end
+      end
+
+      def perform(traces)
+        write_traces(traces)
+      end
+
+      def write(trace)
+        write_traces([trace])
+      end
+
+      def write_traces(traces)
+        traces = process_traces(traces)
+        flush_traces(traces)
+      rescue StandardError => e
+        Datadog.logger.error(
+          "Error while writing traces: dropped #{traces.length} items. Cause: #{e} Location: #{e.backtrace.first}"
+        )
+      end
+
+      def process_traces(traces)
+        # Run traces through the processing pipeline
+        traces = Pipeline.process!(traces)
+
+        # Inject hostname if configured to do so
+        inject_hostname!(traces) if Datadog.configuration.report_hostname
+
+        traces
+      end
+
+      def flush_traces(traces)
+        transport.send_traces(traces).tap do |response|
+          flush_completed.publish(response)
+        end
+      end
+
+      def inject_hostname!(traces)
+        traces.each do |trace|
+          next if trace.first.nil?
+
+          hostname = Datadog::Runtime::Socket.hostname
+          trace.first.set_tag(Ext::NET::TAG_HOSTNAME, hostname) unless hostname.nil? || hostname.empty?
+        end
+      end
+
+      # TODO: Register `Datadog::Diagnostics::EnvironmentLogger.log!`
+      # TODO: as a flush_completed subscriber when the `TraceWriter`
+      # TODO: instantiation code is implemented.
+      def flush_completed
+        @flush_completed ||= FlushCompleted.new
+      end
+
+      # Flush completed event for worker
+      class FlushCompleted < Event
+        def initialize
+          super(:flush_completed)
+        end
+      end
+    end
+
+    # Writes traces to transport asynchronously,
+    # using a thread & buffer.
+    class AsyncTraceWriter < TraceWriter
+      include Workers::Queue
+      include Workers::Polling
+
+      DEFAULT_BUFFER_MAX_SIZE = 1000
+      FORK_POLICY_ASYNC = :async
+      FORK_POLICY_SYNC = :sync
+
+      attr_writer \
+        :async
+
+      def initialize(options = {})
+        # Workers::TraceWriter settings
+        super
+
+        # Workers::Polling settings
+        self.enabled = options.fetch(:enabled, true)
+
+        # Workers::Async::Thread settings
+        @async = true
+        self.fork_policy = options.fetch(:fork_policy, FORK_POLICY_ASYNC)
+
+        # Workers::IntervalLoop settings
+        self.loop_base_interval = options[:interval] if options.key?(:interval)
+        self.loop_back_off_ratio = options[:back_off_ratio] if options.key?(:back_off_ratio)
+        self.loop_back_off_max = options[:back_off_max] if options.key?(:back_off_max)
+
+        # Workers::Queue settings
+        @buffer_size = options.fetch(:buffer_size, DEFAULT_BUFFER_MAX_SIZE)
+        self.buffer = TraceBuffer.new(@buffer_size)
+      end
+
+      # NOTE: #perform is wrapped by other modules:
+      #       Polling --> Async --> IntervalLoop --> AsyncTraceWriter --> TraceWriter
+      #
+      # WARNING: This method breaks the Liskov Substitution Principle -- TraceWriter#perform is spec'd to return the
+      # result from the writer, whereas this method always returns nil.
+      def perform(traces)
+        super(traces).tap do |responses|
+          loop_back_off! if responses.find(&:server_error?)
+        end
+
+        nil
+      end
+
+      def stop(*args)
+        buffer.close if running?
+        super
+      end
+
+      def enqueue(trace)
+        buffer.push(trace)
+      end
+
+      def dequeue
+        # Wrap results in Array because they are
+        # splatted as args against TraceWriter#perform.
+        [buffer.pop]
+      end
+
+      # Are there more traces to be processed next?
+      def work_pending?
+        !buffer.empty?
+      end
+
+      def async?
+        @async == true
+      end
+
+      def fork_policy=(policy)
+        # Translate to Workers::Async::Thread policy
+        thread_fork_policy = case policy
+                             when Workers::Async::Thread::FORK_POLICY_STOP
+                               policy
+                             when FORK_POLICY_SYNC
+                               # Stop the async thread because the writer
+                               # will bypass and run synchronously.
+                               Workers::Async::Thread::FORK_POLICY_STOP
+                             else
+                               Workers::Async::Thread::FORK_POLICY_RESTART
+                             end
+
+        # Update thread fork policy
+        super(thread_fork_policy)
+
+        # Update local policy
+        @writer_fork_policy = policy
+      end
+
+      def after_fork
+        # In multiprocess environments, forks will share the same buffer until its written to.
+        # A.K.A. copy-on-write. We don't want forks to write traces generated from another process.
+        # Instead, we reset it after the fork. (Make sure any enqueue operations happen after this.)
+        self.buffer = TraceBuffer.new(@buffer_size)
+
+        # Switch to synchronous mode if configured to do so.
+        # In some cases synchronous writing is preferred because the fork will be short lived.
+        @async = false if @writer_fork_policy == FORK_POLICY_SYNC
+      end
+
+      # WARNING: This method breaks the Liskov Substitution Principle -- TraceWriter#write is spec'd to return the
+      # result from the writer, whereas this method returns something else when running in async mode.
+      def write(trace)
+        # Start worker thread. If the process has forked, it will trigger #after_fork to
+        # reconfigure the worker accordingly.
+        # NOTE: It's important we do this before queuing or it will drop the current trace,
+        #       because #after_fork resets the buffer.
+        perform
+
+        # Queue the trace if running asynchronously, otherwise short-circuit and write it directly.
+        async? ? enqueue(trace) : write_traces([trace])
+      end
+    end
+  end
+end