ddtrace 0.54.2 → 1.2.0

data/lib/datadog/tracing/workers.rb

@@ -0,0 +1,125 @@
+# typed: true
+
+require 'datadog/core'
+
+require 'datadog/tracing/buffer'
+require 'datadog/tracing/pipeline'
+
+module Datadog
+  module Tracing
+    module Workers
+      # Asynchronous worker that executes a +Send()+ operation after given
+      # seconds. Under the hood, it uses +Concurrent::TimerTask+ so that the thread
+      # will perform a task at regular intervals. The thread can be stopped
+      # with the +stop()+ method and can start with the +start()+ method.
+      class AsyncTransport
+        DEFAULT_BUFFER_MAX_SIZE = 1000
+        DEFAULT_FLUSH_INTERVAL = 1
+        DEFAULT_TIMEOUT = 5
+        BACK_OFF_RATIO = 1.2
+        BACK_OFF_MAX = 5
+        SHUTDOWN_TIMEOUT = 1
+
+        attr_reader \
+          :trace_buffer
+
+        def initialize(options = {})
+          @transport = options[:transport]
+
+          # Callbacks
+          @trace_task = options[:on_trace]
+
+          # Intervals
+          interval = options.fetch(:interval, DEFAULT_FLUSH_INTERVAL)
+          @flush_interval = interval
+          @back_off = interval
+
+          # Buffers
+          buffer_size = options.fetch(:buffer_size, DEFAULT_BUFFER_MAX_SIZE)
+          @trace_buffer = TraceBuffer.new(buffer_size)
+
+          # Threading
+          @shutdown = ConditionVariable.new
+          @mutex = Mutex.new
+          @worker = nil
+          @run = false
+        end
+
+        # Callback function that process traces and executes the +send_traces()+ method.
+        def callback_traces
+          return true if @trace_buffer.empty?
+
+          begin
+            traces = @trace_buffer.pop
+            traces = Pipeline.process!(traces)
+            @trace_task.call(traces, @transport) unless @trace_task.nil? || traces.empty?
+          rescue StandardError => e
+            # ensures that the thread will not die because of an exception.
+            # TODO[manu]: findout the reason and reschedule the send if it's not
+            # a fatal exception
+            Datadog.logger.error(
+              "Error during traces flush: dropped #{traces.length} items. Cause: #{e} Location: #{Array(e.backtrace).first}"
+            )
+          end
+        end
+
+        # Start the timer execution.
+        def start
+          @mutex.synchronize do
+            return if @run
+
+            @run = true
+            Datadog.logger.debug { "Starting thread for: #{self}" }
+            @worker = Thread.new { perform }
+            @worker.name = self.class.name unless Gem::Version.new(RUBY_VERSION) < Gem::Version.new('2.3')
+
+            nil
+          end
+        end
+
+        # Closes all available queues and waits for the trace buffer to flush
+        def stop
+          @mutex.synchronize do
+            return unless @run
+
+            @trace_buffer.close
+            @run = false
+            @shutdown.signal
+          end
+
+          join
+          true
+        end
+
+        # Block until executor shutdown is complete or until timeout seconds have passed.
+        def join
+          @worker.join(SHUTDOWN_TIMEOUT)
+        end
+
+        # Enqueue an item in the trace internal buffer. This operation is thread-safe
+        # because uses the +TraceBuffer+ data structure.
+        def enqueue_trace(trace)
+          return unless trace && !trace.empty?
+
+          @trace_buffer.push(trace)
+        end
+
+        private
+
+        alias flush_data callback_traces
+
+        def perform
+          loop do
+            @back_off = flush_data ? @flush_interval : [@back_off * BACK_OFF_RATIO, BACK_OFF_MAX].min
+
+            @mutex.synchronize do
+              return if !@run && @trace_buffer.empty?
+
+              @shutdown.wait(@mutex, @back_off) if @run # do not wait when shutting down
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/tracing/writer.rb

@@ -0,0 +1,180 @@
+# typed: true
+
+require 'datadog/core'
+
+require 'datadog/tracing/event'
+require 'datadog/tracing/runtime/metrics'
+require 'datadog/tracing/workers'
+
+require 'ddtrace/transport/http'
+
+module Datadog
+  module Tracing
+    # Processor that sends traces and metadata to the agent
+    # DEV: Our goal is for {Datadog::Tracing::Workers::TraceWriter} to replace this class in the future
+    # @public_api
+    class Writer
+      attr_reader \
+        :transport,
+        :worker,
+        :events
+
+      def initialize(options = {})
+        # writer and transport parameters
+        @buff_size = options.fetch(:buffer_size, Workers::AsyncTransport::DEFAULT_BUFFER_MAX_SIZE)
+        @flush_interval = options.fetch(:flush_interval, Workers::AsyncTransport::DEFAULT_FLUSH_INTERVAL)
+        transport_options = options.fetch(:transport_options, {})
+
+        transport_options[:agent_settings] = options[:agent_settings] if options.key?(:agent_settings)
+
+        # transport and buffers
+        @transport = options.fetch(:transport) do
+          Transport::HTTP.default(**transport_options)
+        end
+
+        # handles the thread creation after an eventual fork
+        @mutex_after_fork = Mutex.new
+        @pid = nil
+
+        @traces_flushed = 0
+
+        # one worker for traces
+        @worker = nil
+
+        # Once stopped, this writer instance cannot be restarted.
+        # This allow for graceful shutdown, while preventing
+        # the host application from inadvertently start new
+        # threads during shutdown.
+        @stopped = false
+
+        # Callback handler
+        @events = Events.new
+      end
+
+      # Explicitly starts the {Writer}'s internal worker.
+      #
+      # The {Writer} is also automatically started when necessary during calls to {.write}.
+      def start
+        @mutex_after_fork.synchronize do
+          return false if @stopped
+
+          pid = Process.pid
+          return if @worker && pid == @pid
+
+          @pid = pid
+
+          start_worker
+          true
+        end
+      end
+
+      # spawns a worker for spans; they share the same transport which is thread-safe
+      # @!visibility private
+      def start_worker
+        @trace_handler = ->(items, transport) { send_spans(items, transport) }
+        @worker = Workers::AsyncTransport.new(
+          transport: @transport,
+          buffer_size: @buff_size,
+          on_trace: @trace_handler,
+          interval: @flush_interval
+        )
+
+        @worker.start
+      end
+
+      # Gracefully shuts down this writer.
+      #
+      # Once stopped methods calls won't fail, but
+      # no internal work will be performed.
+      #
+      # It is not possible to restart a stopped writer instance.
+      def stop
+        @mutex_after_fork.synchronize { stop_worker }
+      end
+
+      def stop_worker
+        @stopped = true
+
+        return if @worker.nil?
+
+        @worker.stop
+        @worker = nil
+
+        true
+      end
+
+      private :start_worker, :stop_worker
+
+      # flush spans to the trace-agent, handles spans only
+      # @!visibility private
+      def send_spans(traces, transport)
+        return true if traces.empty?
+
+        # Send traces and get responses
+        responses = transport.send_traces(traces)
+
+        # Tally up successful flushes
+        responses.reject { |x| x.internal_error? || x.server_error? }.each do |response|
+          @traces_flushed += response.trace_count
+        end
+
+        events.after_send.publish(self, responses)
+
+        # Return if server error occurred.
+        !responses.find(&:server_error?)
+      end
+
+      # enqueue the trace for submission to the API
+      def write(trace)
+        # In multiprocess environments, the main process initializes the +Writer+ instance and if
+        # the process forks (i.e. a web server like Unicorn or Puma with multiple workers) the new
+        # processes will share the same +Writer+ until the first write (COW). Because of that,
+        # each process owns a different copy of the +@buffer+ after each write and so the
+        # +AsyncTransport+ will not send data to the trace agent.
+        #
+        # This check ensures that if a process doesn't own the current +Writer+, async workers
+        # will be initialized again (but only once for each process).
+        start if @worker.nil? || @pid != Process.pid
+
+        # TODO: Remove this, and have the tracer pump traces directly to runtime metrics
+        #       instead of working through the trace writer.
+        # Associate trace with runtime metrics
+        Runtime::Metrics.associate_trace(trace)
+
+        worker_local = @worker
+
+        if worker_local
+          worker_local.enqueue_trace(trace)
+        elsif !@stopped
+          Datadog.logger.debug('Writer either failed to start or was stopped before #write could complete')
+        end
+      end
+
+      # stats returns a dictionary of stats about the writer.
+      def stats
+        {
+          traces_flushed: @traces_flushed,
+          transport: @transport.stats
+        }
+      end
+
+      # Callback behavior
+      class Events
+        attr_reader \
+          :after_send
+
+        def initialize
+          @after_send = AfterSend.new
+        end
+
+        # Triggered after the writer sends traces through the transport.
+        # Provides the Writer instance and transport response list to the callback.
+        class AfterSend < Tracing::Event
+          def initialize
+            super(:after_send)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/tracing.rb

@@ -0,0 +1,142 @@
+# typed: false
+
+require 'datadog/core'
+require 'datadog/tracing/pipeline'
+
+module Datadog
+  # Datadog APM tracing public API.
+  #
+  # The Datadog team ensures that public methods in this module
+  # only receive backwards compatible changes, and breaking changes
+  # will only occur in new major versions releases.
+  # @public_api
+  module Tracing
+    class << self
+      # (see Datadog::Tracing::Tracer#trace)
+      # @public_api
+      def trace(name, continue_from: nil, **span_options, &block)
+        tracer.trace(name, continue_from: continue_from, **span_options, &block)
+      end
+
+      # (see Datadog:::Tracing::Tracer#continue_trace!)
+      # @public_api
+      def continue_trace!(digest, &block)
+        tracer.continue_trace!(digest, &block)
+      end
+
+      # The tracer's internal logger instance.
+      # All tracing log output is handled by this object.
+      #
+      # The logger can be configured through {.configure},
+      # through {Datadog::Core::Configuration::Settings::DSL::Logger} options.
+      #
+      # @!attribute [r] logger
+      # @public_api
+      def logger
+        Datadog.logger
+      end
+
+      # (see Datadog::Tracing::Tracer#active_trace)
+      # @public_api
+      def active_trace
+        current_tracer = tracer
+        return unless current_tracer
+
+        current_tracer.active_trace
+      end
+
+      # (see Datadog:::Tracing::Tracer#active_span)
+      # @public_api
+      def active_span
+        current_tracer = tracer
+        return unless current_tracer
+
+        current_tracer.active_span
+      end
+
+      # (see Datadog:::Tracing::TraceSegment#keep!)
+      # If no trace is active, no action is taken.
+      # @public_api
+      def keep!
+        trace = active_trace
+        active_trace.keep! if trace
+      end
+
+      # (see Datadog:::Tracing::TraceSegment#reject!)
+      # If no trace is active, no action is taken.
+      # @public_api
+      def reject!
+        trace = active_trace
+        active_trace.reject! if trace
+      end
+
+      # (see Datadog:::Tracing::Tracer#active_correlation)
+      # @public_api
+      def correlation
+        current_tracer = tracer
+        return unless current_tracer
+
+        current_tracer.active_correlation
+      end
+
+      # Textual representation of {.correlation}, which can be
+      # added to individual log lines in order to correlate them with the active
+      # trace.
+      #
+      # Example:
+      #
+      # ```
+      # MyLogger.log("#{Datadog::Tracing.log_correlation}] My message")
+      # # dd.env=prod dd.service=auth dd.version=13.8 dd.trace_id=5458478252992251 dd.span_id=7117552347370098 My message
+      # ```
+      #
+      # @return [String] correlation information
+      # @public_api
+      def log_correlation
+        correlation.to_log_format
+      end
+
+      # Gracefully shuts down the tracer.
+      #
+      # The public tracing API will still respond to method calls as usual
+      # but might not internally perform the expected internal work after shutdown.
+      #
+      # This avoids errors being raised across the host application
+      # during shutdown while allowing for the graceful decommission of resources.
+      #
+      # {.shutdown!} cannot be reversed.
+      # @public_api
+      def shutdown!
+        current_tracer = tracer
+        return unless current_tracer
+
+        current_tracer.shutdown!
+      end
+
+      # (see Datadog:::Tracing::Pipeline.before_flush)
+      def before_flush(*processors, &processor_block)
+        Pipeline.before_flush(*processors, &processor_block)
+      end
+
+      # Is the tracer collecting telemetry data in this process?
+      # @return [Boolean] `true` if the tracer is collecting data in this process, otherwise `false`.
+      def enabled?
+        current_tracer = tracer
+        return false unless current_tracer
+
+        current_tracer.enabled
+      end
+
+      private
+
+      # DEV: components hosts both tracing and profiling inner objects today
+      def components
+        Datadog.send(:components)
+      end
+
+      def tracer
+        components.tracer
+      end
+    end
+  end
+end

data/lib/ddtrace/auto_instrument.rb

@@ -1,5 +1,9 @@
 # typed: strict
+
+# Entrypoint file for auto instrumentation.
+#
+# This file's path is part of the @public_api.
 require 'ddtrace'
+require 'datadog/tracing/contrib/auto_instrument'
 
-Datadog.add_auto_instrument
-Datadog.profiler.start if Datadog.profiler
+Datadog::Profiling.start_if_enabled

data/lib/ddtrace/auto_instrument_base.rb

@@ -1,4 +1,5 @@
 # typed: true
+
 module Datadog
   # base methods stubbed for adding auto instrument extensions
   module AutoInstrumentBase

data/lib/ddtrace/profiling/preload.rb

@@ -1,4 +1,4 @@
 # typed: strict
-require 'ddtrace'
 
-Datadog.profiler.start if Datadog.profiler
+# TODO: Warns users after 1.0 upgrade. Remove this file in 2.0.
+raise LoadError, 'This file has been moved to `datadog/profiling/preload` in 1.0.0'

data/lib/ddtrace/transport/ext.rb

@@ -0,0 +1,35 @@
+# typed: true
+
+module Datadog
+  module Transport
+    # @public_api
+    module Ext
+      # @public_api
+      module HTTP
+        ADAPTER = :net_http # DEV: Rename to simply `:http`, as Net::HTTP is an implementation detail.
+        DEFAULT_HOST = '127.0.0.1'.freeze
+        DEFAULT_PORT = 8126
+        DEFAULT_TIMEOUT_SECONDS = 1
+
+        HEADER_CONTAINER_ID = 'Datadog-Container-ID'.freeze
+        HEADER_DD_API_KEY = 'DD-API-KEY'.freeze
+        HEADER_META_LANG = 'Datadog-Meta-Lang'.freeze
+        HEADER_META_LANG_VERSION = 'Datadog-Meta-Lang-Version'.freeze
+        HEADER_META_LANG_INTERPRETER = 'Datadog-Meta-Lang-Interpreter'.freeze
+        HEADER_META_TRACER_VERSION = 'Datadog-Meta-Tracer-Version'.freeze
+      end
+
+      # @public_api
+      module Test
+        ADAPTER = :test
+      end
+
+      # @public_api
+      module UnixSocket
+        ADAPTER = :unix
+        DEFAULT_PATH = '/var/run/datadog/apm.socket'.freeze
+        DEFAULT_TIMEOUT_SECONDS = 1
+      end
+    end
+  end
+end

data/lib/ddtrace/transport/http/adapters/net.rb

@@ -1,6 +1,7 @@
 # typed: true
+
 require 'ddtrace/transport/response'
-require 'ddtrace/vendor/multipart-post/net/http/post/multipart'
+require 'datadog/core/vendor/multipart-post/net/http/post/multipart'
 
 module Datadog
   module Transport
@@ -60,7 +61,7 @@ module Datadog
               post = ::Net::HTTP::Post.new(env.path, env.headers)
               post.body = env.body
             else
-              post = ::Datadog::Vendor::Net::HTTP::Post::Multipart.new(
+              post = ::Datadog::Core::Vendor::Net::HTTP::Post::Multipart.new(
                 env.path,
                 env.form,
                 env.headers

data/lib/ddtrace/transport/http/adapters/registry.rb

@@ -1,4 +1,5 @@
 # typed: true
+
 module Datadog
   module Transport
     module HTTP

data/lib/ddtrace/transport/http/adapters/test.rb

@@ -1,4 +1,5 @@
 # typed: true
+
 require 'ddtrace/transport/response'
 
 module Datadog

data/lib/ddtrace/transport/http/adapters/unix_socket.rb

@@ -1,6 +1,7 @@
 # typed: false
+
 require 'net/http'
-require 'ddtrace/ext/transport'
+require 'ddtrace/transport/ext'
 require 'ddtrace/transport/http/adapters/net'
 
 module Datadog
@@ -18,7 +19,7 @@ module Datadog
           # @deprecated Positional parameters are deprecated. Use named parameters instead.
           def initialize(uds_path = nil, **options)
             @filepath = uds_path || options.fetch(:uds_path)
-            @timeout = options[:timeout] || Ext::Transport::UnixSocket::DEFAULT_TIMEOUT_SECONDS
+            @timeout = options[:timeout] || Transport::Ext::UnixSocket::DEFAULT_TIMEOUT_SECONDS
           end
 
           def self.build(agent_settings)

data/lib/ddtrace/transport/http/api/endpoint.rb

@@ -1,4 +1,5 @@
 # typed: true
+
 require 'json'
 
 module Datadog

data/lib/ddtrace/transport/http/api/fallbacks.rb

@@ -1,4 +1,5 @@
 # typed: false
+
 module Datadog
   module Transport
     module HTTP

data/lib/ddtrace/transport/http/api/instance.rb

@@ -1,4 +1,5 @@
 # typed: true
+
 module Datadog
   module Transport
     module HTTP

data/lib/ddtrace/transport/http/api/map.rb

@@ -1,4 +1,5 @@
 # typed: true
+
 require 'ddtrace/transport/http/api/fallbacks'
 
 module Datadog

data/lib/ddtrace/transport/http/api/spec.rb

@@ -1,4 +1,5 @@
 # typed: true
+
 module Datadog
   module Transport
     module HTTP

data/lib/ddtrace/transport/http/api.rb

@@ -1,5 +1,6 @@
 # typed: false
-require 'ddtrace/encoding'
+
+require 'datadog/core/encoding'
 
 require 'ddtrace/transport/http/api/map'
 require 'ddtrace/transport/http/api/spec'
@@ -14,7 +15,6 @@ module Datadog
         # Default API versions
         V4 = 'v0.4'.freeze
         V3 = 'v0.3'.freeze
-        V2 = 'v0.2'.freeze
 
         module_function
 
@@ -23,23 +23,17 @@ module Datadog
             V4 => Spec.new do |s|
               s.traces = Traces::API::Endpoint.new(
                 '/v0.4/traces'.freeze,
-                Encoding::MsgpackEncoder,
+                Core::Encoding::MsgpackEncoder,
                 service_rates: true
               )
             end,
             V3 => Spec.new do |s|
               s.traces = Traces::API::Endpoint.new(
                 '/v0.3/traces'.freeze,
-                Encoding::MsgpackEncoder
+                Core::Encoding::MsgpackEncoder
               )
             end,
-            V2 => Spec.new do |s|
-              s.traces = Traces::API::Endpoint.new(
-                '/v0.2/traces'.freeze,
-                Encoding::JSONEncoder
-              )
-            end
-          ].with_fallbacks(V4 => V3, V3 => V2)
+          ].with_fallbacks(V4 => V3)
         end
       end
     end

data/lib/ddtrace/transport/http/builder.rb

@@ -1,5 +1,6 @@
 # typed: true
-require 'ddtrace/configuration/agent_settings_resolver'
+
+require 'datadog/core/configuration/agent_settings_resolver'
 require 'ddtrace/transport/http/adapters/registry'
 require 'ddtrace/transport/http/api/map'
 require 'ddtrace/transport/http/api/instance'
@@ -36,7 +37,7 @@ module Datadog
 
         def adapter(config, *args, **kwargs)
           @default_adapter = case config
-                             when Configuration::AgentSettingsResolver::AgentSettings
+                             when Core::Configuration::AgentSettingsResolver::AgentSettings
                                registry_klass = REGISTRY.get(config.adapter)
                                raise UnknownAdapterError, config.adapter if registry_klass.nil?
 

data/lib/ddtrace/transport/http/client.rb

@@ -1,4 +1,5 @@
 # typed: true
+
 require 'ddtrace/transport/http/statistics'
 require 'ddtrace/transport/http/env'
 
@@ -28,7 +29,8 @@ module Datadog
           response
         rescue StandardError => e
           message =
-            "Internal error during HTTP transport request. Cause: #{e.message} Location: #{Array(e.backtrace).first}"
+            "Internal error during #{self.class.name} request. Cause: #{e.class.name} #{e.message} " \
+            "Location: #{Array(e.backtrace).first}"
 
           # Log error
           if stats.consecutive_errors > 0

data/lib/ddtrace/transport/http/env.rb

@@ -1,4 +1,5 @@
 # typed: true
+
 module Datadog
   module Transport
     module HTTP