ddtrace 0.34.2 → 0.35.0

data/lib/ddtrace/utils.rb

@@ -57,7 +57,7 @@ module Datadog
         str.encode(::Encoding::UTF_8)
       end
     rescue => e
-      Logger.log.debug("Error encoding string in UTF-8: #{e}")
+      Datadog.logger.debug("Error encoding string in UTF-8: #{e}")
 
       options.fetch(:placeholder, STRING_PLACEHOLDER)
     end

data/lib/ddtrace/version.rb

@@ -1,8 +1,8 @@
 module Datadog
   module VERSION
     MAJOR = 0
-    MINOR = 34
-    PATCH = 2
+    MINOR = 35
+    PATCH = 0
     PRE = nil
 
     STRING = [MAJOR, MINOR, PATCH, PRE].compact.join('.')

data/lib/ddtrace/workers.rb

@@ -1,5 +1,8 @@
 require 'time'
 
+require 'ddtrace/workers/trace_writer'
+require 'ddtrace/workers/runtime_metrics'
+
 require 'ddtrace/buffer'
 require 'ddtrace/runtime/metrics'
 
@@ -25,7 +28,6 @@ module Datadog
 
         # Callbacks
         @trace_task = options[:on_trace]
-        @runtime_metrics_task = options[:on_runtime_metrics]
 
         # Intervals
         interval = options.fetch(:interval, DEFAULT_FLUSH_INTERVAL)
@@ -55,26 +57,18 @@ module Datadog
           # ensures that the thread will not die because of an exception.
           # TODO[manu]: findout the reason and reschedule the send if it's not
           # a fatal exception
-          Datadog::Logger.log.error(
+          Datadog.logger.error(
             "Error during traces flush: dropped #{traces.length} items. Cause: #{e} Location: #{e.backtrace.first}"
           )
         end
       end
 
-      def callback_runtime_metrics
-        @runtime_metrics_task.call unless @runtime_metrics_task.nil?
-      rescue StandardError => e
-        Datadog::Logger.log.error(
-          "Error during runtime metrics flush. Cause: #{e} Location: #{e.backtrace.first}"
-        )
-      end
-
       # Start the timer execution.
       def start
         @mutex.synchronize do
           return if @run
           @run = true
-          Logger.log.debug("Starting thread in the process: #{Process.pid}")
+          Datadog.logger.debug("Starting thread in the process: #{Process.pid}")
           @worker = Thread.new { perform }
         end
       end
@@ -112,8 +106,6 @@ module Datadog
         loop do
           @back_off = flush_data ? @flush_interval : [@back_off * BACK_OFF_RATIO, BACK_OFF_MAX].min
 
-          callback_runtime_metrics
-
           @mutex.synchronize do
             return if !@run && @trace_buffer.empty?
             @shutdown.wait(@mutex, @back_off) if @run # do not wait when shutting down

data/lib/ddtrace/workers/async.rb

@@ -121,7 +121,7 @@ module Datadog
           @run_async = true
           @pid = Process.pid
           @error = nil
-          Logger.log.debug("Starting thread in the process: #{Process.pid}")
+          Datadog.logger.debug("Starting thread in the process: #{Process.pid}")
 
           @worker = ::Thread.new do
             begin
@@ -129,7 +129,7 @@ module Datadog
             # rubocop:disable Lint/RescueException
             rescue Exception => e
               @error = e
-              Logger.log.debug("Worker thread error. Cause #{e.message} Location: #{e.backtrace.first}")
+              Datadog.logger.debug("Worker thread error. Cause #{e.message} Location: #{e.backtrace.first}")
               raise
             end
           end

data/lib/ddtrace/workers/runtime_metrics.rb

@@ -0,0 +1,47 @@
+require 'forwardable'
+
+require 'ddtrace/runtime/metrics'
+
+require 'ddtrace/worker'
+require 'ddtrace/workers/polling'
+
+module Datadog
+  module Workers
+    # Emits runtime metrics asynchronously on a timed loop
+    class RuntimeMetrics < Worker
+      extend Forwardable
+      include Workers::Polling
+
+      attr_reader \
+        :metrics
+
+      def initialize(options = {})
+        @metrics = options.fetch(:metrics, Runtime::Metrics.new)
+
+        # Workers::Async::Thread settings
+        self.fork_policy = options.fetch(:fork_policy, Workers::Async::Thread::FORK_POLICY_STOP)
+
+        # Workers::IntervalLoop settings
+        self.interval = options[:interval] if options.key?(:interval)
+        self.back_off_ratio = options[:back_off_ratio] if options.key?(:back_off_ratio)
+        self.back_off_max = options[:back_off_max] if options.key?(:back_off_max)
+
+        self.enabled = options.fetch(:enabled, false)
+      end
+
+      def perform
+        metrics.flush
+        true
+      end
+
+      def associate_with_span(*args)
+        # Start the worker
+        metrics.associate_with_span(*args).tap { perform }
+      end
+
+      def_delegators \
+        :metrics,
+        :register_service
+    end
+  end
+end

data/lib/ddtrace/workers/trace_writer.rb

@@ -0,0 +1,199 @@
+require 'ddtrace/logger'
+require 'ddtrace/transport/http'
+
+require 'ddtrace/event'
+require 'ddtrace/worker'
+require 'ddtrace/workers/polling'
+require 'ddtrace/workers/queue'
+
+module Datadog
+  module Workers
+    # Writes traces to transport synchronously
+    class TraceWriter < Worker
+      attr_reader \
+        :transport
+
+      def initialize(options = {})
+        transport_options = options.fetch(:transport_options, {})
+
+        if transport_options.is_a?(Proc)
+          transport_options = { on_build: transport_options }
+        end
+
+        transport_options[:hostname] = options[:hostname] if options.key?(:hostname)
+        transport_options[:port] = options[:port] if options.key?(:port)
+
+        @transport = options.fetch(:transport) do
+          Transport::HTTP.default(transport_options)
+        end
+      end
+
+      def perform(traces)
+        write_traces(traces)
+      end
+
+      def write(trace)
+        write_traces([trace])
+      end
+
+      def write_traces(traces)
+        traces = process_traces(traces)
+        flush_traces(traces)
+      rescue StandardError => e
+        Datadog.logger.error(
+          "Error while writing traces: dropped #{traces.length} items. Cause: #{e} Location: #{e.backtrace.first}"
+        )
+      end
+
+      def process_traces(traces)
+        # Run traces through the processing pipeline
+        traces = Pipeline.process!(traces)
+
+        # Inject hostname if configured to do so
+        inject_hostname!(traces) if Datadog.configuration.report_hostname
+
+        traces
+      end
+
+      def flush_traces(traces)
+        transport.send_traces(traces).tap do |response|
+          flush_completed.publish(response)
+        end
+      end
+
+      def inject_hostname!(traces)
+        traces.each do |trace|
+          next if trace.first.nil?
+
+          hostname = Datadog::Runtime::Socket.hostname
+          unless hostname.nil? || hostname.empty?
+            trace.first.set_tag(Ext::NET::TAG_HOSTNAME, hostname)
+          end
+        end
+      end
+
+      def flush_completed
+        @flush_completed ||= FlushCompleted.new
+      end
+
+      # Flush completed event for worker
+      class FlushCompleted < Event
+        def initialize
+          super(:flush_completed)
+        end
+
+        def publish(response)
+          super(response)
+        end
+      end
+    end
+
+    # Writes traces to transport asynchronously,
+    # using a thread & buffer.
+    class AsyncTraceWriter < TraceWriter
+      include Workers::Queue
+      include Workers::Polling
+
+      DEFAULT_BUFFER_MAX_SIZE = 1000
+      FORK_POLICY_ASYNC = :async
+      FORK_POLICY_SYNC = :sync
+
+      attr_writer \
+        :async
+
+      def initialize(options = {})
+        # Workers::TraceWriter settings
+        super
+
+        # Workers::Polling settings
+        self.enabled = options.fetch(:enabled, true)
+
+        # Workers::Async::Thread settings
+        @async = true
+        self.fork_policy = options.fetch(:fork_policy, FORK_POLICY_ASYNC)
+
+        # Workers::IntervalLoop settings
+        self.loop_base_interval = options[:interval] if options.key?(:interval)
+        self.loop_back_off_ratio = options[:back_off_ratio] if options.key?(:back_off_ratio)
+        self.loop_back_off_max = options[:back_off_max] if options.key?(:back_off_max)
+
+        # Workers::Queue settings
+        @buffer_size = options.fetch(:buffer_size, DEFAULT_BUFFER_MAX_SIZE)
+        self.buffer = TraceBuffer.new(@buffer_size)
+      end
+
+      # NOTE: #perform is wrapped by other modules:
+      #       Polling --> Async --> IntervalLoop --> AsyncTraceWriter --> TraceWriter
+      def perform(traces)
+        super(traces).tap do |responses|
+          loop_back_off! if responses.find(&:server_error?)
+        end
+      end
+
+      def stop(*args)
+        buffer.close if running?
+        super
+      end
+
+      def enqueue(trace)
+        buffer.push(trace)
+      end
+
+      def dequeue
+        # Wrap results in Array because they are
+        # splatted as args against TraceWriter#perform.
+        [buffer.pop]
+      end
+
+      def work_pending?
+        !buffer.empty?
+      end
+
+      def async?
+        @async == true
+      end
+
+      def fork_policy=(policy)
+        # Translate to Workers::Async::Thread policy
+        thread_fork_policy = case policy
+                             when Workers::Async::Thread::FORK_POLICY_STOP
+                               policy
+                             when FORK_POLICY_SYNC
+                               # Stop the async thread because the writer
+                               # will bypass and run synchronously.
+                               Workers::Async::Thread::FORK_POLICY_STOP
+                             else
+                               Workers::Async::Thread::FORK_POLICY_RESTART
+                             end
+
+        # Update thread fork policy
+        super(thread_fork_policy)
+
+        # Update local policy
+        @writer_fork_policy = policy
+      end
+
+      def after_fork
+        # In multiprocess environments, forks will share the same buffer until its written to.
+        # A.K.A. copy-on-write. We don't want forks to write traces generated from another process.
+        # Instead, we reset it after the fork. (Make sure any enqueue operations happen after this.)
+        self.buffer = TraceBuffer.new(@buffer_size)
+
+        # Switch to synchronous mode if configured to do so.
+        # In some cases synchronous writing is preferred because the fork will be short lived.
+        @async = false if @writer_fork_policy == FORK_POLICY_SYNC
+      end
+
+      def write(trace)
+        # Start worker thread. If the process has forked, it will trigger #after_fork to
+        # reconfigure the worker accordingly.
+        # NOTE: It's important we do this before queuing or it will drop the current trace,
+        #       because #after_fork resets the buffer.
+        perform
+
+        # Queue the trace if running asynchronously, otherwise short-circuit and write it directly.
+        async? ? enqueue(trace) : write_traces([trace])
+      end
+    end
+  end
+end

data/lib/ddtrace/writer.rb

@@ -13,7 +13,6 @@ module Datadog
   class Writer
     attr_reader \
       :priority_sampler,
-      :runtime_metrics,
       :transport,
       :worker
 
@@ -34,11 +33,6 @@ module Datadog
         Transport::HTTP.default(transport_options)
       end
 
-      # Runtime metrics
-      @runtime_metrics = options.fetch(:runtime_metrics) do
-        Runtime::Metrics.new
-      end
-
       # handles the thread creation after an eventual fork
       @mutex_after_fork = Mutex.new
       @pid = nil
@@ -53,12 +47,10 @@ module Datadog
     def start
       @pid = Process.pid
       @trace_handler = ->(items, transport) { send_spans(items, transport) }
-      @runtime_metrics_handler = -> { send_runtime_metrics }
       @worker = Datadog::Workers::AsyncTransport.new(
         transport: @transport,
         buffer_size: @buff_size,
         on_trace: @trace_handler,
-        on_runtime_metrics: @runtime_metrics_handler,
         interval: @flush_interval
       )
 
@@ -80,33 +72,26 @@ module Datadog
       # Inject hostname if configured to do so
       inject_hostname!(traces) if Datadog.configuration.report_hostname
 
-      # Send traces an get a response.
-      response = transport.send_traces(traces)
-
-      unless response.internal_error?
-        @traces_flushed += traces.length unless response.server_error?
+      # Send traces and get responses
+      responses = transport.send_traces(traces)
 
-        # Update priority sampler
-        unless priority_sampler.nil? || response.service_rates.nil?
-          priority_sampler.update(response.service_rates)
-        end
+      # Tally up successful flushes
+      responses.reject { |x| x.internal_error? || x.server_error? }.each do |response|
+        @traces_flushed += response.trace_count
       end
 
-      # Return if server error occurred.
-      !response.server_error?
-    end
+      # Update priority sampler
+      update_priority_sampler(responses.last)
 
-    def send_runtime_metrics
-      return unless Datadog.configuration.runtime_metrics_enabled
-
-      runtime_metrics.flush
+      # Return if server error occurred.
+      !responses.find(&:server_error?)
     end
 
     # enqueue the trace for submission to the API
     def write(trace, services = nil)
       unless services.nil?
         Datadog::Patcher.do_once('Writer#write') do
-          Datadog::Logger.log.warn(%(
+          Datadog.logger.warn(%(
             write: Writing services has been deprecated and no longer need to be provided.
             write(traces, services) can be updated to write(traces)
           ))
@@ -129,9 +114,11 @@ module Datadog
         end
       end
 
+      # TODO: Remove this, and have the tracer pump traces directly to runtime metrics
+      #       instead of working through the trace writer.
       # Associate root span with runtime metrics
-      if Datadog.configuration.runtime_metrics_enabled && !trace.empty?
-        runtime_metrics.associate_with_span(trace.first)
+      if Datadog.configuration.runtime_metrics.enabled && !trace.empty?
+        Datadog.runtime_metrics.associate_with_span(trace.first)
       end
 
       @worker.enqueue_trace(trace)
@@ -157,5 +144,11 @@ module Datadog
         end
       end
     end
+
+    def update_priority_sampler(response)
+      return unless response && !response.internal_error? && priority_sampler && response.service_rates
+
+      priority_sampler.update(response.service_rates)
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ddtrace
 version: !ruby/object:Gem::Version
-  version: 0.34.2
+  version: 0.35.0
 platform: ruby
 authors:
 - Datadog, Inc.
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-04-09 00:00:00.000000000 Z
+date: 2020-05-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
@@ -136,20 +136,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.2'
-- !ruby/object:Gem::Dependency
-  name: bundler
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "<="
-      - !ruby/object:Gem::Version
-        version: 2.1.2
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "<="
-      - !ruby/object:Gem::Version
-        version: 2.1.2
 - !ruby/object:Gem::Dependency
   name: yard
   requirement: !ruby/object:Gem::Requirement
@@ -192,20 +178,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: ruby-prof
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: sqlite3
   requirement: !ruby/object:Gem::Requirement
@@ -290,6 +262,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.4.9.2
+- !ruby/object:Gem::Dependency
+  name: warning
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
 description: |
   ddtrace is Datadog’s tracing client for Ruby. It is used to trace requests
   as they flow across web servers, databases and microservices so that developers
@@ -309,6 +295,7 @@ files:
 - ".circleci/images/primary/Dockerfile-2.5.6"
 - ".circleci/images/primary/Dockerfile-2.6.4"
 - ".circleci/images/primary/Dockerfile-2.7.0"
+- ".circleci/images/primary/Dockerfile-jruby-9.2"
 - ".dockerignore"
 - ".env"
 - ".github/CODEOWNERS"
@@ -339,8 +326,10 @@ files:
 - lib/ddtrace/augmentation/method_wrapping.rb
 - lib/ddtrace/augmentation/shim.rb
 - lib/ddtrace/buffer.rb
+- lib/ddtrace/chunker.rb
 - lib/ddtrace/configuration.rb
 - lib/ddtrace/configuration/base.rb
+- lib/ddtrace/configuration/components.rb
 - lib/ddtrace/configuration/dependency_resolver.rb
 - lib/ddtrace/configuration/option.rb
 - lib/ddtrace/configuration/option_definition.rb
@@ -700,6 +689,8 @@ files:
 - lib/ddtrace/workers/loop.rb
 - lib/ddtrace/workers/polling.rb
 - lib/ddtrace/workers/queue.rb
+- lib/ddtrace/workers/runtime_metrics.rb
+- lib/ddtrace/workers/trace_writer.rb
 - lib/ddtrace/writer.rb
 - tasks/release_gem.rake
 homepage: https://github.com/DataDog/dd-trace-rb
@@ -722,8 +713,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: 2.0.0
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.14
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: Datadog tracing code for your Ruby applications