ddtrace 1.1.0 → 1.9.0

data/lib/datadog/opentracer/thread_local_scope_manager.rb

@@ -1,10 +1,19 @@
-# typed: true
+# typed: false
+
+require_relative '../core/utils/sequence'
 
 module Datadog
   module OpenTracer
     # OpenTracing adapter for thread local scope management
     # @public_api
     class ThreadLocalScopeManager < ScopeManager
+      def initialize(*args, &block)
+        super(*args, &block)
+        @thread_key = "dd_opentracer_context_#{ThreadLocalScopeManager.next_instance_id}".to_sym
+      end
+
+      ruby2_keywords :initialize if respond_to?(:ruby2_keywords, true)
+
       # Make a span instance active.
       #
       # @param span [Span] the Span that should become active
@@ -30,13 +39,27 @@ module Datadog
       # (as Reference#CHILD_OF) of any newly-created Span at Tracer#start_active_span
       # or Tracer#start_span time.
       def active
-        Thread.current[object_id.to_s]
+        Thread.current[@thread_key]
+      end
+
+      # Ensure two instances of {FiberLocalContext} do not conflict.
+      # We previously used {FiberLocalContext#object_id} to ensure uniqueness
+      # but the VM is allowed to reuse `object_id`, allow for the possibility that
+      # a new FiberLocalContext would be able to read an old FiberLocalContext's
+      # value.
+      UNIQUE_INSTANCE_MUTEX = Mutex.new
+      UNIQUE_INSTANCE_GENERATOR = Datadog::Core::Utils::Sequence.new
+
+      private_constant :UNIQUE_INSTANCE_MUTEX, :UNIQUE_INSTANCE_GENERATOR
+
+      def self.next_instance_id
+        UNIQUE_INSTANCE_MUTEX.synchronize { UNIQUE_INSTANCE_GENERATOR.next }
       end
 
       private
 
       def set_scope(scope)
-        Thread.current[object_id.to_s] = scope
+        Thread.current[@thread_key] = scope
       end
     end
   end

data/lib/datadog/opentracer/tracer.rb

@@ -2,8 +2,8 @@
 
 require 'time'
 
-require 'datadog/tracing/context'
-require 'datadog/tracing/tracer'
+require_relative '../tracing/context'
+require_relative '../tracing/tracer'
 
 module Datadog
   module OpenTracer
@@ -52,13 +52,15 @@ module Datadog
       #   yield the newly-started Scope. If `finish_on_close` is true then the
       #   Span will be finished automatically after the block is executed.
       # @return [Scope] The newly-started and activated Scope
-      def start_active_span(operation_name,
-                            child_of: nil,
-                            references: nil,
-                            start_time: Time.now,
-                            tags: nil,
-                            ignore_active_scope: false,
-                            finish_on_close: true)
+      def start_active_span(
+        operation_name,
+        child_of: nil,
+        references: nil,
+        start_time: Time.now,
+        tags: nil,
+        ignore_active_scope: false,
+        finish_on_close: true
+      )
 
         # When meant to automatically determine the parent,
         # Use the active scope first, otherwise fall back to any
@@ -124,12 +126,14 @@ module Datadog
       #   References#CHILD_OF reference to the ScopeManager#active.
       # @return [Span] the newly-started Span instance, which has not been
       #   automatically registered via the ScopeManager
-      def start_span(operation_name,
-                     child_of: nil,
-                     references: nil,
-                     start_time: Time.now,
-                     tags: nil,
-                     ignore_active_scope: false)
+      def start_span(
+        operation_name,
+        child_of: nil,
+        references: nil,
+        start_time: Time.now,
+        tags: nil,
+        ignore_active_scope: false
+      )
 
         # Derive the OpenTracer::SpanContext to inherit from.
         parent_span_context = inherited_span_context(child_of, ignore_active_scope: ignore_active_scope)
@@ -147,12 +151,10 @@ module Datadog
           tags: tags || {}
         )
 
-        # Build or extend the OpenTracer::SpanContext
-        span_context = if parent_span_context
-                         SpanContextFactory.clone(span_context: parent_span_context)
-                       else
-                         SpanContextFactory.build(datadog_context: datadog_context)
-                       end
+        span_context = SpanContextFactory.build(
+          datadog_context: datadog_context || datadog_tracer.send(:call_context),
+          baggage: parent_span_context ? parent_span_context.baggage.dup : {}
+        )
 
         # Wrap the Datadog span and OpenTracer::Span context in a OpenTracer::Span
         Span.new(datadog_span: datadog_span, span_context: span_context)

data/lib/datadog/opentracer.rb

@@ -2,23 +2,23 @@
 
 require 'opentracing'
 require 'opentracing/carrier'
-require 'datadog/tracing'
+require_relative 'tracing'
 
-require 'datadog/opentracer/carrier'
-require 'datadog/opentracer/tracer'
-require 'datadog/opentracer/span'
-require 'datadog/opentracer/span_context'
-require 'datadog/opentracer/span_context_factory'
-require 'datadog/opentracer/scope'
-require 'datadog/opentracer/scope_manager'
-require 'datadog/opentracer/thread_local_scope'
-require 'datadog/opentracer/thread_local_scope_manager'
-require 'datadog/opentracer/distributed_headers'
-require 'datadog/opentracer/propagator'
-require 'datadog/opentracer/text_map_propagator'
-require 'datadog/opentracer/binary_propagator'
-require 'datadog/opentracer/rack_propagator'
-require 'datadog/opentracer/global_tracer'
+require_relative 'opentracer/carrier'
+require_relative 'opentracer/tracer'
+require_relative 'opentracer/span'
+require_relative 'opentracer/span_context'
+require_relative 'opentracer/span_context_factory'
+require_relative 'opentracer/scope'
+require_relative 'opentracer/scope_manager'
+require_relative 'opentracer/thread_local_scope'
+require_relative 'opentracer/thread_local_scope_manager'
+require_relative 'opentracer/distributed_headers'
+require_relative 'opentracer/propagator'
+require_relative 'opentracer/text_map_propagator'
+require_relative 'opentracer/binary_propagator'
+require_relative 'opentracer/rack_propagator'
+require_relative 'opentracer/global_tracer'
 
 # Modify the OpenTracing module functions
 ::OpenTracing.singleton_class.prepend(Datadog::OpenTracer::GlobalTracer)

data/lib/datadog/profiling/buffer.rb

@@ -1,8 +1,8 @@
 # typed: true
 
-require 'datadog/core/buffer/thread_safe'
-require 'datadog/core/utils/object_set'
-require 'datadog/core/utils/string_table'
+require_relative '../core/buffer/thread_safe'
+require_relative '../core/utils/object_set'
+require_relative '../core/utils/string_table'
 
 module Datadog
   module Profiling

data/lib/datadog/profiling/collectors/code_provenance.rb

@@ -14,6 +14,7 @@ module Datadog
       #
       # This metadata powers grouping and categorization of stack trace data.
       #
+      # This class acts both as a collector (collecting data) as well as a recorder (records/serializes it)
       class CodeProvenance
         def initialize(standard_library_path: RbConfig::CONFIG.fetch('rubylibdir'))
           @libraries_by_name = {}

data/lib/datadog/profiling/collectors/cpu_and_wall_time.rb

@@ -0,0 +1,43 @@
+# typed: false
+
+module Datadog
+  module Profiling
+    module Collectors
+      # Used to periodically sample threads, recording elapsed CPU-time and Wall-time between samples.
+      # Triggering of this component (e.g. deciding when to take a sample) is implemented in
+      # Collectors::CpuAndWallTimeWorker.
+      # The stack collection itself is handled using the Datadog::Profiling::Collectors::Stack.
+      # Almost all of this class is implemented as native code.
+      #
+      # Methods prefixed with _native_ are implemented in `collectors_cpu_and_wall_time.c`
+      class CpuAndWallTime
+        def initialize(recorder:, max_frames:, tracer:)
+          tracer_context_key = safely_extract_context_key_from(tracer)
+          self.class._native_initialize(self, recorder, max_frames, tracer_context_key)
+        end
+
+        def inspect
+          # Compose Ruby's default inspect with our custom inspect for the native parts
+          result = super()
+          result[-1] = "#{self.class._native_inspect(self)}>"
+          result
+        end
+
+        def reset_after_fork
+          self.class._native_reset_after_fork(self)
+        end
+
+        private
+
+        def safely_extract_context_key_from(tracer)
+          provider = tracer && tracer.respond_to?(:provider) && tracer.provider
+
+          return unless provider
+
+          context = provider.instance_variable_get(:@context)
+          context && context.instance_variable_get(:@key)
+        end
+      end
+    end
+  end
+end

data/lib/datadog/profiling/collectors/cpu_and_wall_time_worker.rb

@@ -0,0 +1,91 @@
+# typed: false
+
+module Datadog
+  module Profiling
+    module Collectors
+      # Used to trigger the periodic execution of Collectors::CpuAndWallTime, which implements all of the sampling logic
+      # itself; this class only implements the "doing it periodically" part.
+      # Almost all of this class is implemented as native code.
+      #
+      # Methods prefixed with _native_ are implemented in `collectors_cpu_and_wall_time_worker.c`
+      class CpuAndWallTimeWorker
+        private
+
+        attr_accessor :failure_exception
+
+        public
+
+        def initialize(
+          recorder:,
+          max_frames:,
+          tracer:,
+          gc_profiling_enabled:,
+          cpu_and_wall_time_collector: CpuAndWallTime.new(recorder: recorder, max_frames: max_frames, tracer: tracer),
+          idle_sampling_helper: IdleSamplingHelper.new
+        )
+          self.class._native_initialize(self, cpu_and_wall_time_collector, gc_profiling_enabled, idle_sampling_helper)
+          @worker_thread = nil
+          @failure_exception = nil
+          @start_stop_mutex = Mutex.new
+          @idle_sampling_helper = idle_sampling_helper
+        end
+
+        def start
+          @start_stop_mutex.synchronize do
+            return if @worker_thread && @worker_thread.alive?
+
+            Datadog.logger.debug { "Starting thread for: #{self}" }
+
+            @idle_sampling_helper.start
+
+            @worker_thread = Thread.new do
+              begin
+                Thread.current.name = self.class.name unless Gem::Version.new(RUBY_VERSION) < Gem::Version.new('2.3')
+
+                self.class._native_sampling_loop(self)
+
+                Datadog.logger.debug('CpuAndWallTimeWorker thread stopping cleanly')
+              rescue Exception => e # rubocop:disable Lint/RescueException
+                @failure_exception = e
+                Datadog.logger.warn(
+                  'CpuAndWallTimeWorker thread error. ' \
+                  "Cause: #{e.class.name} #{e.message} Location: #{Array(e.backtrace).first}"
+                )
+              end
+            end
+          end
+
+          true
+        end
+
+        # TODO: Provided only for compatibility with the API for Collectors::OldStack used in the Profiler class.
+        # Can be removed once we remove OldStack.
+        def enabled=(_); end
+
+        def stop(*_)
+          @start_stop_mutex.synchronize do
+            Datadog.logger.debug('Requesting CpuAndWallTimeWorker thread shut down')
+
+            @idle_sampling_helper.stop
+
+            return unless @worker_thread
+
+            self.class._native_stop(self, @worker_thread)
+
+            @worker_thread.join
+            @worker_thread = nil
+            @failure_exception = nil
+          end
+        end
+
+        def reset_after_fork
+          self.class._native_reset_after_fork(self)
+        end
+
+        def stats
+          self.class._native_stats(self)
+        end
+      end
+    end
+  end
+end

data/lib/datadog/profiling/collectors/dynamic_sampling_rate.rb

@@ -0,0 +1,14 @@
+# typed: false
+
+module Datadog
+  module Profiling
+    module Collectors
+      # Used to pace the rate of profiling samples based on the last observed time for a sample.
+      # All of this module is implemented as native code.
+      #
+      # Methods prefixed with _native_ are implemented in `collectors_dynamic_sampling_rate.c`
+      module DynamicSamplingRate
+      end
+    end
+  end
+end

data/lib/datadog/profiling/collectors/idle_sampling_helper.rb

@@ -0,0 +1,68 @@
+# typed: false
+
+module Datadog
+  module Profiling
+    module Collectors
+      # Used by the Collectors::CpuAndWallTimeWorker to gather samples when the Ruby process is idle.
+      # Almost all of this class is implemented as native code.
+      #
+      # Methods prefixed with _native_ are implemented in `collectors_idle_sampling_helper.c`
+      class IdleSamplingHelper
+        private
+
+        attr_accessor :failure_exception
+
+        public
+
+        def initialize
+          @worker_thread = nil
+          @start_stop_mutex = Mutex.new
+        end
+
+        def start
+          @start_stop_mutex.synchronize do
+            return if @worker_thread && @worker_thread.alive?
+
+            Datadog.logger.debug { "Starting thread for: #{self}" }
+
+            # The same instance of the IdleSamplingHelper can be reused multiple times, and this resets it back to
+            # a pristine state before recreating the worker thread
+            self.class._native_reset(self)
+
+            @worker_thread = Thread.new do
+              begin
+                Thread.current.name = self.class.name unless Gem::Version.new(RUBY_VERSION) < Gem::Version.new('2.3')
+
+                self.class._native_idle_sampling_loop(self)
+
+                Datadog.logger.debug('IdleSamplingHelper thread stopping cleanly')
+              rescue Exception => e # rubocop:disable Lint/RescueException
+                @failure_exception = e
+                Datadog.logger.warn(
+                  'IdleSamplingHelper thread error. ' \
+                  "Cause: #{e.class.name} #{e.message} Location: #{Array(e.backtrace).first}"
+                )
+              end
+            end
+          end
+
+          true
+        end
+
+        def stop(*_)
+          @start_stop_mutex.synchronize do
+            Datadog.logger.debug('Requesting IdleSamplingHelper thread shut down')
+
+            return unless @worker_thread
+
+            self.class._native_stop(self)
+
+            @worker_thread.join
+            @worker_thread = nil
+            @failure_exception = nil
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/profiling/collectors/old_stack.rb

@@ -1,12 +1,12 @@
 # typed: true
 
-require 'datadog/core/utils/only_once'
-require 'datadog/core/utils/time'
-require 'datadog/core/worker'
-require 'datadog/core/workers/polling'
-require 'datadog/profiling/backtrace_location'
-require 'datadog/profiling/events/stack'
-require 'datadog/profiling/native_extension'
+require_relative '../../core/utils/only_once'
+require_relative '../../core/utils/time'
+require_relative '../../core/worker'
+require_relative '../../core/workers/polling'
+require_relative '../backtrace_location'
+require_relative '../events/stack'
+require_relative '../native_extension'
 
 module Datadog
   module Profiling
@@ -15,7 +15,7 @@ module Datadog
       # Runs on its own background thread.
       #
       # This class has the prefix "Old" because it will be deprecated by the new native CPU Profiler
-      class OldStack < Core::Worker # rubocop:disable Metrics/ClassLength
+      class OldStack < Core::Worker
         include Core::Workers::Polling
 
         DEFAULT_MAX_TIME_USAGE_PCT = 2.0
@@ -218,6 +218,13 @@ module Datadog
           )
         end
 
+        def reset_after_fork
+          recorder.reset_after_fork
+
+          # NOTE: We could perhaps also call #reset_cpu_time_tracking here, although it's not needed because we always
+          # call in in #start.
+        end
+
         private
 
         # If the profiler is started for a while, stopped and then restarted OR whenever the process forks, we need to

data/lib/datadog/profiling/collectors/stack.rb

@@ -3,13 +3,10 @@
 module Datadog
   module Profiling
     module Collectors
-      # Used to gather a stack trace from a given Ruby thread. Almost all of this class is implemented as native code.
+      # Used to gather a stack trace from a given Ruby thread. Stores its output on a `StackRecorder`.
       #
-      # Methods prefixed with _native_ are implemented in `collectors_stack.c`
-      class Stack
-        def sample(thread, recorder_instance, metric_values_hash, labels_array, max_frames: 400)
-          self.class._native_sample(thread, recorder_instance, metric_values_hash, labels_array, max_frames)
-        end
+      # This class is not empty; all of this class is implemented as native code.
+      class Stack # rubocop:disable Lint/EmptyClass
       end
     end
   end

data/lib/datadog/profiling/encoding/profile.rb

@@ -1,10 +1,8 @@
 # typed: true
 
-require 'set'
 require 'time'
 
-require 'datadog/profiling/flush'
-require 'datadog/profiling/pprof/template'
+require_relative '../pprof/template'
 
 module Datadog
   module Profiling
@@ -14,31 +12,29 @@ module Datadog
         module Protobuf
           module_function
 
-          def encode(flush)
-            return unless flush
-
+          def encode(event_count:, event_groups:, start:, finish:)
             # Create a pprof template from the list of event types
-            event_classes = flush.event_groups.collect(&:event_class).uniq
+            event_classes = event_groups.collect(&:event_class).uniq
             template = Pprof::Template.for_event_classes(event_classes)
 
             # Add all events to the pprof
-            flush.event_groups.each { |event_group| template.add_events!(event_group.event_class, event_group.events) }
+            event_groups.each { |event_group| template.add_events!(event_group.event_class, event_group.events) }
 
             Datadog.logger.debug do
               max_events = Datadog.configuration.profiling.advanced.max_events
               events_sampled =
-                if flush.event_count == max_events
+                if event_count == max_events
                   'max events limit hit, events were sampled [profile will be biased], '
                 else
                   ''
                 end
 
-              "Encoding profile covering #{flush.start.iso8601} to #{flush.finish.iso8601}, " \
-              "events: #{flush.event_count} (#{events_sampled}#{template.debug_statistics})"
+              "Encoding profile covering #{start.iso8601} to #{finish.iso8601}, " \
+              "events: #{event_count} (#{events_sampled}#{template.debug_statistics})"
             end
 
             # Build the profile and encode it
-            template.to_pprof(start: flush.start, finish: flush.finish)
+            template.to_pprof(start: start, finish: finish)
           end
         end
       end

data/lib/datadog/profiling/events/stack.rb

@@ -1,6 +1,6 @@
 # typed: true
 
-require 'datadog/profiling/event'
+require_relative '../event'
 
 module Datadog
   module Profiling

data/lib/datadog/profiling/exporter.rb

@@ -1,24 +1,84 @@
 # typed: true
 
-require 'datadog/profiling/transport/io/client'
+require_relative 'ext'
+require_relative 'tag_builder'
 
 module Datadog
   module Profiling
-    # Writes profiling data to a given transport
+    # Exports profiling data gathered by the multiple recorders in a `Flush`.
+    #
+    # @ivoanjo: Note that the recorder that gathers pprof data is special, since we use its start/finish/empty? to
+    # decide if there's data to flush, as well as the timestamp for that data.
+    # I could've made the whole design more generic, but I'm unsure if we'll ever have more than a handful of
+    # recorders, so I've decided to make it specific until we actually need to support more recorders.
+    #
     class Exporter
+      # Profiles with duration less than this will not be reported
+      PROFILE_DURATION_THRESHOLD_SECONDS = 1
+
+      private
+
       attr_reader \
-        :transport
+        :pprof_recorder,
+        :code_provenance_collector, # The code provenance collector acts both as collector and as a recorder
+        :minimum_duration_seconds,
+        :time_provider,
+        :last_flush_finish_at,
+        :created_at
+
+      public
+
+      def initialize(
+        pprof_recorder:,
+        code_provenance_collector:,
+        minimum_duration_seconds: PROFILE_DURATION_THRESHOLD_SECONDS,
+        time_provider: Time
+      )
+        @pprof_recorder = pprof_recorder
+        @code_provenance_collector = code_provenance_collector
+        @minimum_duration_seconds = minimum_duration_seconds
+        @time_provider = time_provider
+        @last_flush_finish_at = nil
+        @created_at = time_provider.now.utc
+      end
+
+      def flush
+        start, finish, uncompressed_pprof = pprof_recorder.serialize
+        @last_flush_finish_at = finish
+
+        return if uncompressed_pprof.nil? # We don't want to report empty profiles
 
-      def initialize(transport)
-        unless transport.is_a?(Profiling::Transport::Client)
-          raise ArgumentError, 'Unsupported transport for profiling exporter.'
+        if duration_below_threshold?(start, finish)
+          Datadog.logger.debug('Skipped exporting profiling events as profile duration is below minimum')
+          return
         end
 
-        @transport = transport
+        uncompressed_code_provenance = code_provenance_collector.refresh.generate_json if code_provenance_collector
+
+        Flush.new(
+          start: start,
+          finish: finish,
+          pprof_file_name: Datadog::Profiling::Ext::Transport::HTTP::PPROF_DEFAULT_FILENAME,
+          pprof_data: uncompressed_pprof.to_s,
+          code_provenance_file_name: Datadog::Profiling::Ext::Transport::HTTP::CODE_PROVENANCE_FILENAME,
+          code_provenance_data: uncompressed_code_provenance,
+          tags_as_array: Datadog::Profiling::TagBuilder.call(settings: Datadog.configuration).to_a,
+        )
+      end
+
+      def can_flush?
+        !duration_below_threshold?(last_flush_finish_at || created_at, time_provider.now.utc)
       end
 
-      def export(flush)
-        transport.send_profiling_flush(flush)
+      def reset_after_fork
+        @last_flush_finish_at = time_provider.now.utc
+        nil
+      end
+
+      private
+
+      def duration_below_threshold?(start, finish)
+        (finish - start) < minimum_duration_seconds
       end
     end
   end