ddtrace 1.2.0 → 1.3.0

data/lib/datadog/opentracer/text_map_propagator.rb

@@ -1,9 +1,9 @@
 # typed: true
 
-require 'datadog/tracing/context'
-require 'datadog/tracing/distributed/headers/ext'
-require 'datadog/tracing/trace_operation'
-require 'datadog/opentracer/propagator'
+require_relative '../tracing/context'
+require_relative '../tracing/distributed/headers/ext'
+require_relative '../tracing/trace_operation'
+require_relative 'propagator'
 
 module Datadog
   module OpenTracer
@@ -27,8 +27,11 @@ module Datadog
           end
 
           # Inject Datadog trace properties
-          active_trace = span_context.datadog_context.active_trace
-          digest = active_trace.to_digest if active_trace
+          digest = if span_context.datadog_context && span_context.datadog_context.active_trace
+                     span_context.datadog_context.active_trace.to_digest
+                   else
+                     span_context.datadog_trace_digest
+                   end
           return unless digest
 
           carrier[HTTP_HEADER_ORIGIN] = digest.trace_origin

data/lib/datadog/opentracer/thread_local_scope_manager.rb

@@ -1,6 +1,6 @@
 # typed: false
 
-require 'datadog/core/utils/sequence'
+require_relative '../core/utils/sequence'
 
 module Datadog
   module OpenTracer

data/lib/datadog/opentracer/tracer.rb

@@ -2,8 +2,8 @@
 
 require 'time'
 
-require 'datadog/tracing/context'
-require 'datadog/tracing/tracer'
+require_relative '../tracing/context'
+require_relative '../tracing/tracer'
 
 module Datadog
   module OpenTracer
@@ -52,13 +52,15 @@ module Datadog
       #   yield the newly-started Scope. If `finish_on_close` is true then the
       #   Span will be finished automatically after the block is executed.
       # @return [Scope] The newly-started and activated Scope
-      def start_active_span(operation_name,
-                            child_of: nil,
-                            references: nil,
-                            start_time: Time.now,
-                            tags: nil,
-                            ignore_active_scope: false,
-                            finish_on_close: true)
+      def start_active_span(
+        operation_name,
+        child_of: nil,
+        references: nil,
+        start_time: Time.now,
+        tags: nil,
+        ignore_active_scope: false,
+        finish_on_close: true
+      )
 
         # When meant to automatically determine the parent,
         # Use the active scope first, otherwise fall back to any
@@ -124,12 +126,14 @@ module Datadog
       #   References#CHILD_OF reference to the ScopeManager#active.
       # @return [Span] the newly-started Span instance, which has not been
       #   automatically registered via the ScopeManager
-      def start_span(operation_name,
-                     child_of: nil,
-                     references: nil,
-                     start_time: Time.now,
-                     tags: nil,
-                     ignore_active_scope: false)
+      def start_span(
+        operation_name,
+        child_of: nil,
+        references: nil,
+        start_time: Time.now,
+        tags: nil,
+        ignore_active_scope: false
+      )
 
         # Derive the OpenTracer::SpanContext to inherit from.
         parent_span_context = inherited_span_context(child_of, ignore_active_scope: ignore_active_scope)

data/lib/datadog/opentracer.rb

@@ -2,23 +2,23 @@
 
 require 'opentracing'
 require 'opentracing/carrier'
-require 'datadog/tracing'
+require_relative 'tracing'
 
-require 'datadog/opentracer/carrier'
-require 'datadog/opentracer/tracer'
-require 'datadog/opentracer/span'
-require 'datadog/opentracer/span_context'
-require 'datadog/opentracer/span_context_factory'
-require 'datadog/opentracer/scope'
-require 'datadog/opentracer/scope_manager'
-require 'datadog/opentracer/thread_local_scope'
-require 'datadog/opentracer/thread_local_scope_manager'
-require 'datadog/opentracer/distributed_headers'
-require 'datadog/opentracer/propagator'
-require 'datadog/opentracer/text_map_propagator'
-require 'datadog/opentracer/binary_propagator'
-require 'datadog/opentracer/rack_propagator'
-require 'datadog/opentracer/global_tracer'
+require_relative 'opentracer/carrier'
+require_relative 'opentracer/tracer'
+require_relative 'opentracer/span'
+require_relative 'opentracer/span_context'
+require_relative 'opentracer/span_context_factory'
+require_relative 'opentracer/scope'
+require_relative 'opentracer/scope_manager'
+require_relative 'opentracer/thread_local_scope'
+require_relative 'opentracer/thread_local_scope_manager'
+require_relative 'opentracer/distributed_headers'
+require_relative 'opentracer/propagator'
+require_relative 'opentracer/text_map_propagator'
+require_relative 'opentracer/binary_propagator'
+require_relative 'opentracer/rack_propagator'
+require_relative 'opentracer/global_tracer'
 
 # Modify the OpenTracing module functions
 ::OpenTracing.singleton_class.prepend(Datadog::OpenTracer::GlobalTracer)

data/lib/datadog/profiling/buffer.rb

@@ -1,8 +1,8 @@
 # typed: true
 
-require 'datadog/core/buffer/thread_safe'
-require 'datadog/core/utils/object_set'
-require 'datadog/core/utils/string_table'
+require_relative '../core/buffer/thread_safe'
+require_relative '../core/utils/object_set'
+require_relative '../core/utils/string_table'
 
 module Datadog
   module Profiling

data/lib/datadog/profiling/collectors/cpu_and_wall_time.rb

@@ -3,8 +3,11 @@
 module Datadog
   module Profiling
     module Collectors
-      # Used to periodically (time-based) sample threads, recording elapsed CPU-time and Wall-time between samples.
+      # Used to periodically sample threads, recording elapsed CPU-time and Wall-time between samples.
+      # Triggering of this component (e.g. deciding when to take a sample) is implemented in
+      # Collectors::CpuAndWallTimeWorker.
       # The stack collection itself is handled using the Datadog::Profiling::Collectors::Stack.
+      # Almost all of this class is implemented as native code.
       #
       # Methods prefixed with _native_ are implemented in `collectors_cpu_and_wall_time.c`
       class CpuAndWallTime
@@ -12,24 +15,6 @@ module Datadog
           self.class._native_initialize(self, recorder, max_frames)
         end
 
-        # This method exists only to enable testing Datadog::Profiling::Collectors::CpuAndWallTime behavior using RSpec.
-        # It SHOULD NOT be used for other purposes.
-        def sample
-          self.class._native_sample(self)
-        end
-
-        # This method exists only to enable testing Datadog::Profiling::Collectors::CpuAndWallTime behavior using RSpec.
-        # It SHOULD NOT be used for other purposes.
-        def thread_list
-          self.class._native_thread_list
-        end
-
-        # This method exists only to enable testing Datadog::Profiling::Collectors::CpuAndWallTime behavior using RSpec.
-        # It SHOULD NOT be used for other purposes.
-        def per_thread_context
-          self.class._native_per_thread_context(self)
-        end
-
         def inspect
           # Compose Ruby's default inspect with our custom inspect for the native parts
           result = super()

data/lib/datadog/profiling/collectors/old_stack.rb

@@ -1,12 +1,12 @@
 # typed: true
 
-require 'datadog/core/utils/only_once'
-require 'datadog/core/utils/time'
-require 'datadog/core/worker'
-require 'datadog/core/workers/polling'
-require 'datadog/profiling/backtrace_location'
-require 'datadog/profiling/events/stack'
-require 'datadog/profiling/native_extension'
+require_relative '../../core/utils/only_once'
+require_relative '../../core/utils/time'
+require_relative '../../core/worker'
+require_relative '../../core/workers/polling'
+require_relative '../backtrace_location'
+require_relative '../events/stack'
+require_relative '../native_extension'
 
 module Datadog
   module Profiling

data/lib/datadog/profiling/collectors/stack.rb

@@ -3,15 +3,10 @@
 module Datadog
   module Profiling
     module Collectors
-      # Used to gather a stack trace from a given Ruby thread. Almost all of this class is implemented as native code.
+      # Used to gather a stack trace from a given Ruby thread. Stores its output on a `StackRecorder`.
       #
-      # Methods prefixed with _native_ are implemented in `collectors_stack.c`
-      class Stack
-        # This method exists only to enable testing Datadog::Profiling::Collectors::Stack behavior using RSpec.
-        # It SHOULD NOT be used for other purposes.
-        def sample(thread, recorder_instance, metric_values_hash, labels_array, max_frames: 400)
-          self.class._native_sample(thread, recorder_instance, metric_values_hash, labels_array, max_frames)
-        end
+      # This class is not empty; all of this class is implemented as native code.
+      class Stack # rubocop:disable Lint/EmptyClass
       end
     end
   end

data/lib/datadog/profiling/encoding/profile.rb

@@ -2,7 +2,7 @@
 
 require 'time'
 
-require 'datadog/profiling/pprof/template'
+require_relative '../pprof/template'
 
 module Datadog
   module Profiling

data/lib/datadog/profiling/events/stack.rb

@@ -1,6 +1,6 @@
 # typed: true
 
-require 'datadog/profiling/event'
+require_relative '../event'
 
 module Datadog
   module Profiling

data/lib/datadog/profiling/exporter.rb

@@ -1,8 +1,8 @@
 # typed: true
 
-require 'datadog/profiling/ext'
-require 'datadog/core/utils/compression'
-require 'datadog/profiling/tag_builder'
+require_relative 'ext'
+require_relative '../core/utils/compression'
+require_relative 'tag_builder'
 
 module Datadog
   module Profiling
@@ -22,22 +22,30 @@ module Datadog
       attr_reader \
         :pprof_recorder,
         :code_provenance_collector, # The code provenance collector acts both as collector and as a recorder
-        :minimum_duration
+        :minimum_duration_seconds,
+        :time_provider,
+        :last_flush_finish_at,
+        :created_at
 
       public
 
       def initialize(
         pprof_recorder:,
         code_provenance_collector:,
-        minimum_duration: PROFILE_DURATION_THRESHOLD_SECONDS
+        minimum_duration_seconds: PROFILE_DURATION_THRESHOLD_SECONDS,
+        time_provider: Time
       )
         @pprof_recorder = pprof_recorder
         @code_provenance_collector = code_provenance_collector
-        @minimum_duration = minimum_duration
+        @minimum_duration_seconds = minimum_duration_seconds
+        @time_provider = time_provider
+        @last_flush_finish_at = nil
+        @created_at = time_provider.now.utc
       end
 
       def flush
         start, finish, uncompressed_pprof = pprof_recorder.serialize
+        @last_flush_finish_at = finish
 
         return if uncompressed_pprof.nil? # We don't want to report empty profiles
 
@@ -60,14 +68,14 @@ module Datadog
         )
       end
 
-      def empty?
-        pprof_recorder.empty?
+      def can_flush?
+        !duration_below_threshold?(last_flush_finish_at || created_at, time_provider.now.utc)
       end
 
       private
 
       def duration_below_threshold?(start, finish)
-        (finish - start) < @minimum_duration
+        (finish - start) < minimum_duration_seconds
       end
     end
   end

data/lib/datadog/profiling/ext/forking.rb

@@ -5,6 +5,11 @@ module Datadog
     module Ext
       # Monkey patches `Kernel#fork`, adding a `Kernel#at_fork` callback mechanism which is used to restore
       # profiling abilities after the VM forks.
+      #
+      # Known limitations: Does not handle `BasicObject`s that include `Kernel` directly; e.g.
+      # `Class.new(BasicObject) { include(::Kernel); def call; fork { }; end }.new.call`.
+      #
+      # This will be fixed once we moved to hooking into `Process._fork`
       module Forking
         def self.supported?
           Process.respond_to?(:fork)
@@ -13,33 +18,22 @@ module Datadog
         def self.apply!
           return false unless supported?
 
-          modules = [::Process, ::Kernel]
-          # TODO: Ruby < 2.3 doesn't support Binding#receiver.
-          #       Remove "else #eval" clause when Ruby < 2.3 support is dropped.
-          # NOTE: Modifying the "main" object as we do here is (as far as I know) irreversible. During tests, this change
-          #       will stick around even if we otherwise stub `Process` and `Kernel`.
-          modules << (TOPLEVEL_BINDING.respond_to?(:receiver) ? TOPLEVEL_BINDING.receiver : TOPLEVEL_BINDING.eval('self'))
-
-          # Patch top-level binding, Kernel, Process.
-          # NOTE: We could instead do Kernel.module_eval { def fork; ... end }
-          #       however, this method rewrite is more invasive and irreversible.
-          #       It could also have collisions with other libraries that patch.
-          #       Opt to modify the inheritance of each relevant target instead.
-          modules.each do |mod|
-            clazz = if mod.class <= Module
-                      mod.singleton_class
-                    else
-                      mod.class
-                    end
-
-            clazz.prepend(Kernel)
-          end
+          [
+            ::Process.singleton_class, # Process.fork
+            ::Kernel.singleton_class,  # Kernel.fork
+            ::Object,                  # fork without explicit receiver (it's defined as a method in ::Kernel)
+            # Note: Modifying Object as we do here is irreversible. During tests, this
+            # change will stick around even if we otherwise stub `Process` and `Kernel`
+          ].each { |target| target.prepend(Kernel) }
+
+          ::Process.singleton_class.prepend(ProcessDaemonPatch)
         end
 
         # Extensions for kernel
+        #
+        # TODO: Consider hooking into `Process._fork` on Ruby 3.1+ instead, see
+        #       https://github.com/ruby/ruby/pull/5017 and https://bugs.ruby-lang.org/issues/17795
         module Kernel
-          FORK_STAGES = [:prepare, :parent, :child].freeze
-
           def fork
             # If a block is provided, it must be wrapped to trigger callbacks.
             child_block = if block_given?
@@ -52,9 +46,6 @@ module Datadog
                             end
                           end
 
-            # Trigger :prepare callback
-            ddtrace_at_fork_blocks[:prepare].each(&:call) if ddtrace_at_fork_blocks.key?(:prepare)
-
             # Start fork
             # If a block is provided, use the wrapped version.
             result = child_block.nil? ? super : super(&child_block)
@@ -62,22 +53,14 @@ module Datadog
             # Trigger correct callbacks depending on whether we're in the parent or child.
             # If we're in the fork, result = nil: trigger child callbacks.
             # If we're in the parent, result = fork PID: trigger parent callbacks.
-            # rubocop:disable Style/IfInsideElse
-            if result.nil?
-              # Trigger :child callback
-              ddtrace_at_fork_blocks[:child].each(&:call) if ddtrace_at_fork_blocks.key?(:child)
-            else
-              # Trigger :parent callback
-              ddtrace_at_fork_blocks[:parent].each(&:call) if ddtrace_at_fork_blocks.key?(:parent)
-            end
-            # rubocop:enable Style/IfInsideElse
+            ddtrace_at_fork_blocks[:child].each(&:call) if result.nil? && ddtrace_at_fork_blocks.key?(:child)
 
             # Return PID from #fork
             result
           end
 
-          def at_fork(stage = :prepare, &block)
-            raise ArgumentError, 'Bad \'stage\' for ::at_fork' unless FORK_STAGES.include?(stage)
+          def at_fork(stage, &block)
+            raise ArgumentError, 'Bad \'stage\' for ::at_fork' unless stage == :child
 
             ddtrace_at_fork_blocks[stage] = [] unless ddtrace_at_fork_blocks.key?(stage)
             ddtrace_at_fork_blocks[stage] << block
@@ -93,6 +76,22 @@ module Datadog
             # rubocop:enable Style/ClassVars
           end
         end
+
+        # A call to Process.daemon ( https://rubyapi.org/3.1/o/process#method-c-daemon ) forks the current process and
+        # keeps executing code in the child process, killing off the parent, thus effectively replacing it.
+        #
+        # This monkey patch makes the `Kernel#at_fork` mechanism defined above also work in this situation.
+        module ProcessDaemonPatch
+          def daemon(*args)
+            ddtrace_at_fork_blocks = Datadog::Profiling::Ext::Forking::Kernel.ddtrace_at_fork_blocks
+
+            result = super
+
+            ddtrace_at_fork_blocks[:child].each(&:call) if ddtrace_at_fork_blocks.key?(:child)
+
+            result
+          end
+        end
       end
     end
   end

data/lib/datadog/profiling/ext.rb

@@ -9,6 +9,7 @@ module Datadog
       ENV_AGENTLESS = 'DD_PROFILING_AGENTLESS'.freeze
       ENV_ENDPOINT_COLLECTION_ENABLED = 'DD_PROFILING_ENDPOINT_COLLECTION_ENABLED'.freeze
 
+      # TODO: Consider removing this once the Ruby-based pprof encoding is removed and replaced by libdatadog
       module Pprof
         LABEL_KEY_LOCAL_ROOT_SPAN_ID = 'local root span id'.freeze
         LABEL_KEY_SPAN_ID = 'span id'.freeze

data/lib/datadog/profiling/flush.rb

@@ -1,8 +1,5 @@
 # typed: false
 
-require 'datadog/core/environment/identity'
-require 'datadog/core/environment/socket'
-
 module Datadog
   module Profiling
     # Represents a collection of events of a specific type being flushed.

data/lib/datadog/profiling/http_transport.rb

@@ -28,7 +28,7 @@ module Datadog
           upload_timeout_milliseconds: @upload_timeout_milliseconds,
 
           # why "timespec"?
-          # libddprof represents time using POSIX's struct timespec, see
+          # libdatadog represents time using POSIX's struct timespec, see
           # https://www.gnu.org/software/libc/manual/html_node/Time-Types.html
           # aka it represents the seconds part separate from the nanoseconds part
           start_timespec_seconds: flush.start.tv_sec,
@@ -80,8 +80,9 @@ module Datadog
       def validate_agent_settings(agent_settings)
         supported_adapters = [Datadog::Transport::Ext::HTTP::ADAPTER, Datadog::Transport::Ext::UnixSocket::ADAPTER]
         unless supported_adapters.include?(agent_settings.adapter)
-          raise ArgumentError, "Unsupported transport configuration for profiling: Adapter #{agent_settings.adapter} " \
-            ' is not supported'
+          raise ArgumentError,
+            "Unsupported transport configuration for profiling: Adapter #{agent_settings.adapter} " \
+                        ' is not supported'
         end
 
         if agent_settings.deprecated_for_removal_transport_configuration_proc

data/lib/datadog/profiling/old_recorder.rb

@@ -1,7 +1,7 @@
 # typed: true
 
-require 'datadog/profiling/buffer'
-require 'datadog/profiling/encoding/profile'
+require_relative 'buffer'
+require_relative 'encoding/profile'
 
 module Datadog
   module Profiling
@@ -73,11 +73,6 @@ module Datadog
         [start, finish, encoded_pprof]
       end
 
-      # NOTE: Remember that if the recorder is being accessed by multiple threads, this is an inherently racy operation.
-      def empty?
-        @buffers.values.all?(&:empty?)
-      end
-
       # Error when event of an unknown type is used with the OldRecorder
       class UnknownEventError < StandardError
         attr_reader :event_class

data/lib/datadog/profiling/pprof/builder.rb

@@ -2,10 +2,10 @@
 
 # typed: true
 
-require 'datadog/profiling/flush'
-require 'datadog/profiling/pprof/message_set'
-require 'datadog/profiling/pprof/string_table'
-require 'datadog/core/utils/time'
+require_relative '../flush'
+require_relative 'message_set'
+require_relative 'string_table'
+require_relative '../../core/utils/time'
 
 module Datadog
   module Profiling

data/lib/datadog/profiling/pprof/converter.rb

@@ -1,6 +1,6 @@
 # typed: true
 
-require 'datadog/profiling/ext'
+require_relative '../ext'
 
 module Datadog
   module Profiling

data/lib/datadog/profiling/pprof/message_set.rb

@@ -1,6 +1,6 @@
 # typed: true
 
-require 'datadog/core/utils/object_set'
+require_relative '../../core/utils/object_set'
 
 module Datadog
   module Profiling

data/lib/datadog/profiling/pprof/stack_sample.rb

@@ -1,9 +1,9 @@
 # typed: true
 
-require 'datadog/profiling/ext'
-require 'datadog/profiling/events/stack'
-require 'datadog/profiling/pprof/builder'
-require 'datadog/profiling/pprof/converter'
+require_relative '../ext'
+require_relative '../events/stack'
+require_relative 'builder'
+require_relative 'converter'
 
 module Datadog
   module Profiling

data/lib/datadog/profiling/pprof/string_table.rb

@@ -1,6 +1,6 @@
 # typed: strict
 
-require 'datadog/core/utils/string_table'
+require_relative '../../core/utils/string_table'
 
 module Datadog
   module Profiling

data/lib/datadog/profiling/pprof/template.rb

@@ -1,11 +1,11 @@
 # typed: false
 
-require 'datadog/profiling/pprof/payload'
-require 'datadog/profiling/pprof/message_set'
-require 'datadog/profiling/pprof/builder'
+require_relative 'payload'
+require_relative 'message_set'
+require_relative 'builder'
 
-require 'datadog/profiling/events/stack'
-require 'datadog/profiling/pprof/stack_sample'
+require_relative '../events/stack'
+require_relative 'stack_sample'
 
 module Datadog
   module Profiling

data/lib/datadog/profiling/preload.rb

@@ -1,5 +1,5 @@
 # typed: strict
 
-require 'ddtrace'
+require_relative '../../ddtrace'
 
 Datadog::Profiling.start_if_enabled

data/lib/datadog/profiling/scheduler.rb

@@ -1,9 +1,9 @@
 # typed: true
 
-require 'datadog/core/utils/time'
+require_relative '../core/utils/time'
 
-require 'datadog/core/worker'
-require 'datadog/core/workers/polling'
+require_relative '../core/worker'
+require_relative '../core/workers/polling'
 
 module Datadog
   module Profiling
@@ -69,6 +69,7 @@ module Datadog
       def after_fork
         # Clear any existing profiling state.
         # We don't want the child process to report profiling data from its parent.
+        Datadog.logger.debug('Flushing exporter in child process #after_fork and discarding data')
         exporter.flush
       end
 
@@ -82,7 +83,7 @@ module Datadog
       end
 
       def work_pending?
-        !exporter.empty?
+        exporter.can_flush?
       end
 
       private

data/lib/datadog/profiling/stack_recorder.rb

@@ -2,11 +2,22 @@
 
 module Datadog
   module Profiling
-    # Used to wrap a ddprof_ffi_Profile in a Ruby object and expose Ruby-level serialization APIs
+    # Stores stack samples in a native libdatadog data structure and expose Ruby-level serialization APIs
+    # Note that `record_sample` is only accessible from native code.
     # Methods prefixed with _native_ are implemented in `stack_recorder.c`
     class StackRecorder
+      def initialize
+        # This mutex works in addition to the fancy C-level mutexes we have in the native side (see the docs there).
+        # It prevents multiple Ruby threads calling serialize at the same time -- something like
+        # `10.times { Thread.new { stack_recorder.serialize } }`.
+        # This isn't something we expect to happen normally, but because it would break the assumptions of the
+        # C-level mutexes (that there is a single serializer thread), we add it here as an extra safeguard against it
+        # accidentally happening.
+        @no_concurrent_synchronize_mutex = Thread::Mutex.new
+      end
+
       def serialize
-        status, result = self.class._native_serialize(self)
+        status, result = @no_concurrent_synchronize_mutex.synchronize { self.class._native_serialize(self) }
 
         if status == :ok
           start, finish, encoded_pprof = result
@@ -23,8 +34,7 @@ module Datadog
         end
       end
 
-      # Used only for Ruby 2.2 and below which don't have the native `rb_time_timespec_new` API
-      # Called from native code
+      # Used only for Ruby 2.2 which doesn't have the native `rb_time_timespec_new` API; called from native code
       def self.ruby_time_from(timespec_seconds, timespec_nanoseconds)
         Time.at(0).utc + timespec_seconds + (timespec_nanoseconds.to_r / 1_000_000_000)
       end