ddtrace 1.2.0 → 1.10.1

data/lib/datadog/opentracer/text_map_propagator.rb

@@ -1,17 +1,13 @@
-# typed: true
-
-require 'datadog/tracing/context'
-require 'datadog/tracing/distributed/headers/ext'
-require 'datadog/tracing/trace_operation'
-require 'datadog/opentracer/propagator'
+require_relative '../tracing/context'
+require_relative '../tracing/distributed/datadog'
+require_relative '../tracing/trace_operation'
+require_relative 'propagator'
 
 module Datadog
   module OpenTracer
     # OpenTracing propagator for Datadog::OpenTracer::Tracer
     module TextMapPropagator
       extend Propagator
-      extend Tracing::Distributed::Headers::Ext
-      include Tracing::Distributed::Headers::Ext
 
       BAGGAGE_PREFIX = 'ot-baggage-'.freeze
 
@@ -27,14 +23,17 @@ module Datadog
           end
 
           # Inject Datadog trace properties
-          active_trace = span_context.datadog_context.active_trace
-          digest = active_trace.to_digest if active_trace
+          digest = if span_context.datadog_context && span_context.datadog_context.active_trace
+                     span_context.datadog_context.active_trace.to_digest
+                   else
+                     span_context.datadog_trace_digest
+                   end
           return unless digest
 
-          carrier[HTTP_HEADER_ORIGIN] = digest.trace_origin
-          carrier[HTTP_HEADER_PARENT_ID] = digest.span_id
-          carrier[HTTP_HEADER_SAMPLING_PRIORITY] = digest.trace_sampling_priority
-          carrier[HTTP_HEADER_TRACE_ID] = digest.trace_id
+          carrier[Tracing::Distributed::Datadog::ORIGIN_KEY] = digest.trace_origin
+          carrier[Tracing::Distributed::Datadog::PARENT_ID_KEY] = digest.span_id
+          carrier[Tracing::Distributed::Datadog::SAMPLING_PRIORITY_KEY] = digest.trace_sampling_priority
+          carrier[Tracing::Distributed::Datadog::TRACE_ID_KEY] = digest.trace_id
 
           nil
         end

data/lib/datadog/opentracer/thread_local_scope.rb

@@ -1,5 +1,3 @@
-# typed: true
-
 module Datadog
   module OpenTracer
     # OpenTracing adapter for thread local scopes

data/lib/datadog/opentracer/thread_local_scope_manager.rb

@@ -1,6 +1,4 @@
-# typed: false
-
-require 'datadog/core/utils/sequence'
+require_relative '../core/utils/sequence'
 
 module Datadog
   module OpenTracer

data/lib/datadog/opentracer/tracer.rb

@@ -1,9 +1,7 @@
-# typed: false
-
 require 'time'
 
-require 'datadog/tracing/context'
-require 'datadog/tracing/tracer'
+require_relative '../tracing/context'
+require_relative '../tracing/tracer'
 
 module Datadog
   module OpenTracer
@@ -52,13 +50,15 @@ module Datadog
       #   yield the newly-started Scope. If `finish_on_close` is true then the
       #   Span will be finished automatically after the block is executed.
       # @return [Scope] The newly-started and activated Scope
-      def start_active_span(operation_name,
-                            child_of: nil,
-                            references: nil,
-                            start_time: Time.now,
-                            tags: nil,
-                            ignore_active_scope: false,
-                            finish_on_close: true)
+      def start_active_span(
+        operation_name,
+        child_of: nil,
+        references: nil,
+        start_time: Time.now,
+        tags: nil,
+        ignore_active_scope: false,
+        finish_on_close: true
+      )
 
         # When meant to automatically determine the parent,
         # Use the active scope first, otherwise fall back to any
@@ -124,12 +124,14 @@ module Datadog
       #   References#CHILD_OF reference to the ScopeManager#active.
       # @return [Span] the newly-started Span instance, which has not been
       #   automatically registered via the ScopeManager
-      def start_span(operation_name,
-                     child_of: nil,
-                     references: nil,
-                     start_time: Time.now,
-                     tags: nil,
-                     ignore_active_scope: false)
+      def start_span(
+        operation_name,
+        child_of: nil,
+        references: nil,
+        start_time: Time.now,
+        tags: nil,
+        ignore_active_scope: false
+      )
 
         # Derive the OpenTracer::SpanContext to inherit from.
         parent_span_context = inherited_span_context(child_of, ignore_active_scope: ignore_active_scope)
@@ -147,12 +149,10 @@ module Datadog
           tags: tags || {}
         )
 
-        # Build or extend the OpenTracer::SpanContext
-        span_context = if parent_span_context
-                         SpanContextFactory.clone(span_context: parent_span_context)
-                       else
-                         SpanContextFactory.build(datadog_context: datadog_context)
-                       end
+        span_context = SpanContextFactory.build(
+          datadog_context: datadog_context || datadog_tracer.send(:call_context),
+          baggage: parent_span_context ? parent_span_context.baggage.dup : {}
+        )
 
         # Wrap the Datadog span and OpenTracer::Span context in a OpenTracer::Span
         Span.new(datadog_span: datadog_span, span_context: span_context)

data/lib/datadog/opentracer.rb

@@ -1,24 +1,22 @@
-# typed: strict
-
 require 'opentracing'
 require 'opentracing/carrier'
-require 'datadog/tracing'
+require_relative 'tracing'
 
-require 'datadog/opentracer/carrier'
-require 'datadog/opentracer/tracer'
-require 'datadog/opentracer/span'
-require 'datadog/opentracer/span_context'
-require 'datadog/opentracer/span_context_factory'
-require 'datadog/opentracer/scope'
-require 'datadog/opentracer/scope_manager'
-require 'datadog/opentracer/thread_local_scope'
-require 'datadog/opentracer/thread_local_scope_manager'
-require 'datadog/opentracer/distributed_headers'
-require 'datadog/opentracer/propagator'
-require 'datadog/opentracer/text_map_propagator'
-require 'datadog/opentracer/binary_propagator'
-require 'datadog/opentracer/rack_propagator'
-require 'datadog/opentracer/global_tracer'
+require_relative 'opentracer/carrier'
+require_relative 'opentracer/tracer'
+require_relative 'opentracer/span'
+require_relative 'opentracer/span_context'
+require_relative 'opentracer/span_context_factory'
+require_relative 'opentracer/scope'
+require_relative 'opentracer/scope_manager'
+require_relative 'opentracer/thread_local_scope'
+require_relative 'opentracer/thread_local_scope_manager'
+require_relative 'opentracer/distributed_headers'
+require_relative 'opentracer/propagator'
+require_relative 'opentracer/text_map_propagator'
+require_relative 'opentracer/binary_propagator'
+require_relative 'opentracer/rack_propagator'
+require_relative 'opentracer/global_tracer'
 
 # Modify the OpenTracing module functions
 ::OpenTracing.singleton_class.prepend(Datadog::OpenTracer::GlobalTracer)

data/lib/datadog/profiling/backtrace_location.rb

@@ -1,5 +1,3 @@
-# typed: true
-
 module Datadog
   module Profiling
     # Entity class used to represent an entry in a stack trace.

data/lib/datadog/profiling/buffer.rb

@@ -1,8 +1,6 @@
-# typed: true
-
-require 'datadog/core/buffer/thread_safe'
-require 'datadog/core/utils/object_set'
-require 'datadog/core/utils/string_table'
+require_relative '../core/buffer/thread_safe'
+require_relative '../core/utils/object_set'
+require_relative '../core/utils/string_table'
 
 module Datadog
   module Profiling

data/lib/datadog/profiling/collectors/code_provenance.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-# typed: false
-
 require 'set'
 require 'json'
 

data/lib/datadog/profiling/collectors/cpu_and_wall_time_worker.rb

@@ -0,0 +1,96 @@
+module Datadog
+  module Profiling
+    module Collectors
+      # Used to trigger the periodic execution of Collectors::ThreadState, which implements all of the sampling logic
+      # itself; this class only implements the "when to do it" part.
+      # Almost all of this class is implemented as native code.
+      #
+      # Methods prefixed with _native_ are implemented in `collectors_cpu_and_wall_time_worker.c`
+      class CpuAndWallTimeWorker
+        private
+
+        attr_accessor :failure_exception
+
+        public
+
+        def initialize(
+          recorder:,
+          max_frames:,
+          tracer:,
+          gc_profiling_enabled:,
+          allocation_counting_enabled:,
+          thread_context_collector: ThreadContext.new(recorder: recorder, max_frames: max_frames, tracer: tracer),
+          idle_sampling_helper: IdleSamplingHelper.new
+        )
+          self.class._native_initialize(
+            self,
+            thread_context_collector,
+            gc_profiling_enabled,
+            idle_sampling_helper,
+            allocation_counting_enabled
+          )
+          @worker_thread = nil
+          @failure_exception = nil
+          @start_stop_mutex = Mutex.new
+          @idle_sampling_helper = idle_sampling_helper
+        end
+
+        def start
+          @start_stop_mutex.synchronize do
+            return if @worker_thread && @worker_thread.alive?
+
+            Datadog.logger.debug { "Starting thread for: #{self}" }
+
+            @idle_sampling_helper.start
+
+            @worker_thread = Thread.new do
+              begin
+                Thread.current.name = self.class.name
+
+                self.class._native_sampling_loop(self)
+
+                Datadog.logger.debug('CpuAndWallTimeWorker thread stopping cleanly')
+              rescue Exception => e # rubocop:disable Lint/RescueException
+                @failure_exception = e
+                Datadog.logger.warn(
+                  'CpuAndWallTimeWorker thread error. ' \
+                  "Cause: #{e.class.name} #{e.message} Location: #{Array(e.backtrace).first}"
+                )
+              end
+            end
+          end
+
+          true
+        end
+
+        # TODO: Provided only for compatibility with the API for Collectors::OldStack used in the Profiler class.
+        # Can be removed once we remove OldStack.
+        def enabled=(_); end
+
+        def stop(*_)
+          @start_stop_mutex.synchronize do
+            Datadog.logger.debug('Requesting CpuAndWallTimeWorker thread shut down')
+
+            @idle_sampling_helper.stop
+
+            return unless @worker_thread
+
+            self.class._native_stop(self, @worker_thread)
+
+            @worker_thread.join
+            @worker_thread = nil
+            @failure_exception = nil
+          end
+        end
+
+        def reset_after_fork
+          self.class._native_reset_after_fork(self)
+        end
+
+        def stats
+          self.class._native_stats(self)
+        end
+      end
+    end
+  end
+end

data/lib/datadog/profiling/collectors/dynamic_sampling_rate.rb

@@ -0,0 +1,12 @@
+module Datadog
+  module Profiling
+    module Collectors
+      # Used to pace the rate of profiling samples based on the last observed time for a sample.
+      # All of this module is implemented as native code.
+      #
+      # Methods prefixed with _native_ are implemented in `collectors_dynamic_sampling_rate.c`
+      module DynamicSamplingRate
+      end
+    end
+  end
+end

data/lib/datadog/profiling/collectors/idle_sampling_helper.rb

@@ -0,0 +1,66 @@
+module Datadog
+  module Profiling
+    module Collectors
+      # Used by the Collectors::CpuAndWallTimeWorker to gather samples when the Ruby process is idle.
+      # Almost all of this class is implemented as native code.
+      #
+      # Methods prefixed with _native_ are implemented in `collectors_idle_sampling_helper.c`
+      class IdleSamplingHelper
+        private
+
+        attr_accessor :failure_exception
+
+        public
+
+        def initialize
+          @worker_thread = nil
+          @start_stop_mutex = Mutex.new
+        end
+
+        def start
+          @start_stop_mutex.synchronize do
+            return if @worker_thread && @worker_thread.alive?
+
+            Datadog.logger.debug { "Starting thread for: #{self}" }
+
+            # The same instance of the IdleSamplingHelper can be reused multiple times, and this resets it back to
+            # a pristine state before recreating the worker thread
+            self.class._native_reset(self)
+
+            @worker_thread = Thread.new do
+              begin
+                Thread.current.name = self.class.name
+
+                self.class._native_idle_sampling_loop(self)
+
+                Datadog.logger.debug('IdleSamplingHelper thread stopping cleanly')
+              rescue Exception => e # rubocop:disable Lint/RescueException
+                @failure_exception = e
+                Datadog.logger.warn(
+                  'IdleSamplingHelper thread error. ' \
+                  "Cause: #{e.class.name} #{e.message} Location: #{Array(e.backtrace).first}"
+                )
+              end
+            end
+          end
+
+          true
+        end
+
+        def stop(*_)
+          @start_stop_mutex.synchronize do
+            Datadog.logger.debug('Requesting IdleSamplingHelper thread shut down')
+
+            return unless @worker_thread
+
+            self.class._native_stop(self)
+
+            @worker_thread.join
+            @worker_thread = nil
+            @failure_exception = nil
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/profiling/collectors/old_stack.rb

@@ -1,12 +1,10 @@
-# typed: true
-
-require 'datadog/core/utils/only_once'
-require 'datadog/core/utils/time'
-require 'datadog/core/worker'
-require 'datadog/core/workers/polling'
-require 'datadog/profiling/backtrace_location'
-require 'datadog/profiling/events/stack'
-require 'datadog/profiling/native_extension'
+require_relative '../../core/utils/only_once'
+require_relative '../../core/utils/time'
+require_relative '../../core/worker'
+require_relative '../../core/workers/polling'
+require_relative '../backtrace_location'
+require_relative '../events/stack'
+require_relative '../native_extension'
 
 module Datadog
   module Profiling
@@ -15,7 +13,7 @@ module Datadog
       # Runs on its own background thread.
       #
       # This class has the prefix "Old" because it will be deprecated by the new native CPU Profiler
-      class OldStack < Core::Worker # rubocop:disable Metrics/ClassLength
+      class OldStack < Core::Worker
         include Core::Workers::Polling
 
         DEFAULT_MAX_TIME_USAGE_PCT = 2.0
@@ -75,9 +73,7 @@ module Datadog
           # Cache this buffer, since it's pretty expensive to keep accessing it
           @stack_sample_event_recorder = recorder[Events::StackSample]
           # See below for details on why this is needed
-          @needs_process_waiter_workaround =
-            Gem::Version.new(RUBY_VERSION) >= Gem::Version.new('2.3') &&
-            Gem::Version.new(RUBY_VERSION) < Gem::Version.new('2.7')
+          @needs_process_waiter_workaround = Gem::Version.new(RUBY_VERSION) < Gem::Version.new('2.7')
         end
 
         def start
@@ -218,6 +214,13 @@ module Datadog
           )
         end
 
+        def reset_after_fork
+          recorder.reset_after_fork
+
+          # NOTE: We could perhaps also call #reset_cpu_time_tracking here, although it's not needed because we always
+          # call in in #start.
+        end
+
         private
 
         # If the profiler is started for a while, stopped and then restarted OR whenever the process forks, we need to

data/lib/datadog/profiling/collectors/stack.rb

@@ -1,17 +1,10 @@
-# typed: false
-
 module Datadog
   module Profiling
     module Collectors
-      # Used to gather a stack trace from a given Ruby thread. Almost all of this class is implemented as native code.
+      # Used to gather a stack trace from a given Ruby thread. Stores its output on a `StackRecorder`.
       #
-      # Methods prefixed with _native_ are implemented in `collectors_stack.c`
-      class Stack
-        # This method exists only to enable testing Datadog::Profiling::Collectors::Stack behavior using RSpec.
-        # It SHOULD NOT be used for other purposes.
-        def sample(thread, recorder_instance, metric_values_hash, labels_array, max_frames: 400)
-          self.class._native_sample(thread, recorder_instance, metric_values_hash, labels_array, max_frames)
-        end
+      # This class is not empty; all of this class is implemented as native code.
+      class Stack # rubocop:disable Lint/EmptyClass
       end
     end
   end

data/lib/datadog/profiling/collectors/thread_context.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Profiling
+    module Collectors
+      # Used to trigger sampling of threads, based on external "events", such as:
+      # * periodic timer for cpu-time and wall-time
+      # * VM garbage collection events
+      # * VM object allocation events
+      # Triggering of this component (e.g. watching for the above "events") is implemented by
+      # Collectors::CpuAndWallTimeWorker.
+      # The stack collection itself is handled using the Datadog::Profiling::Collectors::Stack.
+      # Almost all of this class is implemented as native code.
+      #
+      # Methods prefixed with _native_ are implemented in `collectors_thread_context.c`
+      class ThreadContext
+        def initialize(recorder:, max_frames:, tracer:)
+          tracer_context_key = safely_extract_context_key_from(tracer)
+          self.class._native_initialize(self, recorder, max_frames, tracer_context_key)
+        end
+
+        def inspect
+          # Compose Ruby's default inspect with our custom inspect for the native parts
+          result = super()
+          result[-1] = "#{self.class._native_inspect(self)}>"
+          result
+        end
+
+        def reset_after_fork
+          self.class._native_reset_after_fork(self)
+        end
+
+        private
+
+        def safely_extract_context_key_from(tracer)
+          provider = tracer && tracer.respond_to?(:provider) && tracer.provider
+
+          return unless provider
+
+          context = provider.instance_variable_get(:@context)
+          context && context.instance_variable_get(:@key)
+        end
+      end
+    end
+  end
+end

data/lib/datadog/profiling/component.rb

@@ -0,0 +1,161 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Profiling
+    # Profiling component
+    module Component
+      def build_profiler(settings, agent_settings, tracer)
+        return unless settings.profiling.enabled
+
+        # Workaround for weird dependency direction: the Core::Configuration::Components class currently has a
+        # dependency on individual products, in this case the Profiler.
+        # (Note "currently": in the future we want to change this so core classes don't depend on specific products)
+        #
+        # If the current file included a `require 'datadog/profiler'` at its beginning, we would generate circular
+        # requires when used from profiling:
+        #
+        # datadog/profiling
+        #     └─requires─> datadog/core
+        #                      └─requires─> datadog/core/configuration/components
+        #                                       └─requires─> datadog/profiling       # Loop!
+        #
+        # ...thus in #1998 we removed such a require.
+        #
+        # On the other hand, if datadog/core is loaded by a different product and no general `require 'ddtrace'` is
+        # done, then profiling may not be loaded, and thus to avoid this issue we do a require here (which is a
+        # no-op if profiling is already loaded).
+        require_relative '../profiling'
+        return unless Profiling.supported?
+
+        unless defined?(Profiling::Tasks::Setup)
+          # In #1545 a user reported a NameError due to this constant being uninitialized
+          # I've documented my suspicion on why that happened in
+          # https://github.com/DataDog/dd-trace-rb/issues/1545#issuecomment-856049025
+          #
+          # > Thanks for the info! It seems to feed into my theory: there's two moments in the code where we check if
+          # > profiler is "supported": 1) when loading ddtrace (inside preload) and 2) when starting the profile
+          # > after Datadog.configure gets run.
+          # > The problem is that the code assumes that both checks 1) and 2) will always reach the same conclusion:
+          # > either profiler is supported, or profiler is not supported.
+          # > In the problematic case, it looks like in your case check 1 decides that profiler is not
+          # > supported => doesn't load it, and then check 2 decides that it is => assumes it is loaded and tries to
+          # > start it.
+          #
+          # I was never able to validate if this was the issue or why exactly .supported? would change its mind BUT
+          # just in case it happens again, I've left this check which avoids breaking the user's application AND
+          # would instead direct them to report it to us instead, so that we can investigate what's wrong.
+          #
+          # TODO: As of June 2021, most checks in .supported? are related to the google-protobuf gem; so it's
+          # very likely that it was the origin of the issue we saw. Thus, if, as planned we end up moving away from
+          # protobuf OR enough time has passed and no users saw the issue again, we can remove this check altogether.
+          Datadog.logger.error(
+            'Profiling was marked as supported and enabled, but setup task was not loaded properly. ' \
+            'Please report this at https://github.com/DataDog/dd-trace-rb/blob/master/CONTRIBUTING.md#found-a-bug'
+          )
+
+          return
+        end
+
+        # Load extensions needed to support some of the Profiling features
+        Profiling::Tasks::Setup.new.run
+
+        # NOTE: Please update the Initialization section of ProfilingDevelopment.md with any changes to this method
+
+        if settings.profiling.advanced.force_enable_new_profiler
+          print_new_profiler_warnings
+
+          recorder = Datadog::Profiling::StackRecorder.new(
+            cpu_time_enabled: RUBY_PLATFORM.include?('linux'), # Only supported on Linux currently
+            alloc_samples_enabled: false, # Always disabled for now -- work in progress
+          )
+          collector = Datadog::Profiling::Collectors::CpuAndWallTimeWorker.new(
+            recorder: recorder,
+            max_frames: settings.profiling.advanced.max_frames,
+            tracer: tracer,
+            gc_profiling_enabled: should_enable_gc_profiling?(settings),
+            allocation_counting_enabled: settings.profiling.advanced.allocation_counting_enabled,
+          )
+        else
+          trace_identifiers_helper = Profiling::TraceIdentifiers::Helper.new(
+            tracer: tracer,
+            endpoint_collection_enabled: settings.profiling.advanced.endpoint.collection.enabled
+          )
+
+          recorder = build_profiler_old_recorder(settings)
+          collector = build_profiler_oldstack_collector(settings, recorder, trace_identifiers_helper)
+        end
+
+        exporter = build_profiler_exporter(settings, recorder)
+        transport = build_profiler_transport(settings, agent_settings)
+        scheduler = Profiling::Scheduler.new(exporter: exporter, transport: transport)
+
+        Profiling::Profiler.new([collector], scheduler)
+      end
+
+      private
+
+      def build_profiler_old_recorder(settings)
+        Profiling::OldRecorder.new([Profiling::Events::StackSample], settings.profiling.advanced.max_events)
+      end
+
+      def build_profiler_exporter(settings, recorder)
+        code_provenance_collector =
+          (Profiling::Collectors::CodeProvenance.new if settings.profiling.advanced.code_provenance_enabled)
+
+        Profiling::Exporter.new(pprof_recorder: recorder, code_provenance_collector: code_provenance_collector)
+      end
+
+      def build_profiler_oldstack_collector(settings, old_recorder, trace_identifiers_helper)
+        Profiling::Collectors::OldStack.new(
+          old_recorder,
+          trace_identifiers_helper: trace_identifiers_helper,
+          max_frames: settings.profiling.advanced.max_frames
+        )
+      end
+
+      def build_profiler_transport(settings, agent_settings)
+        settings.profiling.exporter.transport ||
+          Profiling::HttpTransport.new(
+            agent_settings: agent_settings,
+            site: settings.site,
+            api_key: settings.api_key,
+            upload_timeout_seconds: settings.profiling.upload.timeout_seconds,
+          )
+      end
+
+      def should_enable_gc_profiling?(settings)
+        # See comments on the setting definition for more context on why it exists.
+        if settings.profiling.advanced.force_enable_gc_profiling
+          if Gem::Version.new(RUBY_VERSION) >= Gem::Version.new('3')
+            Datadog.logger.debug(
+              'Profiling time/resources spent in Garbage Collection force enabled. Do not use Ractors in combination ' \
+              'with this option as profiles will be incomplete.'
+            )
+          end
+
+          true
+        else
+          false
+        end
+      end
+
+      def print_new_profiler_warnings
+        if Gem::Version.new(RUBY_VERSION) >= Gem::Version.new('2.6')
+          Datadog.logger.warn(
+            'New Ruby profiler has been force-enabled. This is a beta feature. Please report any issues ' \
+            'you run into to Datadog support or via <https://github.com/datadog/dd-trace-rb/issues/new>!'
+          )
+        else
+          # For more details on the issue, see the "BIG Issue" comment on `gvl_owner` function in
+          # `private_vm_api_access.c`.
+          Datadog.logger.warn(
+            'New Ruby profiler has been force-enabled on a legacy Ruby version (< 2.6). This is not recommended in ' \
+            'production environments, as due to limitations in Ruby APIs, we suspect it may lead to crashes in very ' \
+            'rare situations. Please report any issues you run into to Datadog support or ' \
+            'via <https://github.com/datadog/dd-trace-rb/issues/new>!'
+          )
+        end
+      end
+    end
+  end
+end