ddtrace 0.49.0 → 0.52.0

data/lib/ddtrace/opentracer/binary_propagator.rb

@@ -1,3 +1,4 @@
+# typed: true
 module Datadog
   module OpenTracer
     # OpenTracing propagator for Datadog::OpenTracer::Tracer

data/lib/ddtrace/opentracer/carrier.rb

@@ -1,3 +1,4 @@
+# typed: strict
 module Datadog
   module OpenTracer
     class Carrier < ::OpenTracing::Carrier

data/lib/ddtrace/opentracer/distributed_headers.rb

@@ -1,3 +1,4 @@
+# typed: true
 require 'ddtrace/span'
 require 'ddtrace/ext/distributed'
 

data/lib/ddtrace/opentracer/global_tracer.rb

@@ -1,3 +1,4 @@
+# typed: true
 module Datadog
   module OpenTracer
     # Patch for OpenTracing module

data/lib/ddtrace/opentracer/propagator.rb

@@ -1,3 +1,4 @@
+# typed: false
 module Datadog
   module OpenTracer
     # OpenTracing propagator for Datadog::OpenTracer::Tracer

data/lib/ddtrace/opentracer/rack_propagator.rb

@@ -1,3 +1,4 @@
+# typed: true
 require 'ddtrace/propagation/http_propagator'
 
 module Datadog

data/lib/ddtrace/opentracer/scope.rb

@@ -1,3 +1,4 @@
+# typed: true
 module Datadog
   module OpenTracer
     # OpenTracing adapter for scope

data/lib/ddtrace/opentracer/scope_manager.rb

@@ -1,3 +1,4 @@
+# typed: strict
 module Datadog
   module OpenTracer
     class ScopeManager < ::OpenTracing::ScopeManager

data/lib/ddtrace/opentracer/span.rb

@@ -1,3 +1,4 @@
+# typed: true
 module Datadog
   module OpenTracer
     # OpenTracing adapter for Datadog::Span

data/lib/ddtrace/opentracer/span_context.rb

@@ -1,3 +1,4 @@
+# typed: true
 module Datadog
   module OpenTracer
     # OpenTracing adapter for SpanContext

data/lib/ddtrace/opentracer/span_context_factory.rb

@@ -1,3 +1,4 @@
+# typed: true
 module Datadog
   module OpenTracer
     # Creates new Datadog::OpenTracer::SpanContext

data/lib/ddtrace/opentracer/text_map_propagator.rb

@@ -1,3 +1,4 @@
+# typed: true
 require 'ddtrace/ext/distributed'
 
 module Datadog

data/lib/ddtrace/opentracer/thread_local_scope.rb

@@ -1,3 +1,4 @@
+# typed: true
 module Datadog
   module OpenTracer
     # OpenTracing adapter for thread local scopes

data/lib/ddtrace/opentracer/thread_local_scope_manager.rb

@@ -1,3 +1,4 @@
+# typed: true
 module Datadog
   module OpenTracer
     # OpenTracing adapter for thread local scope management

data/lib/ddtrace/opentracer/tracer.rb

@@ -1,3 +1,4 @@
+# typed: true
 require 'ddtrace/tracer'
 
 module Datadog

data/lib/ddtrace/patcher.rb

@@ -1,3 +1,4 @@
+# typed: false
 require 'ddtrace/utils/only_once'
 
 module Datadog
@@ -16,8 +17,8 @@ module Datadog
         )
       end
 
-      base.send(:extend, CommonMethods)
-      base.send(:include, CommonMethods)
+      base.extend(CommonMethods)
+      base.include(CommonMethods)
     end
 
     # Defines some common methods for patching, that can be used

data/lib/ddtrace/pin.rb

@@ -1,3 +1,4 @@
+# typed: true
 require 'ddtrace/patcher'
 require 'ddtrace/utils/only_once'
 

data/lib/ddtrace/pipeline.rb

@@ -1,3 +1,4 @@
+# typed: true
 module Datadog
   # Pipeline
   module Pipeline

data/lib/ddtrace/pipeline/span_filter.rb

@@ -1,3 +1,4 @@
+# typed: true
 module Datadog
   module Pipeline
     # SpanFilter implements a processor that filters entire span subtrees

data/lib/ddtrace/pipeline/span_processor.rb

@@ -1,3 +1,4 @@
+# typed: true
 module Datadog
   module Pipeline
     # SpanProcessor

data/lib/ddtrace/profiling.rb

@@ -1,54 +1,59 @@
+# typed: true
+require 'datadog/core/environment/variable_helpers'
+require 'ddtrace/utils/only_once'
+
 module Datadog
   # Contains profiler for generating stack profiles, etc.
-  module Profiling
-    module_function
-
+  module Profiling # rubocop:disable Metrics/ModuleLength
     GOOGLE_PROTOBUF_MINIMUM_VERSION = Gem::Version.new('3.0')
     private_constant :GOOGLE_PROTOBUF_MINIMUM_VERSION
 
-    def supported?
+    SKIPPED_NATIVE_EXTENSION_ONLY_ONCE = Datadog::Utils::OnlyOnce.new
+    private_constant :SKIPPED_NATIVE_EXTENSION_ONLY_ONCE
+
+    def self.supported?
       unsupported_reason.nil?
     end
 
-    def unsupported_reason
+    def self.unsupported_reason
       # NOTE: Only the first matching reason is returned, so try to keep a nice order on reasons -- e.g. tell users
       # first that they can't use this on JRuby before telling them that they are missing protobuf
 
-      ruby_engine_unsupported? || ruby_version_unsupported? ||
-        protobuf_gem_unavailable? || protobuf_version_unsupported? || protobuf_failed_to_load?
+      ruby_engine_unsupported? ||
+        native_library_failed_to_load? ||
+        protobuf_gem_unavailable? ||
+        protobuf_version_unsupported? ||
+        protobuf_failed_to_load?
     end
 
-    def self.ruby_engine_unsupported?
+    private_class_method def self.ruby_engine_unsupported?
       'JRuby is not supported' if RUBY_ENGINE == 'jruby'
     end
-    private_class_method :ruby_engine_unsupported?
-
-    def self.ruby_version_unsupported?
-      'Ruby >= 2.1 is required' if Gem::Version.new(RUBY_VERSION) < Gem::Version.new('2.1')
-    end
-    private_class_method :ruby_version_unsupported?
 
-    def self.protobuf_gem_unavailable?
-      if Gem.loaded_specs['google-protobuf'].nil?
+    private_class_method def self.protobuf_gem_unavailable?
+      # NOTE: On environments where protobuf is already loaded, we skip the check. This allows us to support environments
+      # where no Gem.loaded_version is NOT available but customers are able to load protobuf; see for instance
+      # https://github.com/teamcapybara/capybara/commit/caf3bcd7664f4f2691d0ca9ef3be9a2a954fecfb
+      if !defined?(::Google::Protobuf) && Gem.loaded_specs['google-protobuf'].nil?
         "Missing google-protobuf dependency; please add `gem 'google-protobuf', '~> 3.0'` to your Gemfile or gems.rb file"
       end
     end
-    private_class_method :protobuf_gem_unavailable?
 
-    def self.protobuf_version_unsupported?
-      if Gem.loaded_specs['google-protobuf'].version < GOOGLE_PROTOBUF_MINIMUM_VERSION
+    private_class_method def self.protobuf_version_unsupported?
+      # See above for why we skip the check when protobuf is already loaded; note that when protobuf was already loaded
+      # we skip the version check to avoid the call to Gem.loaded_specs. Unfortunately, protobuf does not seem to
+      # expose the gem version constant elsewhere, so in that setup we are not able to check the version.
+      if !defined?(::Google::Protobuf) && Gem.loaded_specs['google-protobuf'].version < GOOGLE_PROTOBUF_MINIMUM_VERSION
         'Your google-protobuf is too old; ensure that you have google-protobuf >= 3.0 by ' \
         "adding `gem 'google-protobuf', '~> 3.0'` to your Gemfile or gems.rb file"
       end
     end
-    private_class_method :protobuf_version_unsupported?
 
-    def self.protobuf_failed_to_load?
+    private_class_method def self.protobuf_failed_to_load?
       unless protobuf_loaded_successfully?
         'There was an error loading the google-protobuf library; see previous warning message for details'
       end
     end
-    private_class_method :protobuf_failed_to_load?
 
     # The `google-protobuf` gem depends on a native component, and its creators helpfully tried to provide precompiled
     # versions of this extension on rubygems.org.
@@ -59,15 +64,18 @@ module Datadog
     #
     # Thus, the gem can still be installed, but can be in a broken state. To avoid breaking customer applications, we
     # use this helper to load it and gracefully handle failures.
-    def self.protobuf_loaded_successfully?
+    private_class_method def self.protobuf_loaded_successfully?
       return @protobuf_loaded if defined?(@protobuf_loaded)
 
       begin
         require 'google/protobuf'
         @protobuf_loaded = true
       rescue LoadError => e
+        # NOTE: We use Kernel#warn here because this code gets run BEFORE Datadog.logger is actually set up.
+        # In the future it'd be nice to shuffle the logger startup to happen first to avoid this special case.
         Kernel.warn(
-          "[DDTRACE] Error while loading google-protobuf gem. Cause: '#{e.message}' Location: '#{e.backtrace.first}'. " \
+          '[DDTRACE] Error while loading google-protobuf gem. ' \
+          "Cause: '#{e.message}' Location: '#{Array(e.backtrace).first}'. " \
           'This can happen when google-protobuf is missing its native components. ' \
           'To fix this, try removing and reinstalling the gem, forcing it to recompile the components: ' \
           '`gem uninstall google-protobuf -a; BUNDLE_FORCE_RUBY_PLATFORM=true bundle install`. ' \
@@ -77,9 +85,47 @@ module Datadog
         @protobuf_loaded = false
       end
     end
-    private_class_method :protobuf_loaded_successfully?
 
-    def self.load_profiling
+    private_class_method def self.native_library_failed_to_load?
+      success, exception = try_loading_native_library
+
+      unless success
+        if exception
+          'There was an error loading the profiling native extension due to ' \
+          "'#{exception.message}' at '#{exception.backtrace.first}'"
+        else
+          'The profiling native extension did not load correctly. ' \
+          'If the error persists, please contact support via <https://docs.datadoghq.com/help/> or ' \
+          'file a bug at <https://github.com/DataDog/dd-trace-rb/blob/master/CONTRIBUTING.md#found-a-bug>.'
+        end
+      end
+    end
+
+    private_class_method def self.try_loading_native_library
+      if Datadog::Core::Environment::VariableHelpers.env_to_bool('DD_PROFILING_NO_EXTENSION', false)
+        SKIPPED_NATIVE_EXTENSION_ONLY_ONCE.run do
+          Kernel.warn(
+            '[DDTRACE] Skipped loading of profiling native extension due to DD_PROFILING_NO_EXTENSION environment ' \
+            'variable being set. ' \
+            'This option is experimental and will lead to the profiler not working in future releases. ' \
+            'If you needed to use this, please tell us why on <https://github.com/DataDog/dd-trace-rb/issues/new>.'
+          )
+        end
+
+        return [true, nil]
+      end
+
+      begin
+        require "ddtrace_profiling_native_extension.#{RUBY_VERSION}_#{RUBY_PLATFORM}"
+        success =
+          defined?(Datadog::Profiling::NativeExtension) && Datadog::Profiling::NativeExtension.send(:native_working?)
+        [success, nil]
+      rescue StandardError, LoadError => e
+        [false, e]
+      end
+    end
+
+    private_class_method def self.load_profiling
       return false unless supported?
 
       require 'ddtrace/profiling/ext/cpu'
@@ -93,12 +139,12 @@ module Datadog
       require 'ddtrace/profiling/transport/io'
       require 'ddtrace/profiling/transport/http'
       require 'ddtrace/profiling/profiler'
-
+      require 'ddtrace/profiling/native_extension'
+      require 'ddtrace/profiling/trace_identifiers/helper'
       require 'ddtrace/profiling/pprof/pprof_pb'
 
       true
     end
-    private_class_method :load_profiling
 
     load_profiling if supported?
   end

data/lib/ddtrace/profiling/backtrace_location.rb

@@ -1,3 +1,4 @@
+# typed: true
 module Datadog
   module Profiling
     # Entity class used to represent an entry in a stack trace.

data/lib/ddtrace/profiling/buffer.rb

@@ -1,3 +1,4 @@
+# typed: true
 require 'ddtrace/buffer'
 require 'ddtrace/utils/string_table'
 require 'ddtrace/utils/object_set'

data/lib/ddtrace/profiling/collectors/stack.rb

@@ -1,3 +1,4 @@
+# typed: true
 require 'ddtrace/profiling/backtrace_location'
 require 'ddtrace/profiling/events/stack'
 require 'ddtrace/utils/only_once'
@@ -11,8 +12,7 @@ module Datadog
       # Collects stack trace samples from Ruby threads for both CPU-time (if available) and wall-clock.
       # Runs on its own background thread.
       #
-      # rubocop:disable Metrics/ClassLength
-      class Stack < Worker
+      class Stack < Worker # rubocop:disable Metrics/ClassLength
         include Workers::Polling
 
         DEFAULT_MAX_TIME_USAGE_PCT = 2.0
@@ -22,6 +22,7 @@ module Datadog
         attr_reader \
           :recorder,
           :max_frames,
+          :trace_identifiers_helper,
           :ignore_thread,
           :max_time_usage_pct,
           :thread_api
@@ -29,6 +30,7 @@ module Datadog
         def initialize(
           recorder,
           max_frames:,
+          trace_identifiers_helper:, # Usually an instance of Datadog::Profiling::TraceIdentifiers::Helper
           ignore_thread: nil,
           max_time_usage_pct: DEFAULT_MAX_TIME_USAGE_PCT,
           thread_api: Thread,
@@ -38,6 +40,7 @@ module Datadog
         )
           @recorder = recorder
           @max_frames = max_frames
+          @trace_identifiers_helper = trace_identifiers_helper
           @ignore_thread = ignore_thread
           @max_time_usage_pct = max_time_usage_pct
           @thread_api = thread_api
@@ -122,8 +125,8 @@ module Datadog
           # Convert backtrace locations into structs
           locations = convert_backtrace_locations(locations)
 
-          thread_id = thread.respond_to?(:native_thread_id) ? thread.native_thread_id : thread.object_id
-          trace_id, span_id = get_trace_identifiers(thread)
+          thread_id = thread.respond_to?(:pthread_thread_id) ? thread.pthread_thread_id : thread.object_id
+          trace_id, span_id, trace_resource_container = trace_identifiers_helper.trace_identifiers_for(thread)
           cpu_time = get_cpu_time_interval!(thread)
 
           Events::StackSample.new(
@@ -133,6 +136,7 @@ module Datadog
             thread_id,
             trace_id,
             span_id,
+            trace_resource_container,
             cpu_time,
             wall_time_interval_ns
           )
@@ -152,24 +156,16 @@ module Datadog
           # *before* the thread had time to finish the initialization
           return unless current_cpu_time_ns
 
-          last_cpu_time_ns = (thread[THREAD_LAST_CPU_TIME_KEY] || current_cpu_time_ns)
+          last_cpu_time_ns = (thread.thread_variable_get(THREAD_LAST_CPU_TIME_KEY) || current_cpu_time_ns)
           interval = current_cpu_time_ns - last_cpu_time_ns
 
           # Update CPU time for thread
-          thread[THREAD_LAST_CPU_TIME_KEY] = current_cpu_time_ns
+          thread.thread_variable_set(THREAD_LAST_CPU_TIME_KEY, current_cpu_time_ns)
 
           # Return interval
           interval
         end
 
-        def get_trace_identifiers(thread)
-          return unless thread.is_a?(::Thread)
-          return unless Datadog.respond_to?(:tracer) && Datadog.tracer.respond_to?(:active_correlation)
-
-          identifier = Datadog.tracer.active_correlation(thread)
-          [identifier.trace_id, identifier.span_id]
-        end
-
         def compute_wait_time(used_time)
           # We took used_time to get the last sample.
           #
@@ -224,6 +220,7 @@ module Datadog
             #    In this case, the fix is to make sure ddtrace gets loaded before any other parts of the application.
             #
             # b) The thread was started using the Ruby native APIs (e.g. from a C extension such as ffi).
+            #    Known cases right now that trigger this are the ethon/typhoeus gems.
             #    We currently have no solution for this case; these threads will always be missing our CPU instrumentation.
             #
             # c) The thread was started with `Thread.start`/`Thread.fork` and hasn't yet enabled the instrumentation.
@@ -232,13 +229,15 @@ module Datadog
             #    it to run and our instrumentation to be applied.
             #
             if thread_api.current.respond_to?(:cpu_time) && thread_api.current.cpu_time
-              Datadog.logger.debug("Detected thread ('#{thread}') with missing CPU profiling instrumentation.")
+              Datadog.logger.debug(
+                "Thread ('#{thread}') is missing profiling instrumentation; other threads should be unaffected"
+              )
             end
           end
         end
 
         # If the profiler is started for a while, stopped and then restarted OR whenever the process forks, we need to
-        # clean up the per-thread cpu time counters we keep, so that the first sample after starting doesn't end up with:
+        # clean up any leftover per-thread cpu time counters, so that the first sample after starting doesn't end up with:
         #
         # a) negative time: At least on my test docker container, and on the reliability environment, after the process
         #    forks, the clock reference changes and (old cpu time - new cpu time) can be < 0
@@ -249,11 +248,10 @@ module Datadog
         # By resetting the last cpu time seen, we start with a clean slate every time we start the stack collector.
         def reset_cpu_time_tracking
           thread_api.list.each do |thread|
-            thread[THREAD_LAST_CPU_TIME_KEY] = nil if thread[THREAD_LAST_CPU_TIME_KEY]
+            thread.thread_variable_set(THREAD_LAST_CPU_TIME_KEY, nil)
           end
         end
       end
-      # rubocop:enable Metrics/ClassLength
     end
   end
 end

data/lib/ddtrace/profiling/encoding/profile.rb

@@ -1,4 +1,6 @@
+# typed: true
 require 'set'
+require 'time'
 
 require 'ddtrace/profiling/flush'
 require 'ddtrace/profiling/pprof/template'
@@ -21,6 +23,11 @@ module Datadog
             # Add all events to the pprof
             flush.event_groups.each { |event_group| template.add_events!(event_group.event_class, event_group.events) }
 
+            Datadog.logger.debug do
+              "Encoding profile covering #{flush.start.iso8601} to #{flush.finish.iso8601}, " \
+              "events: #{flush.event_count} (#{template.debug_statistics})"
+            end
+
             # Build the profile and encode it
             template.to_pprof
           end