ddtrace 0.48.0 → 0.51.1

data/lib/ddtrace/profiling/ext/forking.rb

@@ -24,13 +24,13 @@ module Datadog
           #       It could also have collisions with other libraries that patch.
           #       Opt to modify the inheritance of each relevant target instead.
           modules.each do |mod|
-            if mod.class <= Module
-              mod.singleton_class.class_eval do
-                prepend Kernel
-              end
-            else
-              mod.class.send(:prepend, Kernel)
-            end
+            clazz = if mod.class <= Module
+                      mod.singleton_class
+                    else
+                      mod.class
+                    end
+
+            clazz.prepend(Kernel)
           end
         end
 

data/lib/ddtrace/profiling/flush.rb

@@ -1,5 +1,5 @@
-require 'ddtrace/runtime/identity'
-require 'ddtrace/runtime/socket'
+require 'datadog/core/environment/identity'
+require 'datadog/core/environment/socket'
 
 module Datadog
   module Profiling
@@ -18,20 +18,22 @@ module Datadog
       :runtime_engine,
       :runtime_platform,
       :runtime_version,
-      :profiler_version
+      :profiler_version,
+      :tags
     ) do
       def initialize(*args)
         super
-        self.runtime_id = runtime_id || Datadog::Runtime::Identity.id
+        self.runtime_id = runtime_id || Datadog::Core::Environment::Identity.id
         self.service = service || Datadog.configuration.service
         self.env = env || Datadog.configuration.env
         self.version = version || Datadog.configuration.version
-        self.host = host || Datadog::Runtime::Socket.hostname
-        self.language = language || Datadog::Runtime::Identity.lang
-        self.runtime_engine = runtime_engine || Datadog::Runtime::Identity.lang_engine
-        self.runtime_platform = runtime_platform || Datadog::Runtime::Identity.lang_platform
-        self.runtime_version = runtime_version || Datadog::Runtime::Identity.lang_version
-        self.profiler_version = profiler_version || Datadog::Runtime::Identity.tracer_version
+        self.host = host || Datadog::Core::Environment::Socket.hostname
+        self.language = language || Datadog::Core::Environment::Identity.lang
+        self.runtime_engine = runtime_engine || Datadog::Core::Environment::Identity.lang_engine
+        self.runtime_platform = runtime_platform || Datadog::Core::Environment::Identity.lang_platform
+        self.runtime_version = runtime_version || Datadog::Core::Environment::Identity.lang_version
+        self.profiler_version = profiler_version || Datadog::Core::Environment::Identity.tracer_version
+        self.tags = tags || Datadog.configuration.tags
       end
     end
 

data/lib/ddtrace/profiling/pprof/builder.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'ddtrace/profiling/flush'
 require 'ddtrace/profiling/pprof/message_set'
 require 'ddtrace/profiling/pprof/string_table'
@@ -7,9 +9,9 @@ module Datadog
     module Pprof
       # Accumulates profile data and produces a Perftools::Profiles::Profile
       class Builder
-        DEFAULT_ENCODING = 'UTF-8'.freeze
-        DESC_FRAME_OMITTED = 'frame omitted'.freeze
-        DESC_FRAMES_OMITTED = 'frames omitted'.freeze
+        DEFAULT_ENCODING = 'UTF-8'
+        DESC_FRAME_OMITTED = 'frame omitted'
+        DESC_FRAMES_OMITTED = 'frames omitted'
 
         attr_reader \
           :functions,
@@ -21,11 +23,23 @@ module Datadog
 
         def initialize
           @functions = MessageSet.new(1)
-          @locations = MessageSet.new(1)
+          @locations = initialize_locations_hash
           @mappings = MessageSet.new(1)
           @sample_types = MessageSet.new
           @samples = []
           @string_table = StringTable.new
+
+          # Cache this proc, since it's pretty expensive to keep recreating it
+          @build_function = method(:build_function).to_proc
+        end
+
+        # The locations hash maps unique BacktraceLocation instances to their corresponding pprof Location objects;
+        # there's a 1:1 correspondence, since BacktraceLocations were already deduped
+        def initialize_locations_hash
+          sequence = Utils::Sequence.new(1)
+          Hash.new do |locations_hash, backtrace_location|
+            locations_hash[backtrace_location] = build_location(sequence.next, backtrace_location)
+          end
         end
 
         def encode_profile(profile)
@@ -37,7 +51,7 @@ module Datadog
             sample_type: @sample_types.messages,
             sample: @samples,
             mapping: @mappings.messages,
-            location: @locations.messages,
+            location: @locations.values,
             function: @functions.messages,
             string_table: @string_table.strings
           )
@@ -51,45 +65,29 @@ module Datadog
         end
 
         def build_locations(backtrace_locations, length)
-          locations = backtrace_locations.collect do |backtrace_location|
-            @locations.fetch(
-              # Filename
-              backtrace_location.path,
-              # Line number
-              backtrace_location.lineno,
-              # Function name
-              backtrace_location.base_label,
-              # Build function
-              &method(:build_location)
-            )
-          end
+          locations = backtrace_locations.collect { |backtrace_location| @locations[backtrace_location] }
 
           omitted = length - backtrace_locations.length
 
           # Add placeholder stack frame if frames were truncated
           if omitted > 0
             desc = omitted == 1 ? DESC_FRAME_OMITTED : DESC_FRAMES_OMITTED
-            locations << @locations.fetch(
-              ''.freeze,
-              0,
-              "#{omitted} #{desc}",
-              &method(:build_location)
-            )
+            locations << @locations[Profiling::BacktraceLocation.new('', 0, "#{omitted} #{desc}")]
           end
 
           locations
         end
 
-        def build_location(id, filename, line_number, function_name = nil)
+        def build_location(id, backtrace_location)
           Perftools::Profiles::Location.new(
             id: id,
             line: [build_line(
               @functions.fetch(
-                filename,
-                function_name,
-                &method(:build_function)
+                backtrace_location.path,
+                backtrace_location.base_label,
+                &@build_function
               ).id,
-              line_number
+              backtrace_location.lineno
             )]
           )
         end

data/lib/ddtrace/profiling/pprof/converter.rb

@@ -62,6 +62,10 @@ module Datadog
           Array.new(@sample_type_mappings.length, Datadog::Ext::Profiling::Pprof::SAMPLE_VALUE_NO_VALUE)
         end
 
+        def debug_statistics
+          # Empty; can be used by subclasses to report a string containing debug statistics to be logged
+        end
+
         # Represents a grouped event
         # 'sample' is an example event object from the group.
         # 'values' is the the summation of the group's sample values

data/lib/ddtrace/profiling/pprof/stack_sample.rb

@@ -7,6 +7,9 @@ module Datadog
   module Profiling
     module Pprof
       # Builds a profile from a StackSample
+      #
+      # NOTE: This class may appear stateless but is in fact stateful; a new instance should be created for every
+      # encoded profile.
       class StackSample < Converter
         SAMPLE_TYPES = {
           cpu_time_ns: [
@@ -23,6 +26,13 @@ module Datadog
           SAMPLE_TYPES
         end
 
+        def initialize(*_)
+          super
+
+          @processed_unique_stacks = 0
+          @processed_with_trace_ids = 0
+        end
+
         def add_events!(stack_samples)
           new_samples = build_samples(stack_samples)
           builder.samples.concat(new_samples)
@@ -35,6 +45,7 @@ module Datadog
         def build_samples(stack_samples)
           groups = group_events(stack_samples, &method(:stack_sample_group_key))
           groups.collect do |_group_key, group|
+            @processed_unique_stacks += 1
             build_sample(group.sample, group.values)
           end
         end
@@ -46,7 +57,7 @@ module Datadog
           )
 
           Perftools::Profiles::Sample.new(
-            location_id: locations.collect(&:id),
+            location_id: locations.collect { |location| location['id'.freeze] },
             value: values,
             label: build_sample_labels(stack_sample)
           )
@@ -69,6 +80,7 @@ module Datadog
           ]
 
           unless stack_sample.trace_id.nil? || stack_sample.trace_id.zero?
+            @processed_with_trace_ids += 1
             labels << Perftools::Profiles::Label.new(
               key: builder.string_table.fetch(Datadog::Ext::Profiling::Pprof::LABEL_KEY_TRACE_ID),
               str: builder.string_table.fetch(stack_sample.trace_id.to_s)
@@ -84,6 +96,10 @@ module Datadog
 
           labels
         end
+
+        def debug_statistics
+          "unique stacks: #{@processed_unique_stacks}, of which had active traces: #{@processed_with_trace_ids}"
+        end
       end
     end
   end

data/lib/ddtrace/profiling/pprof/template.rb

@@ -75,6 +75,10 @@ module Datadog
           converters[event_class].add_events!(events)
         end
 
+        def debug_statistics
+          converters.values.map(&:debug_statistics).join(', ')
+        end
+
         def to_pprof
           profile = builder.build_profile
           data = builder.encode_profile(profile)

data/lib/ddtrace/profiling/profiler.rb

@@ -16,6 +16,8 @@ module Datadog
     end
 
     def shutdown!
+      Datadog.logger.debug('Shutting down profiler')
+
       collectors.each do |collector|
         collector.enabled = false
         collector.stop(true)

data/lib/ddtrace/profiling/recorder.rb

@@ -16,6 +16,9 @@ module Datadog
         event_classes.each do |event_class|
           @buffers[event_class] = Profiling::Buffer.new(max_size)
         end
+
+        # Event classes can only be added ahead of time
+        @buffers.freeze
       end
 
       def [](event_class)
@@ -59,6 +62,11 @@ module Datadog
         )
       end
 
+      # NOTE: Remember that if the recorder is being accessed by multiple threads, this is an inherently racy operation.
+      def empty?
+        @buffers.values.all?(&:empty?)
+      end
+
       # Error when event of an unknown type is used with the Recorder
       class UnknownEventError < StandardError
         attr_reader :event_class

data/lib/ddtrace/profiling/scheduler.rb

@@ -5,31 +5,36 @@ require 'ddtrace/workers/polling'
 
 module Datadog
   module Profiling
-    # Periodically (every DEFAULT_INTERVAL seconds) takes data from the `Recorder` and pushes them to all configured
+    # Periodically (every DEFAULT_INTERVAL_SECONDS) takes data from the `Recorder` and pushes them to all configured
     # `Exporter`s. Runs on its own background thread.
     class Scheduler < Worker
       include Workers::Polling
 
-      DEFAULT_INTERVAL = 60
-      MIN_INTERVAL = 0
+      DEFAULT_INTERVAL_SECONDS = 60
+      MIN_INTERVAL_SECONDS = 0
 
       attr_reader \
         :exporters,
         :recorder
 
-      def initialize(recorder, exporters, options = {})
+      def initialize(
+        recorder,
+        exporters,
+        fork_policy: Workers::Async::Thread::FORK_POLICY_RESTART, # Restart in forks by default
+        interval: DEFAULT_INTERVAL_SECONDS,
+        enabled: true
+      )
         @recorder = recorder
         @exporters = [exporters].flatten
 
         # Workers::Async::Thread settings
-        # Restart in forks by default
-        self.fork_policy = options[:fork_policy] || Workers::Async::Thread::FORK_POLICY_RESTART
+        self.fork_policy = fork_policy
 
         # Workers::IntervalLoop settings
-        self.loop_base_interval = options[:interval] || DEFAULT_INTERVAL
+        self.loop_base_interval = interval
 
         # Workers::Polling settings
-        self.enabled = options.key?(:enabled) ? options[:enabled] == true : true
+        self.enabled = enabled
       end
 
       def start
@@ -37,7 +42,18 @@ module Datadog
       end
 
       def perform
-        flush_and_wait
+        # A profiling flush may be called while the VM is shutting down, to report the last profile. When we do so,
+        # we impose a strict timeout. This means this last profile may or may not be sent, depending on if the flush can
+        # successfully finish in the strict timeout.
+        # This can be somewhat confusing (why did it not get reported?), so let's at least log what happened.
+        interrupted = true
+
+        begin
+          flush_and_wait
+          interrupted = false
+        ensure
+          Datadog.logger.debug('#flush was interrupted or failed before it could complete') if interrupted
+        end
       end
 
       def loop_back_off?
@@ -51,6 +67,21 @@ module Datadog
         recorder.flush
       end
 
+      # Configure Workers::IntervalLoop to not report immediately when scheduler starts
+      #
+      # When a scheduler gets created (or reset), we don't want it to immediately try to flush; we want it to wait for
+      # the loop wait time first. This avoids an issue where the scheduler reported a mostly-empty profile if the
+      # application just started but this thread took a bit longer so there's already samples in the recorder.
+      def loop_wait_before_first_iteration?
+        true
+      end
+
+      def work_pending?
+        !recorder.empty?
+      end
+
+      private
+
       def flush_and_wait
         run_time = Datadog::Utils::Time.measure do
           flush_events
@@ -58,7 +89,7 @@ module Datadog
 
         # Update wait time to try to wake consistently on time.
         # Don't drop below the minimum interval.
-        self.loop_wait_time = [loop_base_interval - run_time, MIN_INTERVAL].max
+        self.loop_wait_time = [loop_base_interval - run_time, MIN_INTERVAL_SECONDS].max
       end
 
       def flush_events
@@ -71,8 +102,9 @@ module Datadog
             begin
               exporter.export(flush)
             rescue StandardError => e
-              error_details = "Cause: #{e} Location: #{e.backtrace.first}"
-              Datadog.logger.error("Unable to export #{flush.event_count} profiling events. #{error_details}")
+              Datadog.logger.error(
+                "Unable to export #{flush.event_count} profiling events. Cause: #{e} Location: #{e.backtrace.first}"
+              )
             end
           end
         end

data/lib/ddtrace/profiling/tasks/setup.rb

@@ -18,7 +18,7 @@ module Datadog
               activate_cpu_extensions
               setup_at_fork_hooks
             rescue StandardError, ScriptError => e
-              log "[DDTRACE] Main extensions unavailable. Cause: #{e.message} Location: #{e.backtrace.first}"
+              Datadog.logger.warn { "Profiler extensions unavailable. Cause: #{e.message} Location: #{e.backtrace.first}" }
             end
           end
         end
@@ -29,22 +29,27 @@ module Datadog
           if Ext::Forking.supported?
             Ext::Forking.apply!
           elsif Datadog.configuration.profiling.enabled
-            # Log warning if profiling was supposed to be activated.
-            log '[DDTRACE] Forking extensions skipped; forking not supported.'
+            Datadog.logger.debug('Profiler forking extensions skipped; forking not supported.')
           end
         rescue StandardError, ScriptError => e
-          log "[DDTRACE] Forking extensions unavailable. Cause: #{e.message} Location: #{e.backtrace.first}"
+          Datadog.logger.warn do
+            "Profiler forking extensions unavailable. Cause: #{e.message} Location: #{e.backtrace.first}"
+          end
         end
 
         def activate_cpu_extensions
           if Ext::CPU.supported?
             Ext::CPU.apply!
           elsif Datadog.configuration.profiling.enabled
-            # Log warning if profiling was supposed to be activated.
-            log "[DDTRACE] CPU profiling skipped because native CPU time is not supported: #{Ext::CPU.unsupported_reason}."
+            Datadog.logger.info do
+              'CPU time profiling skipped because native CPU time is not supported: ' \
+              "#{Ext::CPU.unsupported_reason}. Profiles containing Wall time will still be reported."
+            end
           end
         rescue StandardError, ScriptError => e
-          log "[DDTRACE] CPU profiling unavailable. Cause: #{e.message} Location: #{e.backtrace.first}"
+          Datadog.logger.warn do
+            "Profiler CPU profiling extensions unavailable. Cause: #{e.message} Location: #{e.backtrace.first}"
+          end
         end
 
         def setup_at_fork_hooks
@@ -61,16 +66,11 @@ module Datadog
                 # Restart profiler, if enabled
                 Datadog.profiler.start if Datadog.profiler
               rescue StandardError => e
-                log "[DDTRACE] Error during post-fork hooks. Cause: #{e.message} Location: #{e.backtrace.first}"
+                Datadog.logger.warn { "Error during post-fork hooks. Cause: #{e.message} Location: #{e.backtrace.first}" }
               end
             end
           end
         end
-
-        def log(message)
-          # Print to STDOUT for now because logging may not be setup yet...
-          puts message
-        end
       end
     end
   end