ddtrace 0.47.0 → 0.48.0

data/lib/ddtrace/contrib/active_record/configuration/makara_resolver.rb

@@ -0,0 +1,30 @@
+module Datadog
+  module Contrib
+    module ActiveRecord
+      module Configuration
+        # The `makara` gem has the concept of **role**, which can be
+        # inferred from the configuration `name`, in the form of:
+        # `master/0`, `replica/0`, `replica/1`, etc.
+        # The first part of this string is the database role.
+        #
+        # This allows the matching of a connection based on its role,
+        # instead of connection-specific information.
+        module MakaraResolver
+          def normalize_for_config(active_record_config)
+            hash = super
+            hash[:makara_role] = active_record_config[:makara_role]
+            hash
+          end
+
+          def normalize_for_resolve(active_record_config)
+            hash = super
+
+            hash[:makara_role] = active_record_config[:name].split('/')[0].to_s if active_record_config[:name].is_a?(String)
+
+            hash
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ddtrace/contrib/active_record/configuration/resolver.rb

@@ -1,5 +1,6 @@
 require 'ddtrace/contrib/configuration/resolver'
 require 'ddtrace/vendor/active_record/connection_specification'
+require_relative 'makara_resolver'
 
 module Datadog
   module Contrib
@@ -28,6 +29,8 @@ module Datadog
         # When more than one configuration could be matched, the last one to match is selected,
         # based on addition order (`#add`).
         class Resolver < Contrib::Configuration::Resolver
+          prepend MakaraResolver
+
           def initialize(active_record_configuration = nil)
             super()
 
@@ -50,7 +53,7 @@ module Datadog
           def resolve(db_config)
             active_record_config = resolve_connection_key(db_config).symbolize_keys
 
-            hash = normalize(active_record_config)
+            hash = normalize_for_resolve(active_record_config)
 
             # Hashes in Ruby maintain insertion order
             _, config = @configurations.reverse_each.find do |matcher, _|
@@ -73,7 +76,7 @@ module Datadog
 
           def parse_matcher(matcher)
             resolved_pattern = resolve_connection_key(matcher).symbolize_keys
-            normalized = normalize(resolved_pattern)
+            normalized = normalize_for_config(resolved_pattern)
 
             # Remove empty fields to allow for partial matching
             normalized.reject! { |_, v| v.nil? }
@@ -112,7 +115,7 @@ module Datadog
 
           # Extract only fields we'd like to match
           # from the ActiveRecord configuration.
-          def normalize(active_record_config)
+          def normalize_for_config(active_record_config)
             {
               adapter: active_record_config[:adapter],
               host: active_record_config[:host],
@@ -121,6 +124,9 @@ module Datadog
               username: active_record_config[:username]
             }
           end
+
+          # Both resolvers perform the same operations for this implementation, but can be specialized
+          alias_method :normalize_for_resolve, :normalize_for_config
         end
       end
     end

data/lib/ddtrace/contrib/resque/configuration/settings.rb

@@ -23,7 +23,23 @@ module Datadog
           end
 
           option :service_name, default: Ext::SERVICE_NAME
-          option :workers, default: []
+
+          # A list Ruby worker classes to be instrumented.
+          # The value of `nil` has special semantics: it instruments all workers dynamically.
+          #
+          # TODO: 1.0: Automatic patching should be the default behavior.
+          # We should not provide this option in the future,
+          # as our integrations should always instrument all possible scenarios when feasible.
+          option :workers, default: nil do |o|
+            o.on_set do |value|
+              unless value.nil?
+                Datadog.logger.warn(
+                  "DEPRECATED: Resque integration now instruments all workers. \n" \
+                  'The `workers:` option is unnecessary and will be removed in the future.'
+                )
+              end
+            end
+          end
           option :error_handler, default: Datadog::Tracer::DEFAULT_ON_ERROR
         end
       end

data/lib/ddtrace/contrib/resque/patcher.rb

@@ -17,11 +17,11 @@ module Datadog
 
         def patch
           require_relative 'resque_job'
-          get_option(:workers).each { |worker| worker.extend(ResqueJob) }
-        end
 
-        def get_option(option)
-          Datadog.configuration[:resque].get_option(option)
+          ::Resque::Job.send(:prepend, Resque::Job)
+
+          workers = Datadog.configuration[:resque][:workers] || []
+          workers.each { |worker| worker.extend(ResqueJob) }
         end
       end
     end

data/lib/ddtrace/contrib/resque/resque_job.rb

@@ -7,9 +7,30 @@ require 'resque'
 module Datadog
   module Contrib
     module Resque
+      # Automatically configures jobs with {ResqueJob} plugin.
+      module Job
+        def perform
+          if Datadog.configuration[:resque][:workers].nil?
+            job = payload_class
+            job.extend(Datadog::Contrib::Resque::ResqueJob) unless job.is_a? Datadog::Contrib::Resque::ResqueJob
+          end
+        ensure
+          super
+        end
+      end
+
       # Uses Resque job hooks to create traces
       module ResqueJob
-        def around_perform(*args)
+        # `around_perform` hooks are executed in alphabetical order.
+        # we use the lowest printable character that allows for an inline
+        # method definition ('0'), alongside our naming prefix for identification.
+        #
+        # We could, in theory, use any character (e.g "\x00"), but this will lead
+        # to unreadable stack traces that contain this method call.
+        #
+        # We could also just use `around_perform` but this might override the user's
+        # own method.
+        def around_perform0_ddtrace(*args)
           return yield unless datadog_configuration && tracer
 
           tracer.trace(Ext::SPAN_JOB, span_options) do |span|

data/lib/ddtrace/contrib/shoryuken/configuration/settings.rb

@@ -23,6 +23,7 @@ module Datadog
 
           option :service_name, default: Ext::SERVICE_NAME
           option :error_handler, default: Datadog::Tracer::DEFAULT_ON_ERROR
+          option :tag_body, default: false
         end
       end
     end

data/lib/ddtrace/contrib/shoryuken/tracer.rb

@@ -12,8 +12,12 @@ module Datadog
         end
 
         def call(worker_instance, queue, sqs_msg, body)
-          @tracer.trace(Ext::SPAN_JOB, service: @shoryuken_service, span_type: Datadog::Ext::AppTypes::WORKER,
-                                       on_error: @error_handler) do |span|
+          @tracer.trace(
+            Ext::SPAN_JOB,
+            service: @shoryuken_service,
+            span_type: Datadog::Ext::AppTypes::WORKER,
+            on_error: @error_handler
+          ) do |span|
             # Set analytics sample rate
             if Contrib::Analytics.enabled?(configuration[:analytics_enabled])
               Contrib::Analytics.set_sample_rate(span, configuration[:analytics_sample_rate])
@@ -26,7 +30,7 @@ module Datadog
             span.set_tag(Ext::TAG_JOB_ID, sqs_msg.message_id)
             span.set_tag(Ext::TAG_JOB_QUEUE, queue)
             span.set_tag(Ext::TAG_JOB_ATTRIBUTES, sqs_msg.attributes) if sqs_msg.respond_to?(:attributes)
-            span.set_tag(Ext::TAG_JOB_BODY, body)
+            span.set_tag(Ext::TAG_JOB_BODY, body) if configuration[:tag_body]
 
             yield
           end

data/lib/ddtrace/diagnostics/environment_logger.rb

@@ -12,7 +12,7 @@ module Datadog
         # Outputs environment information to {Datadog.logger}.
         # Executes only for the lifetime of the program.
         def log!(transport_responses)
-          return if @executed || !log?
+          return if (defined?(@executed) && @executed) || !log?
 
           @executed = true
 

data/lib/ddtrace/error.rb

@@ -16,6 +16,8 @@ module Datadog
 
     def initialize(type = nil, message = nil, backtrace = nil)
       backtrace = Array(backtrace).join("\n")
+
+      # DEV: We should measure if `Utils.utf8_encode` is still needed in practice.
       @type = Utils.utf8_encode(type)
       @message = Utils.utf8_encode(message)
       @backtrace = Utils.utf8_encode(backtrace)

data/lib/ddtrace/ext/profiling.rb

@@ -0,0 +1,52 @@
+module Datadog
+  module Ext
+    module Profiling
+      ENV_ENABLED = 'DD_PROFILING_ENABLED'.freeze
+      ENV_UPLOAD_TIMEOUT = 'DD_PROFILING_UPLOAD_TIMEOUT'.freeze
+      ENV_MAX_FRAMES = 'DD_PROFILING_MAX_FRAMES'.freeze
+
+      module Pprof
+        LABEL_KEY_SPAN_ID = 'span id'.freeze
+        LABEL_KEY_THREAD_ID = 'thread id'.freeze
+        LABEL_KEY_TRACE_ID = 'trace id'.freeze
+        SAMPLE_VALUE_NO_VALUE = 0
+        VALUE_TYPE_CPU = 'cpu-time'.freeze
+        VALUE_TYPE_WALL = 'wall-time'.freeze
+        VALUE_UNIT_NANOSECONDS = 'nanoseconds'.freeze
+      end
+
+      module Transport
+        module HTTP
+          URI_TEMPLATE_DD_API = 'https://intake.profile.%s/'.freeze
+
+          FORM_FIELD_DATA = 'data[0]'.freeze
+          FORM_FIELD_FORMAT = 'format'.freeze
+          FORM_FIELD_FORMAT_PPROF = 'pprof'.freeze
+          FORM_FIELD_RECORDING_END = 'recording-end'.freeze
+          FORM_FIELD_RECORDING_START = 'recording-start'.freeze
+          FORM_FIELD_RUNTIME = 'runtime'.freeze
+          FORM_FIELD_RUNTIME_ID = 'runtime-id'.freeze
+          FORM_FIELD_TAG_ENV = 'env'.freeze
+          FORM_FIELD_TAG_HOST = 'host'.freeze
+          FORM_FIELD_TAG_LANGUAGE = 'language'.freeze
+          FORM_FIELD_TAG_PROFILER_VERSION = 'profiler_version'.freeze
+          FORM_FIELD_TAG_RUNTIME = 'runtime'.freeze
+          FORM_FIELD_TAG_RUNTIME_ENGINE = 'runtime_engine'.freeze
+          FORM_FIELD_TAG_RUNTIME_ID = 'runtime-id'.freeze
+          FORM_FIELD_TAG_RUNTIME_PLATFORM = 'runtime_platform'.freeze
+          FORM_FIELD_TAG_RUNTIME_VERSION = 'runtime_version'.freeze
+          FORM_FIELD_TAG_SERVICE = 'service'.freeze
+          FORM_FIELD_TAG_VERSION = 'version'.freeze
+          FORM_FIELD_TAGS = 'tags'.freeze
+          FORM_FIELD_TYPES = 'types[0]'.freeze
+          FORM_FIELD_TYPES_AUTO = 'auto'.freeze
+
+          HEADER_CONTENT_TYPE = 'Content-Type'.freeze
+          HEADER_CONTENT_TYPE_OCTET_STREAM = 'application/octet-stream'.freeze
+
+          PPROF_DEFAULT_FILENAME = 'profile.pb.gz'.freeze
+        end
+      end
+    end
+  end
+end

data/lib/ddtrace/ext/transport.rb

@@ -8,6 +8,7 @@ module Datadog
         ENV_DEFAULT_PORT = 'DD_TRACE_AGENT_PORT'.freeze
         ENV_DEFAULT_URL = 'DD_TRACE_AGENT_URL'.freeze
         HEADER_CONTAINER_ID = 'Datadog-Container-ID'.freeze
+        HEADER_DD_API_KEY = 'DD-API-KEY'.freeze
         HEADER_META_LANG = 'Datadog-Meta-Lang'.freeze
         HEADER_META_LANG_VERSION = 'Datadog-Meta-Lang-Version'.freeze
         HEADER_META_LANG_INTERPRETER = 'Datadog-Meta-Lang-Interpreter'.freeze

data/lib/ddtrace/metrics.rb

@@ -120,6 +120,10 @@ module Datadog
       metrics.each { |m| send(m.type, *[m.name, m.value, m.options].compact) }
     end
 
+    def close
+      @statsd.close if @statsd && @statsd.respond_to?(:close)
+    end
+
     Metric = Struct.new(:type, :name, :value, :options) do
       def initialize(*args)
         super

data/lib/ddtrace/profiling.rb

@@ -0,0 +1,54 @@
+module Datadog
+  # Contains profiler for generating stack profiles, etc.
+  module Profiling
+    module_function
+
+    GOOGLE_PROTOBUF_MINIMUM_VERSION = Gem::Version.new('3.0')
+
+    def supported?
+      google_protobuf_supported?
+    end
+
+    def native_cpu_time_supported?
+      require 'ddtrace/profiling/ext/cpu'
+      Ext::CPU.supported?
+    end
+
+    def google_protobuf_supported?
+      RUBY_PLATFORM != 'java' \
+        && !Gem.loaded_specs['google-protobuf'].nil? \
+        && Gem.loaded_specs['google-protobuf'].version >= GOOGLE_PROTOBUF_MINIMUM_VERSION \
+        && !defined?(@failed_to_load_protobuf)
+    end
+
+    def load_profiling
+      require 'ddtrace/profiling/ext/cpu'
+      require 'ddtrace/profiling/ext/forking'
+
+      require 'ddtrace/profiling/collectors/stack'
+      require 'ddtrace/profiling/exporter'
+      require 'ddtrace/profiling/recorder'
+      require 'ddtrace/profiling/scheduler'
+      require 'ddtrace/profiling/tasks/setup'
+      require 'ddtrace/profiling/transport/io'
+      require 'ddtrace/profiling/transport/http'
+      require 'ddtrace/profiling/profiler'
+
+      begin
+        require 'ddtrace/profiling/pprof/pprof_pb' if google_protobuf_supported?
+      rescue LoadError => e
+        @failed_to_load_protobuf = true
+        Kernel.warn(
+          "[DDTRACE] Error while loading google-protobuf gem. Cause: '#{e.message}' Location: '#{e.backtrace.first}'. " \
+          'This can happen when google-protobuf is missing its native components. ' \
+          'To fix this, try removing and reinstalling the gem, forcing it to recompile the components: ' \
+          '`gem uninstall google-protobuf -a; BUNDLE_FORCE_RUBY_PLATFORM=true bundle install`. ' \
+          'If the error persists, please contact support via <https://docs.datadoghq.com/help/> or ' \
+          'file a bug at <https://github.com/DataDog/dd-trace-rb/blob/master/CONTRIBUTING.md#found-a-bug>.'
+        )
+      end
+    end
+
+    load_profiling if supported?
+  end
+end

data/lib/ddtrace/profiling/backtrace_location.rb

@@ -0,0 +1,32 @@
+module Datadog
+  module Profiling
+    # Entity class used to represent an entry in a stack trace.
+    # Its fields are a simplified struct version of `Thread::Backtrace::Location`.
+    class BacktraceLocation
+      attr_reader \
+        :base_label,
+        :lineno,
+        :path,
+        :hash
+
+      def initialize(
+        base_label,
+        lineno,
+        path
+      )
+        @base_label = base_label
+        @lineno = lineno
+        @path = path
+        @hash = [base_label, lineno, path].hash
+      end
+
+      def ==(other)
+        hash == other.hash
+      end
+
+      def eql?(other)
+        hash == other.hash
+      end
+    end
+  end
+end

data/lib/ddtrace/profiling/buffer.rb

@@ -0,0 +1,41 @@
+require 'ddtrace/buffer'
+require 'ddtrace/utils/string_table'
+require 'ddtrace/utils/object_set'
+
+module Datadog
+  module Profiling
+    # Profiling buffer that stores profiling events. The buffer has a maximum size and when
+    # the buffer is full, a random event is discarded. This class is thread-safe.
+    class Buffer < Datadog::ThreadSafeBuffer
+      def initialize(*args)
+        super
+        @caches = {}
+        @string_table = Utils::StringTable.new
+      end
+
+      def cache(cache_name)
+        synchronize do
+          @caches[cache_name] ||= Utils::ObjectSet.new
+        end
+      end
+
+      def string_table
+        synchronize do
+          @string_table
+        end
+      end
+
+      protected
+
+      def drain!
+        items = super
+
+        # Clear caches
+        @caches = {}
+        @string_table = Utils::StringTable.new
+
+        items
+      end
+    end
+  end
+end

data/lib/ddtrace/profiling/collectors/stack.rb

@@ -0,0 +1,253 @@
+require 'ddtrace/profiling/backtrace_location'
+require 'ddtrace/profiling/events/stack'
+require 'ddtrace/utils/only_once'
+require 'ddtrace/utils/time'
+require 'ddtrace/worker'
+require 'ddtrace/workers/polling'
+
+module Datadog
+  module Profiling
+    module Collectors
+      # Collects stack trace samples from Ruby threads for both CPU-time (if available) and wall-clock.
+      # Runs on its own background thread.
+      #
+      class Stack < Worker
+        include Workers::Polling
+
+        DEFAULT_MAX_TIME_USAGE_PCT = 2.0
+        MIN_INTERVAL = 0.01
+        THREAD_LAST_CPU_TIME_KEY = :datadog_profiler_last_cpu_time
+
+        attr_reader \
+          :recorder,
+          :max_frames,
+          :ignore_thread,
+          :max_time_usage_pct,
+          :thread_api
+
+        def initialize(
+          recorder,
+          max_frames: nil,
+          ignore_thread: nil,
+          max_time_usage_pct: DEFAULT_MAX_TIME_USAGE_PCT,
+          thread_api: Thread,
+          fork_policy: Workers::Async::Thread::FORK_POLICY_RESTART, # Restart in forks by default
+          interval: MIN_INTERVAL,
+          enabled: true
+        )
+          @recorder = recorder
+          # TODO: Make this a required named argument after we drop support for Ruby 2.0
+          @max_frames = max_frames || raise(ArgumentError, 'missing keyword :max_frames')
+          @ignore_thread = ignore_thread
+          @max_time_usage_pct = max_time_usage_pct
+          @thread_api = thread_api
+
+          # Workers::Async::Thread settings
+          self.fork_policy = fork_policy
+
+          # Workers::IntervalLoop settings
+          self.loop_base_interval = interval
+
+          # Workers::Polling settings
+          self.enabled = enabled
+
+          @warn_about_missing_cpu_time_instrumentation_only_once = Datadog::Utils::OnlyOnce.new
+        end
+
+        def start
+          @last_wall_time = Datadog::Utils::Time.get_time
+          reset_cpu_time_tracking
+          perform
+        end
+
+        def perform
+          collect_and_wait
+        end
+
+        def loop_back_off?
+          false
+        end
+
+        def collect_and_wait
+          run_time = Datadog::Utils::Time.measure do
+            collect_events
+          end
+
+          # Update wait time to throttle profiling
+          self.loop_wait_time = compute_wait_time(run_time)
+        end
+
+        def collect_events
+          events = []
+
+          # Compute wall time interval
+          current_wall_time = Datadog::Utils::Time.get_time
+          last_wall_time = if instance_variable_defined?(:@last_wall_time)
+                             @last_wall_time
+                           else
+                             current_wall_time
+                           end
+
+          wall_time_interval_ns = ((current_wall_time - last_wall_time).round(9) * 1e9).to_i
+          @last_wall_time = current_wall_time
+
+          # Collect backtraces from each thread
+          thread_api.list.each do |thread|
+            next unless thread.alive?
+            next if ignore_thread.is_a?(Proc) && ignore_thread.call(thread)
+
+            event = collect_thread_event(thread, wall_time_interval_ns)
+            events << event unless event.nil?
+          end
+
+          # Send events to recorder
+          recorder.push(events) unless events.empty?
+
+          events
+        end
+
+        def collect_thread_event(thread, wall_time_interval_ns)
+          locations = thread.backtrace_locations
+          return if locations.nil?
+
+          # Get actual stack size then trim the stack
+          stack_size = locations.length
+          locations = locations[0..(max_frames - 1)]
+
+          # Convert backtrace locations into structs
+          locations = convert_backtrace_locations(locations)
+
+          thread_id = thread.respond_to?(:native_thread_id) ? thread.native_thread_id : thread.object_id
+          trace_id, span_id = get_trace_identifiers(thread)
+          cpu_time = get_cpu_time_interval!(thread)
+
+          Events::StackSample.new(
+            nil,
+            locations,
+            stack_size,
+            thread_id,
+            trace_id,
+            span_id,
+            cpu_time,
+            wall_time_interval_ns
+          )
+        end
+
+        def get_cpu_time_interval!(thread)
+          # Return if we can't get the current CPU time
+          unless thread.respond_to?(:cpu_time_instrumentation_installed?) && thread.cpu_time_instrumentation_installed?
+            warn_about_missing_cpu_time_instrumentation(thread)
+            return
+          end
+
+          current_cpu_time_ns = thread.cpu_time(:nanosecond)
+
+          # NOTE: This can still be nil even when all of the checks above passed because of a race: there's a bit of
+          # initialization that needs to be done by the thread itself, and it's possible for us to try to sample
+          # *before* the thread had time to finish the initialization
+          return unless current_cpu_time_ns
+
+          last_cpu_time_ns = (thread[THREAD_LAST_CPU_TIME_KEY] || current_cpu_time_ns)
+          interval = current_cpu_time_ns - last_cpu_time_ns
+
+          # Update CPU time for thread
+          thread[THREAD_LAST_CPU_TIME_KEY] = current_cpu_time_ns
+
+          # Return interval
+          interval
+        end
+
+        def get_trace_identifiers(thread)
+          return unless thread.is_a?(::Thread)
+          return unless Datadog.respond_to?(:tracer) && Datadog.tracer.respond_to?(:active_correlation)
+
+          identifier = Datadog.tracer.active_correlation(thread)
+          [identifier.trace_id, identifier.span_id]
+        end
+
+        def compute_wait_time(used_time)
+          # We took used_time to get the last sample.
+          #
+          # What we're computing here is -- if used_time corresponds to max_time_usage_pct of the time we should
+          # spend working, how much is (100% - max_time_usage_pct) of the time?
+          #
+          # For instance, if we took 10ms to sample, and max_time_usage_pct is 1%, then the other 99% is 990ms, which
+          # means we need to sleep for 990ms to guarantee that we don't spend more than 1% of the time working.
+          used_time_ns = used_time * 1e9
+          interval = (used_time_ns / (max_time_usage_pct / 100.0)) - used_time_ns
+          [interval / 1e9, MIN_INTERVAL].max
+        end
+
+        # Convert backtrace locations into structs
+        # Re-use old backtrace location objects if they already exist in the buffer
+        def convert_backtrace_locations(locations)
+          locations.collect do |location|
+            # Re-use existing BacktraceLocation if identical copy, otherwise build a new one.
+            recorder[Events::StackSample].cache(:backtrace_locations).fetch(
+              # Function name
+              location.base_label,
+              # Line number
+              location.lineno,
+              # Filename
+              location.path,
+              # Build function
+              &method(:build_backtrace_location)
+            )
+          end
+        end
+
+        def build_backtrace_location(_id, base_label, lineno, path)
+          string_table = recorder[Events::StackSample].string_table
+
+          Profiling::BacktraceLocation.new(
+            string_table.fetch_string(base_label),
+            lineno,
+            string_table.fetch_string(path)
+          )
+        end
+
+        private
+
+        def warn_about_missing_cpu_time_instrumentation(thread)
+          @warn_about_missing_cpu_time_instrumentation_only_once.run do
+            # Is the profiler thread instrumented? If it is, then we know instrumentation is available, but seems to be
+            # missing on this thread we just found.
+            #
+            # As far as we know, it can be missing due to one the following:
+            #
+            # a) The thread was started before we installed our instrumentation.
+            #    In this case, the fix is to make sure ddtrace gets loaded before any other parts of the application.
+            #
+            # b) The thread was started using the Ruby native APIs (e.g. from a C extension such as ffi).
+            #    We currently have no solution for this case; these threads will always be missing our CPU instrumentation.
+            #
+            # c) The thread was started with `Thread.start`/`Thread.fork` and hasn't yet enabled the instrumentation.
+            #    When threads are started using these APIs, there's a small time window during which the thread has started
+            #    but our code to apply the instrumentation hasn't run yet; in these cases it's just a matter of allowing
+            #    it to run and our instrumentation to be applied.
+            #
+            if thread_api.current.respond_to?(:cpu_time) && thread_api.current.cpu_time
+              Datadog.logger.debug("Detected thread ('#{thread}') with missing CPU profiling instrumentation.")
+            end
+          end
+        end
+
+        # If the profiler is started for a while, stopped and then restarted OR whenever the process forks, we need to
+        # clean up the per-thread cpu time counters we keep, so that the first sample after starting doesn't end up with:
+        #
+        # a) negative time: At least on my test docker container, and on the reliability environment, after the process
+        #    forks, the clock reference changes and (old cpu time - new cpu time) can be < 0
+        #
+        # b) large amount of time: if the profiler was started, then stopped for some amount of time, and then
+        #    restarted, we don't want the first sample to be "blamed" for multiple minutes of CPU time
+        #
+        # By resetting the last cpu time seen, we start with a clean slate every time we start the stack collector.
+        def reset_cpu_time_tracking
+          thread_api.list.each do |thread|
+            thread[THREAD_LAST_CPU_TIME_KEY] = nil if thread[THREAD_LAST_CPU_TIME_KEY]
+          end
+        end
+      end
+    end
+  end
+end