scout_apm 3.0.0.pre26 → 4.0.2

data/lib/scout_apm/slow_job_record.rb

@@ -23,7 +23,9 @@ module ScoutApm
     attr_reader :git_sha
     attr_reader :truncated_metrics
 
-    def initialize(agent_context, queue_name, job_name, time, total_time, exclusive_time, context, metrics, allocation_metrics, mem_delta, allocations, score, truncated_metrics)
+    attr_reader :span_trace
+
+    def initialize(agent_context, queue_name, job_name, time, total_time, exclusive_time, context, metrics, allocation_metrics, mem_delta, allocations, score, truncated_metrics, span_trace)
       @queue_name = queue_name
       @job_name = job_name
       @time = time
@@ -40,6 +42,8 @@ module ScoutApm
       @score = score
       @truncated_metrics = truncated_metrics
 
+      @span_trace = span_trace
+
       agent_context.logger.debug { "Slow Job [#{metric_name}] - Call Time: #{total_call_time} Mem Delta: #{mem_delta}"}
     end
 

data/lib/scout_apm/slow_policy/age_policy.rb

@@ -0,0 +1,33 @@
+require 'scout_apm/slow_policy/policy'
+
+module ScoutApm::SlowPolicy
+  class AgePolicy < Policy
+    # For each minute we haven't seen an endpoint
+    POINT_MULTIPLIER_AGE = 0.25
+
+    # A hash of Endpoint Name to the last time we stored a slow transaction for it.
+    #
+    # Defaults to a start time that is pretty close to application boot time.
+    # So the "age" of an endpoint we've never seen is the time the application
+    # has been running.
+    attr_reader :last_seen
+
+    def initialize(context)
+      super
+
+      zero_time = Time.now
+      @last_seen = Hash.new { |h, k| h[k] = zero_time }
+    end
+
+    def call(request)
+      # How long has it been since we've seen this?
+      age = Time.now - last_seen[request.unique_name]
+
+      age / 60.0 * POINT_MULTIPLIER_AGE
+    end
+
+    def stored!(request)
+      last_seen[request.unique_name] = Time.now
+    end
+  end
+end

data/lib/scout_apm/slow_policy/percent_policy.rb

@@ -0,0 +1,22 @@
+require 'scout_apm/slow_policy/policy'
+
+module ScoutApm::SlowPolicy
+  class PercentPolicy < Policy
+    # Points for an endpoint's who's throughput * response time is a large % of
+    # overall time spent processing requests
+    POINT_MULTIPLIER_PERCENT_TIME = 2.5
+
+    # Of the total time spent handling endpoints in this app, if this endpoint
+    # is a higher percent, it should get more points.
+    #
+    # A: 20 calls @ 100ms each => 2 seconds of total time
+    # B: 10 calls @ 100ms each => 1 second of total time
+    #
+    # Then A is 66% of the total call time
+    def call(request) # Scale 0.0 - 1.0
+      percent = context.transaction_time_consumed.percent_of_total(request.unique_name)
+
+      percent * POINT_MULTIPLIER_PERCENT_TIME
+    end
+  end
+end

data/lib/scout_apm/slow_policy/percentile_policy.rb

@@ -0,0 +1,24 @@
+require 'scout_apm/slow_policy/policy'
+
+module ScoutApm::SlowPolicy
+  class PercentilePolicy < Policy
+    def call(request)
+      # What approximate percentile was this request?
+      total_time = request.root_layer.total_call_time
+      percentile = context.request_histograms.approximate_quantile_of_value(request.unique_name, total_time)
+
+      if percentile < 40
+        0.4 # Don't put much emphasis on capturing low percentiles.
+      elsif percentile < 60
+        1.4 # Highest here to get mean traces
+      elsif percentile < 90
+        0.7 # Between 60 & 90% is fine.
+      elsif percentile >= 90
+        1.4 # Highest here to get 90+%ile traces
+      else
+        # impossible.
+        percentile
+      end
+    end
+  end
+end

data/lib/scout_apm/slow_policy/policy.rb

@@ -0,0 +1,21 @@
+# Note that this is semi-internal API. You should not need this, and if you do
+# we're here to help at support@scoutapm.com. TrackedRequest doesn't change
+# often, but we can't promise a perfectly stable API for it either.
+module ScoutApm::SlowPolicy
+  class Policy
+    attr_reader :context
+
+    def initialize(context)
+      @context = context
+    end
+
+    def call(request)
+      raise NotImplementedError
+    end
+
+    # Override in subclasses to execute some behavior if the request gets a
+    # slot in the ScoredItemSet. Defaults to no-op
+    def stored!(request)
+    end
+  end
+end

data/lib/scout_apm/slow_policy/speed_policy.rb

@@ -0,0 +1,16 @@
+require 'scout_apm/slow_policy/policy'
+
+module ScoutApm::SlowPolicy
+  class SpeedPolicy < Policy
+    # Adjust speed points. See the function
+    POINT_MULTIPLIER_SPEED = 0.25
+
+    # Time in seconds
+    # Logarithm keeps huge times from swamping the other metrics.
+    # 1+ is necessary to keep the log function in positive territory.
+    def call(request)
+      total_time = request.root_layer.total_call_time
+      Math.log(1 + total_time) * POINT_MULTIPLIER_SPEED
+    end
+  end
+end

data/lib/scout_apm/slow_request_policy.rb

@@ -3,43 +3,29 @@
 
 module ScoutApm
   class SlowRequestPolicy
-    CAPTURE_TYPES = [
-      CAPTURE_DETAIL  = "capture_detail",
-      CAPTURE_NONE    = "capture_none",
-    ]
-
-    # Adjust speed points. See the function
-    POINT_MULTIPLIER_SPEED = 0.25
-
-    # For each minute we haven't seen an endpoint
-    POINT_MULTIPLIER_AGE = 0.25
-
-    # Outliers are worth up to "1000ms" of weight
-    POINT_MULTIPLIER_PERCENTILE = 1.0
-
-    # Points for an endpoint's who's throughput * response time is a large % of
-    # overall time spent processing requests
-    POINT_MULTIPLIER_PERCENT_TIME = 2.5
-
-    # A hash of Endpoint Name to the last time we stored a slow transaction for it.
-    #
-    # Defaults to a start time that is pretty close to application boot time.
-    # So the "age" of an endpoint we've never seen is the time the application
-    # has been running.
-    attr_reader :last_seen
-
     # The AgentContext we're running in
     attr_reader :context
+    attr_reader :policies
 
     def initialize(context)
       @context = context
+      @policies = []
+    end
 
-      zero_time = Time.now
-      @last_seen = Hash.new { |h, k| h[k] = zero_time }
+    def add_default_policies
+      add(SlowPolicy::SpeedPolicy.new(context))
+      add(SlowPolicy::PercentilePolicy.new(context))
+      add(SlowPolicy::AgePolicy.new(context))
+      add(SlowPolicy::PercentilePolicy.new(context))
     end
 
-    def stored!(request)
-      last_seen[request.unique_name] = Time.now
+    # policy is an object that behaves like a policy (responds to .call(req) for the score, and .store!(req))
+    def add(policy)
+      unless policy.respond_to?(:call) && policy.respond_to?(:stored!)
+        raise "SlowRequestPolicy must implement policy api call(req) and stored!(req)"
+      end
+
+      @policies << policy
     end
 
     # Determine if this request trace should be fully analyzed by scoring it
@@ -56,56 +42,11 @@ module ScoutApm
         return -1 # A negative score, should never be good enough to store.
       end
 
-      total_time = request.root_layer.total_call_time
-
-      # How long has it been since we've seen this?
-      age = Time.now - last_seen[unique_name]
-
-      # What approximate percentile was this request?
-      percentile = context.request_histograms.approximate_quantile_of_value(unique_name, total_time)
-
-      percent_of_total_time = context.transaction_time_consumed.percent_of_total(unique_name)
-
-      return speed_points(total_time) + percentile_points(percentile) + age_points(age) + percent_time_points(percent_of_total_time)
-    end
-
-    private
-
-    # Time in seconds
-    # Logarithm keeps huge times from swamping the other metrics.
-    # 1+ is necessary to keep the log function in positive territory.
-    def speed_points(time)
-      Math.log(1 + time) * POINT_MULTIPLIER_SPEED
-    end
-
-    def percentile_points(percentile)
-      if percentile < 40
-        0.4 # Don't put much emphasis on capturing low percentiles.
-      elsif percentile < 60
-        1.4 # Highest here to get mean traces
-      elsif percentile < 90
-        0.7 # Between 60 & 90% is fine.
-      elsif percentile >= 90
-        1.4 # Highest here to get 90+%ile traces
-      else
-        # impossible.
-        percentile
-      end
-    end
-
-    def age_points(age)
-      age / 60.0 * POINT_MULTIPLIER_AGE
+      policies.map{ |p| p.call(request) }.sum
     end
 
-    # Of the total time spent handling endpoints in this app, if this endpoint
-    # is a higher percent, it should get more points.
-    #
-    # A: 20 calls @ 100ms each => 2 seconds of total time
-    # B: 10 calls @ 100ms each => 1 second of total time
-    #
-    # Then A is 66% of the total call time
-    def percent_time_points(percent) # Scale 0.0 - 1.0
-      percent * POINT_MULTIPLIER_PERCENT_TIME
+    def stored!(request)
+      policies.each{ |p| p.stored!(request) }
     end
   end
 end

data/lib/scout_apm/slow_transaction.rb

@@ -13,13 +13,14 @@ module ScoutApm
     attr_reader :prof
     attr_reader :mem_delta
     attr_reader :allocations
+    attr_reader :span_trace
     attr_accessor :hostname # hack - we need to reset these server side.
     attr_accessor :seconds_since_startup # hack - we need to reset these server side.
     attr_accessor :git_sha # hack - we need to reset these server side.
 
     attr_reader :truncated_metrics # True/False that says if we had to truncate the metrics of this trace
 
-    def initialize(agent_context, uri, metric_name, total_call_time, metrics, allocation_metrics, context, time, raw_stackprof, mem_delta, allocations, score, truncated_metrics)
+    def initialize(agent_context, uri, metric_name, total_call_time, metrics, allocation_metrics, context, time, raw_stackprof, mem_delta, allocations, score, truncated_metrics, span_trace)
       @uri = uri
       @metric_name = metric_name
       @total_call_time = total_call_time
@@ -35,6 +36,7 @@ module ScoutApm
       @score = score
       @git_sha = agent_context.environment.git_revision.sha
       @truncated_metrics = truncated_metrics
+      @span_trace = span_trace
 
       agent_context.logger.debug { "Slow Request [#{uri}] - Call Time: #{total_call_time} Mem Delta: #{mem_delta} Score: #{score}"}
     end

data/lib/scout_apm/store.rb

@@ -5,8 +5,10 @@ module ScoutApm
   class Store
     def initialize(context)
       @context = context
-      @mutex = Mutex.new
-      @reporting_periods = Hash.new { |h,k| h[k] = StoreReportingPeriod.new(k, @context) }
+      @mutex = Monitor.new
+      @reporting_periods = Hash.new { |h,k|
+        @mutex.synchronize { h[k] = StoreReportingPeriod.new(k, @context) }
+      }
       @samplers = []
     end
 
@@ -87,8 +89,13 @@ module ScoutApm
     def write_to_layaway(layaway, force=false)
       logger.debug("Writing to layaway#{" (Forced)" if force}")
 
-      @reporting_periods.select { |time, rp| force || (time.timestamp < current_timestamp.timestamp) }.
-                         each   { |time, rp| write_reporting_period(layaway, time, rp) }
+      to_report = @mutex.synchronize {
+        @reporting_periods.select { |time, rp|
+          force || (time.timestamp < current_timestamp.timestamp)
+        }
+      }
+
+      to_report.each { |time, rp| write_reporting_period(layaway, time, rp) }
     end
 
     # For each tick (minute), be sure we have a reporting period, and that samplers are run for it.
@@ -98,14 +105,12 @@ module ScoutApm
     end
 
     def write_reporting_period(layaway, time, rp)
-      @mutex.synchronize {
         layaway.write_reporting_period(rp)
-      }
     rescue => e
       logger.warn("Failed writing data to layaway file: #{e.message} / #{e.backtrace}")
     ensure
       logger.debug("Before delete, reporting periods length: #{@reporting_periods.size}")
-      deleted_items = @reporting_periods.delete(time)
+      deleted_items = @mutex.synchronize { @reporting_periods.delete(time) }
       logger.debug("After delete, reporting periods length: #{@reporting_periods.size}. Did delete #{deleted_items}")
     end
     private :write_reporting_period
@@ -184,7 +189,6 @@ module ScoutApm
 
   # One period of Storage. Typically 1 minute
   class StoreReportingPeriod
-
     # A ScoredItemSet holding the "best" traces for the period
     attr_reader :request_traces
 

data/lib/scout_apm/tracked_request.rb

@@ -42,6 +42,9 @@ module ScoutApm
     # the name is determined from the name of the Controller or Job layer.
     attr_accessor :name_override
 
+    # A unique, but otherwise meaningless String to identify this request. UUID
+    attr_reader :transaction_id
+
     # When we see these layers, it means a real request is going through the
     # system. We toggle a flag to turn on some slightly more expensive
     # instrumentation (backtrace collection and the like) that would be too
@@ -49,6 +52,10 @@ module ScoutApm
     # see that on Sidekiq.
     REQUEST_TYPES = ["Controller", "Job"]
 
+    # Layers of type 'AutoInstrument' are not recorded if their total_call_time doesn't exceed this threshold.
+    # AutoInstrument layers are frequently of short duration. This throws out this deadweight that is unlikely to be optimized.
+    AUTO_INSTRUMENT_TIMING_THRESHOLD = 5/1_000.0 # units = seconds
+
     def initialize(agent_context, store)
       @agent_context = agent_context
       @store = store #this is passed in so we can use a real store (normal operation) or fake store (instant mode only)
@@ -64,6 +71,7 @@ module ScoutApm
       @mem_start = mem_usage
       @recorder = agent_context.recorder
       @real_request = false
+      @transaction_id = ScoutApm::Utils::TransactionId.new.to_s
       ignore_request! if @recorder.nil?
     end
 
@@ -75,8 +83,6 @@ module ScoutApm
 
       return ignoring_start_layer if ignoring_request?
 
-      layer.start_sampling
-
       start_request(layer) unless @root_layer
 
       if REQUEST_TYPES.include?(layer.type)
@@ -105,10 +111,12 @@ module ScoutApm
         return
       end
 
-      layer.record_traces!
       layer.record_stop_time!
       layer.record_allocations!
 
+      # Must follow layer.record_stop_time! as the total_call_time is used to determine if the layer is significant.
+      return if layer_insignificant?(layer)
+
       @layers[-1].add_child(layer) if @layers.any?
 
       # This must be called before checking if a backtrace should be collected as the call count influences our capture logic.
@@ -121,8 +129,6 @@ module ScoutApm
 
       if finalized?
         stop_request
-      else
-        continue_sampling_for_layers if @agent_context.config.value('profile')
       end
     end
 
@@ -151,6 +157,10 @@ module ScoutApm
     def capture_backtrace?(layer)
       return if ignoring_request?
 
+      # A backtrace has already been recorded. This happens with autoinstruments as
+      # the partial backtrace is set when creating the layer.
+      return false if layer.backtrace
+
       # Never capture backtraces for this kind of layer. The backtrace will
       # always be 100% framework code.
       return false if BACKTRACE_BLACKLIST.include?(layer.type)
@@ -170,6 +180,20 @@ module ScoutApm
       false
     end
 
+    # Returns +true+ if the total call time of AutoInstrument layers exceeds +AUTO_INSTRUMENT_TIMING_THRESHOLD+ and
+    # records a Histogram of insignificant / significant layers by file name.
+    def layer_insignificant?(layer)
+      result = false # default is significant
+      if layer.type == 'AutoInstrument'
+        if layer.total_call_time < AUTO_INSTRUMENT_TIMING_THRESHOLD
+          result = true # not significant
+        end
+        # 0 = not significant, 1 = significant
+        @agent_context.auto_instruments_layer_histograms.add(layer.file_name, (result ? 0 : 1))
+      end
+      result
+    end
+
     # Maintains a lookup Hash of call counts by layer name. Used to determine if we should capture a backtrace.
     def update_call_counts!(layer)
       @call_set[layer.name].update!(layer.desc)
@@ -200,13 +224,6 @@ module ScoutApm
       @layers.none?
     end
 
-    def continue_sampling_for_layers
-      if last_traced_layer = @layers.select{|layer| layer.traced?}.last
-        ScoutApm::Instruments::Stacks.update_indexes(@layers.last.frame_index, @layers.last.trace_index)
-        ScoutApm::Instruments::Stacks.start_sampling
-      end
-    end
-
     # Run at the beginning of the whole request
     #
     # * Capture the first layer as the root_layer
@@ -220,13 +237,8 @@ module ScoutApm
     def stop_request
       @stopping = true
 
-      if @agent_context.config.value('profile')
-        ScoutApm::Instruments::Stacks.stop_sampling(true)
-        ScoutApm::Instruments::Stacks.update_indexes(0, 0)
-      end
-
-      if recorder
-        recorder.record!(self)
+      if @recorder
+        @recorder.record!(self)
       end
     end
 
@@ -234,16 +246,6 @@ module ScoutApm
       @stopping
     end
 
-    # Enable ScoutProf for this thread
-    def enable_profiled_thread!
-        ScoutApm::Instruments::Stacks.add_profiled_thread
-    end
-
-    # Disable ScoutProf for this thread
-    def disable_profiled_thread!
-        ScoutApm::Instruments::Stacks.remove_profiled_thread
-    end
-
     ###################################
     # Annotations
     ###################################
@@ -302,6 +304,8 @@ module ScoutApm
 
       @agent_context.transaction_time_consumed.add(unique_name, root_layer.total_call_time)
 
+      context.add(:transaction_id => transaction_id)
+
       # Make a constant, then call converters.dup.each so it isn't inline?
       converters = {
         :histograms => LayerConverters::Histograms,
@@ -314,6 +318,11 @@ module ScoutApm
 
         :slow_job => LayerConverters::SlowJobConverter,
         :slow_req => LayerConverters::SlowRequestConverter,
+
+        # This is now integrated into the slow_job and slow_req converters, so that
+        # we get the exact same set of traces either way. We can call it
+        # directly when we move away from the legacy trace styles.
+        # :traces => LayerConverters::TraceConverter,
       }
 
       walker = LayerConverters::DepthFirstWalker.new(self.root_layer)
@@ -324,7 +333,7 @@ module ScoutApm
         memo
       end
       walker.walk
-      converter_results = converter_instances.inject({}) do |memo, (slug,i)| 
+      converter_results = converter_instances.inject({}) do |memo, (slug,i)|
         memo[slug] = i.record!
         memo
       end