scout_apm 1.4.6 → 1.5.0.pre

data/lib/scout_apm/serializers/slow_jobs_serializer_to_json.rb

@@ -0,0 +1,29 @@
+module ScoutApm
+  module Serializers
+    class SlowJobsSerializerToJson
+      attr_reader :jobs
+
+      # Jobs is a series of slow job records
+      def initialize(jobs)
+        @jobs = jobs
+      end
+
+      # An array of job records
+      def as_json
+        jobs.map do |job|
+          {
+            "queue" => job.queue_name,
+            "name" => job.job_name,
+            "time" => job.time,
+            "total_time" => job.total_time,
+            "exclusive_time" => job.exclusive_time,
+
+            "metrics" => MetricsToJsonSerializer.new(job.metrics).as_json, # New style of metrics
+            "context" => job.context,
+          }
+        end
+      end
+    end
+  end
+end
+

data/lib/scout_apm/slow_item_set.rb

@@ -0,0 +1,80 @@
+# In order to keep load down, only record a sample of Slow Items (Transactions
+# or Jobs).  In order to make that sampling as fair as possible, follow a basic
+# algorithm:
+#
+# When adding a new Slow Item:
+#  * Just add it if there is an open spot
+#  * If there isn't an open spot, attempt to remove an over-represented
+#    item instead ("attempt_to_evict"). Overrepresented is simply "has more
+#    than @fair number of Matching Items in the set". The fastest of the
+#    overrepresented items is removed.
+#  * If there isn't an open spot, and no Item is valid to evict, drop the
+#    incoming Item without adding.
+#
+# There is no way to remove Items from this set, create a new object
+# for each reporting period.
+#
+# Item must respond to:
+#   #metric_name - string - grouping key to see if one kind of thing is overrepresented
+#   #total_call_time - float - duration of the item
+
+module ScoutApm
+  class SlowItemSet
+    include Enumerable
+
+    DEFAULT_TOTAL = 10
+    DEFAULT_FAIR = 1
+
+    attr_reader :total
+    attr_reader :fair
+
+    def initialize(total=DEFAULT_TOTAL, fair=DEFAULT_FAIR)
+      @total = total
+      @fair = fair
+      @items = []
+    end
+
+    def each
+      @items.each { |s| yield s }
+    end
+
+    def <<(item)
+      return if attempt_append(item)
+      attempt_to_evict
+      attempt_append(item)
+    end
+
+    def empty_slot?
+      @items.length < total
+    end
+
+    def attempt_append(item)
+      if empty_slot?
+        @items.push(item)
+        true
+      else
+        false
+      end
+    end
+
+    def attempt_to_evict
+      return if @items.length == 0
+
+      overrepresented = @items.
+        group_by { |item| unique_name_for(item) }.
+        to_a.
+        sort_by { |(_, items)| items.length }.
+        last
+
+      if overrepresented[1].length > fair
+        fastest = overrepresented[1].sort_by { |item| item.total_call_time }.first
+        @items.delete(fastest)
+      end
+    end
+
+    # Determine this items' "hash key"
+    def unique_name_for(item)
+      item.metric_name
+    end
+  end
+end

data/lib/scout_apm/slow_job_policy.rb

@@ -0,0 +1,29 @@
+# Create one of these at startup time, and ask it if a certain worker's
+# processing time is slow enough for us to collect a slow trace.
+#
+# Keeps track of a histogram of times for each worker class (spearately), and
+# uses a percentile of normal to mark individual runs as "slow".
+#
+# This assumes that all worker calls will be requested once to `slow?`, so that
+# the data can be stored
+module ScoutApm
+  class SlowJobPolicy
+    DEFAULT_HISTOGRAM_SIZE = 50
+
+    QUANTILE = 95
+
+    def initialize(histogram_size = DEFAULT_HISTOGRAM_SIZE)
+      @histograms = Hash.new { |h, k| h[k] = NumericHistogram.new(histogram_size) }
+    end
+
+    # worker: just the worker class name. "PasswordResetJob" or similar
+    # total_time: runtime of the job in seconds
+    # returns true if this request should be stored in higher trace detail, false otherwise
+    def slow?(worker, total_time)
+      @histograms[worker].add(total_time)
+      return false if @histograms[worker].total == 1 # First call is never slow
+
+      total_time >= @histograms[worker].quantile(QUANTILE)
+    end
+  end
+end

data/lib/scout_apm/slow_job_record.rb

@@ -0,0 +1,33 @@
+module ScoutApm
+  class SlowJobRecord
+    attr_reader :queue_name
+    attr_reader :job_name
+
+    # When did this job occur
+    attr_reader :time
+
+    # What else interesting did we learn?
+    attr_reader :context
+
+    attr_reader :total_time
+    attr_reader :exclusive_time
+    alias_method :total_call_time, :total_time
+
+    attr_reader :metrics
+
+    def initialize(queue_name, job_name, time, total_time, exclusive_time, context, metrics)
+      @queue_name = queue_name
+      @job_name = job_name
+      @time = time
+      @total_time = total_time
+      @exclusive_time = exclusive_time
+      @context = context
+      @metrics = metrics
+    end
+
+    def metric_name
+      "Job/#{queue_name}/#{job_name}"
+    end
+
+  end
+end

data/lib/scout_apm/slow_transaction.rb

@@ -2,10 +2,6 @@ module ScoutApm
   class SlowTransaction
     include ScoutApm::BucketNameSplitter
 
-    BACKTRACE_THRESHOLD = 0.5 # the minimum threshold in seconds to record the backtrace for a metric.
-    BACKTRACE_LIMIT = 5 # Max length of callers to display
-    MAX_SIZE = 100 # Limits the size of the metric hash to prevent a metric explosion.
-
     attr_reader :metric_name
     attr_reader :total_call_time
     attr_reader :metrics
@@ -16,24 +12,6 @@ module ScoutApm
     attr_reader :prof
     attr_reader :raw_prof
 
-    # TODO: Move this out of SlowTransaction, it doesn't have much to do w/
-    # slow trans other than being a piece of data that ends up in it.
-    #
-    # Given a call stack, generates a filtered backtrace that:
-    # * Limits to the app/models, app/controllers, or app/views directories
-    # * Limits to 5 total callers
-    # * Makes the app folder the top-level folder used in trace info
-    def self.backtrace_parser(backtrace)
-      stack = []
-      backtrace.each do |c|
-        if m=c.match(/(\/app\/(controllers|models|views)\/.+)/)
-          stack << m[1]
-          break if stack.size == BACKTRACE_LIMIT
-        end
-      end
-      stack
-    end
-
     def initialize(uri, metric_name, total_call_time, metrics, context, time, raw_stackprof)
       @uri = uri
       @metric_name = metric_name

data/lib/scout_apm/stackprof_tree_collapser.rb

@@ -7,14 +7,13 @@ module ScoutApm
       @raw_stackprof = raw_stackprof
 
       # Log the raw stackprof info
-      unless StackProf.respond_to?(:fake?) && StackProf.fake?
-        begin
-          ScoutApm::Agent.instance.logger.debug("StackProf - Samples: #{raw_stackprof[:samples]}, GC: #{raw_stackprof[:gc_samples]}, missed: #{raw_stackprof[:missed_samples]}, Interval: #{raw_stackprof[:interval]}")
-        rescue
-          # Not a useful log message, currently stackprof is disabled
-          # ScoutApm::Agent.instance.logger.debug("StackProf Raw - #{raw_stackprof.inspect}")
-        end
-      end
+      #unless StackProf.respond_to?(:fake?) && StackProf.fake?
+      #  begin
+      #    ScoutApm::Agent.instance.logger.debug("StackProf - Samples: #{raw_stackprof[:samples]}, GC: #{raw_stackprof[:gc_samples]}, missed: #{raw_stackprof[:missed_samples]}, Interval: #{raw_stackprof[:interval]}")
+      #  rescue
+      #    ScoutApm::Agent.instance.logger.debug("StackProf Raw - #{raw_stackprof.inspect}")
+      #  end
+      #end
     end
 
     def call

data/lib/scout_apm/store.rb

@@ -41,6 +41,20 @@ module ScoutApm
       }
     end
 
+    def track_job!(job)
+      return if job.nil?
+      @mutex.synchronize {
+        current_period.merge_jobs!(Array(job))
+      }
+    end
+
+    def track_slow_job!(job)
+      return if job.nil?
+      @mutex.synchronize {
+        current_period.merge_slow_jobs!(Array(job))
+      }
+    end
+
     # Take each completed reporting_period, and write it to the layaway passed
     #
     # force - a boolean argument that forces this function to write
@@ -93,45 +107,79 @@ module ScoutApm
 
   # One period of Storage. Typically 1 minute
   class StoreReportingPeriod
-    # A SlowTransactionSet object. 
+    # A SlowItemSet to store slow transactions in
     attr_reader :slow_transactions
 
+    # A SlowItemSet to store slow jobs in
+    attr_reader :slow_jobs
+
     # A StoreReportingPeriodTimestamp representing the time that this
     # collection of metrics is for
     attr_reader :timestamp
 
+    attr_reader :metric_set
+
     def initialize(timestamp)
       @timestamp = timestamp
 
-      @slow_transactions = SlowTransactionSet.new
-      @aggregate_metrics = Hash.new
+      @slow_transactions = SlowItemSet.new
+      @slow_jobs = SlowItemSet.new
+
+      @metric_set = MetricSet.new
+      @jobs = Hash.new
     end
 
     #################################
     # Add metrics as they are recorded
     #################################
     def merge_metrics!(metrics)
-      metrics.each { |metric| absorb(metric) }
+      metric_set.absorb_all(metrics)
       self
     end
 
     def merge_slow_transactions!(new_transactions)
       Array(new_transactions).each do |one_transaction|
-        @slow_transactions << one_transaction
+        slow_transactions << one_transaction
+      end
+
+      self
+    end
+
+    def merge_jobs!(jobs)
+      jobs.each do |job|
+        if @jobs.has_key?(job)
+          @jobs[job].combine!(job)
+        else
+          @jobs[job] = job
+        end
       end
 
       self
     end
 
+    def merge_slow_jobs!(new_jobs)
+      Array(new_jobs).each do |job|
+        slow_jobs << job
+      end
+    end
+
     #################################
     # Retrieve Metrics for reporting
     #################################
     def metrics_payload
-      @aggregate_metrics
+      metric_set.metrics
     end
 
     def slow_transactions_payload
-      @slow_transactions.to_a
+      slow_transactions.to_a
+    end
+
+    def jobs
+      @jobs.values
+    end
+
+    def slow_jobs_payload
+      slow_jobs.to_a
     end
 
     #################################
@@ -143,34 +191,6 @@ module ScoutApm
         select { |meta,stats| meta.metric_name =~ /\AController/ }.
         inject(0) {|sum, (_, stat)| sum + stat.call_count }
     end
-
-    private
-
-    # We can't aggregate CPU, Memory, Capacity, or Controller, so pass through these metrics directly
-    # TODO: Figure out a way to not have this duplicate what's in Samplers, and also on server's ingest
-    PASSTHROUGH_METRICS = ["CPU", "Memory", "Instance", "Controller", "SlowTransaction"]
-
-    # Absorbs a single new metric into the aggregates
-    def absorb(metric)
-      meta, stat = metric
-
-      if PASSTHROUGH_METRICS.include?(meta.type) # Leave as-is, don't attempt to combine
-        @aggregate_metrics[meta] ||= MetricStats.new
-        @aggregate_metrics[meta].combine!(stat)
-
-      elsif meta.type == "Errors" # Sadly special cased, we want both raw and aggregate values
-        @aggregate_metrics[meta] ||= MetricStats.new
-        @aggregate_metrics[meta].combine!(stat)
-        agg_meta = MetricMeta.new("Errors/Request", :scope => meta.scope)
-        @aggregate_metrics[agg_meta] ||= MetricStats.new
-        @aggregate_metrics[agg_meta].combine!(stat)
-
-      else # Combine down to a single /all key
-        agg_meta = MetricMeta.new("#{meta.type}/all", :scope => meta.scope)
-        @aggregate_metrics[agg_meta] ||= MetricStats.new
-        @aggregate_metrics[agg_meta].combine!(stat)
-      end
-    end
   end
 end
 

data/lib/scout_apm/tracked_request.rb

@@ -19,6 +19,7 @@ module ScoutApm
     # As we go through a request, instrumentation can mark more general data into the Request
     # Known Keys:
     #   :uri - the full URI requested by the user
+    #   :queue_latency - how long a background Job spent in the queue before starting processing
     attr_reader :annotations
 
     # Nil until the request is finalized, at which point it will hold the
@@ -29,8 +30,20 @@ module ScoutApm
     # Can be nil if we never reach a Rails Controller
     attr_reader :headers
 
+    # What kind of request is this? A trace of a web request, or a background job?
+    # Use job! and web! to set, and job? and web? to query
+    attr_reader :request_type
+
+    # This maintains a lookup hash of Layer names and call counts. It's used to trigger fetching a backtrace on n+1 calls.
+    # Note that layer names might not be Strings - can alse be Utils::ActiveRecordMetricName. Also, this would fail for layers
+    # with same names across multiple types.
+    attr_accessor :call_counts
+
+    BACKTRACE_THRESHOLD = 0.5 # the minimum threshold in seconds to record the backtrace for a metric.
+
     def initialize
       @layers = []
+      @call_counts = Hash.new { |h, k| h[k] = CallSet.new }
       @annotations = {}
       @ignoring_children = false
       @context = Context.new
@@ -41,12 +54,11 @@ module ScoutApm
 
     def start_layer(layer)
       if ignoring_children?
-        ScoutApm::Agent.instance.logger.info("Skipping layer because we're ignoring children: #{layer.inspect}")
         return
       end
 
       start_request(layer) unless @root_layer
-
+      update_call_counts!(layer)
       @layers[-1].add_child(layer) if @layers.any?
       @layers.push(layer)
     end
@@ -57,9 +69,8 @@ module ScoutApm
       layer = @layers.pop
       layer.record_stop_time!
 
-      # Do this here, rather than in the layer because we need this caller. Maybe able to move it?
-      if layer.total_exclusive_time > ScoutApm::SlowTransaction::BACKTRACE_THRESHOLD
-        layer.store_backtrace(caller)
+      if capture_backtrace?(layer)
+        layer.capture_backtrace!
       end
 
       if finalized?
@@ -67,6 +78,32 @@ module ScoutApm
       end
     end
 
+    BACKTRACE_BLACKLIST = ["Controller", "Job"]
+    def capture_backtrace?(layer)
+      # Never capture backtraces for this kind of layer. The backtrace will
+      # always be 100% framework code.
+      return false if BACKTRACE_BLACKLIST.include?(layer.type)
+
+      # Only capture backtraces if we're in a real "request". Otherwise we
+      # can spend lot of time capturing backtraces from the internals of
+      # Sidekiq, only to throw them away immediately.
+      return false unless (web? || job?)
+
+      # Capture any individually slow layer.
+      return true if layer.total_exclusive_time > BACKTRACE_THRESHOLD
+
+      # Capture any layer that we've seen many times. Captures n+1 problems
+      return true if @call_counts[layer.name].capture_backtrace?
+
+      # Don't capture otherwise
+      false
+    end
+
+    # Maintains a lookup Hash of call counts by layer name. Used to determine if we should capture a backtrace.
+    def update_call_counts!(layer)
+      @call_counts[layer.name].update!(layer.desc)
+    end
+
     ###################################
     # Request Lifecycle
     ###################################
@@ -124,6 +161,22 @@ module ScoutApm
       @headers = headers
     end
 
+    def job!
+      @request_type = "job"
+    end
+
+    def job?
+      request_type == "job"
+    end
+
+    def web!
+      @request_type = "web"
+    end
+
+    def web?
+      request_type == "web"
+    end
+
     ###################################
     # Persist the Request
     ###################################
@@ -133,20 +186,24 @@ module ScoutApm
     def record!
       @recorded = true
 
-      metrics = LayerMetricConverter.new(self).call
+      metrics = LayerConverters::MetricConverter.new(self).call
       ScoutApm::Agent.instance.store.track!(metrics)
 
-      slow, slow_metrics = LayerSlowTransactionConverter.new(self).call
+      slow, slow_metrics = LayerConverters::SlowRequestConverter.new(self).call
       ScoutApm::Agent.instance.store.track_slow_transaction!(slow)
       ScoutApm::Agent.instance.store.track!(slow_metrics)
 
-      error_metrics = LayerErrorConverter.new(self).call
+      error_metrics = LayerConverters::ErrorConverter.new(self).call
       ScoutApm::Agent.instance.store.track!(error_metrics)
 
-      queue_time_metrics = RequestQueueTime.new(self).call
+      queue_time_metrics = LayerConverters::RequestQueueTimeConverter.new(self).call
       ScoutApm::Agent.instance.store.track!(queue_time_metrics)
 
-      # ScoutApm::Agent.instance.logger.debug("Finished recording request") if metrics.any?
+      job = LayerConverters::JobConverter.new(self).call
+      ScoutApm::Agent.instance.store.track_job!(job)
+
+      slow_job = LayerConverters::SlowJobConverter.new(self).call
+      ScoutApm::Agent.instance.store.track_slow_job!(slow_job)
     end
 
     # Have we already persisted this request?

data/lib/scout_apm/utils/active_record_metric_name.rb

@@ -27,6 +27,19 @@ module ScoutApm
         end
       end
 
+      # For the layer lookup.
+      def hash
+        h = name.downcase.hash
+        h
+      end
+
+      # For the layer lookup.
+      # Reminder: #eql? is for Hash equality: returns true if obj and other refer to the same hash key.
+      def eql?(o)
+        self.class    == o.class &&
+        name.downcase == o.name.downcase
+      end
+
       private
 
       def model

data/lib/scout_apm/utils/backtrace_parser.rb

@@ -0,0 +1,31 @@
+require 'scout_apm/environment'
+
+# Removes actual values from SQL. Used to both obfuscate the SQL and group
+# similar queries in the UI.
+module ScoutApm
+  module Utils
+    class BacktraceParser
+
+      def initialize(call_stack)
+        @call_stack = call_stack
+        # We can't use a constant as it'd be too early to fetch environment info
+        @@app_dir_regex ||= /\A(#{ScoutApm::Environment.instance.root.to_s.gsub('/','\/')}\/)(app\/(.+))/.freeze
+      end
+
+      # Given a call stack Array, grabs the first call within the application root directory.
+      def call
+        # We used to return an array of up to 5 elements...this will return a single element-array for backwards compatibility.
+        # Only the first element is used in Github code display.
+        stack = []
+        @call_stack.each_with_index do |c,i|
+          if m = c.match(@@app_dir_regex)
+            stack << m[2]
+            break
+          end
+        end
+        stack
+      end
+
+    end
+  end
+end

data/lib/scout_apm/utils/fake_stack_prof.rb

@@ -1,5 +1,5 @@
 # A fake implementation of stackprof, for systems that don't support it.
-module StackProf
+class StackProf
   def self.start(*args)
     @running = true
   end

data/lib/scout_apm/utils/sql_sanitizer.rb

@@ -17,6 +17,7 @@ module ScoutApm
       def initialize(sql)
         @raw_sql = sql
         @database_engine = ScoutApm::Environment.instance.database_engine
+        @sanitized = false # only sanitize once.
       end
 
       def sql
@@ -24,6 +25,11 @@ module ScoutApm
       end
 
       def to_s
+        if @sanitized
+          sql
+        else
+          @sanitized = true
+        end
         case database_engine
         when :postgres then to_s_postgres
         when :mysql    then to_s_mysql

data/lib/scout_apm/version.rb

@@ -1,4 +1,4 @@
 module ScoutApm
-  VERSION = "1.4.6"
+  VERSION = "1.5.0.pre"
 end
 

data/lib/scout_apm.rb

@@ -32,8 +32,16 @@ require 'scout_apm/version'
 require 'scout_apm/tracked_request'
 require 'scout_apm/layer'
 require 'scout_apm/request_manager'
-require 'scout_apm/layer_converter'
-require 'scout_apm/request_queue_time'
+require 'scout_apm/call_set'
+
+require 'scout_apm/layer_converters/converter_base'
+require 'scout_apm/layer_converters/depth_first_walker'
+require 'scout_apm/layer_converters/error_converter'
+require 'scout_apm/layer_converters/job_converter'
+require 'scout_apm/layer_converters/slow_job_converter'
+require 'scout_apm/layer_converters/metric_converter'
+require 'scout_apm/layer_converters/slow_request_converter'
+require 'scout_apm/layer_converters/request_queue_time_converter'
 
 require 'scout_apm/server_integrations/passenger'
 require 'scout_apm/server_integrations/puma'
@@ -55,6 +63,8 @@ require 'scout_apm/platform_integrations/heroku'
 require 'scout_apm/platform_integrations/cloud_foundry'
 require 'scout_apm/platform_integrations/server'
 
+require 'scout_apm/histogram'
+
 require 'scout_apm/deploy_integrations/capistrano_3'
 #require 'scout_apm/deploy_integrations/capistrano_2'
 
@@ -79,6 +89,7 @@ require 'scout_apm/instruments/process/process_memory'
 require 'scout_apm/app_server_load'
 
 require 'scout_apm/utils/sql_sanitizer'
+require 'scout_apm/utils/backtrace_parser'
 require 'scout_apm/utils/active_record_metric_name'
 require 'scout_apm/utils/null_logger'
 require 'scout_apm/utils/installed_gems'
@@ -95,21 +106,30 @@ require 'scout_apm/layaway_file'
 require 'scout_apm/reporter'
 require 'scout_apm/background_worker'
 require 'scout_apm/bucket_name_splitter'
-require 'scout_apm/metric_meta'
-require 'scout_apm/metric_stats'
 require 'scout_apm/stack_item'
+require 'scout_apm/metric_set'
 require 'scout_apm/store'
 require 'scout_apm/tracer'
 require 'scout_apm/context'
 require 'scout_apm/stackprof_tree_collapser'
+
+require 'scout_apm/metric_meta'
+require 'scout_apm/metric_stats'
 require 'scout_apm/slow_transaction'
+require 'scout_apm/slow_job_record'
+require 'scout_apm/slow_item_set'
 require 'scout_apm/slow_request_policy'
-require 'scout_apm/slow_transaction_set'
+require 'scout_apm/slow_job_policy'
+require 'scout_apm/job_record'
+
 require 'scout_apm/capacity'
 require 'scout_apm/attribute_arranger'
 
 require 'scout_apm/serializers/payload_serializer'
 require 'scout_apm/serializers/payload_serializer_to_json'
+require 'scout_apm/serializers/jobs_serializer_to_json'
+require 'scout_apm/serializers/slow_jobs_serializer_to_json'
+require 'scout_apm/serializers/metrics_to_json_serializer'
 require 'scout_apm/serializers/directive_serializer'
 require 'scout_apm/serializers/app_server_load_serializer'
 require 'scout_apm/serializers/deploy_serializer'