scout_apm 1.4.6 → 1.5.0.pre

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d09339b4d51454f56825e72d7ae8aa53a8aca5a4
-  data.tar.gz: d50de440976b8f060e5365838941022807c31f0d
+  metadata.gz: 9392c2916f2f51cd4d8f6deb672069f15f3d8e68
+  data.tar.gz: dd764e893e93e818a22c487a0c986701153160eb
 SHA512:
-  metadata.gz: 3884aee4911d383138d928c2afd4aa4c799232c53bcb6e6d91e067963d6992cc0bb3d97851da4badd0e32cda277a12ab1f4f871830c8011c8eaa25e327272202
-  data.tar.gz: 7b9711b230f8804a334339760955f705b879f3dfbaf5abdc485dd25e5ab720096ec87da11e162cca023e13956b472d1d0eaf661d333bc26c2e85475e6dd39964
+  metadata.gz: 8421818c942298451ee690625cb35670088740f2862ecea8d2fceb3b4cc1cb7f15439497e3b2fede3bb52cab4756e0e48f64d30c8fdb71bbafe7b33b577ed8a6
+  data.tar.gz: 3486c303a0509ea19042ffba56c9045329a8e91ea25181a7b0fc4817b79468853ef7d1b5608813923e488f373cf494546d0c2b842cf7b318b6693118173e0f94

data/CHANGELOG.markdown

@@ -1,7 +1,16 @@
+# 1.5.0
+
+* Background Job instrumentation for Sidekiq and Sidekiq-backed ActiveJob
+* Collecting backtraces on n+1 calls
+
 # 1.4.6
 
 * Defend against a nil
 
+# 1.5.0
+
+* Background Job instrumentation for Sidekiq and Sidekiq-backed ActiveJob
+
 # 1.4.5
 
 * Instrument Elasticsearch

data/lib/scout_apm/agent/reporting.rb

@@ -38,6 +38,9 @@ module ScoutApm
       def deliver_period(reporting_period)
         metrics = reporting_period.metrics_payload
         slow_transactions = reporting_period.slow_transactions_payload
+        jobs = reporting_period.jobs
+        slow_jobs = reporting_period.slow_jobs_payload
+
         metadata = {
           :app_root      => ScoutApm::Environment.instance.root.to_s,
           :unique_id     => ScoutApm::Utils::UniqueId.simple,
@@ -49,13 +52,12 @@ module ScoutApm
 
         log_deliver(metrics, slow_transactions, metadata)
 
-        payload = ScoutApm::Serializers::PayloadSerializer.serialize(metadata, metrics, slow_transactions)
-        response = reporter.report(payload, headers)
-        unless response && response.is_a?(Net::HTTPSuccess)
-          logger.warn "Error on checkin to #{reporter.uri.to_s}: #{response.inspect}"
-        end
+        payload = ScoutApm::Serializers::PayloadSerializer.serialize(metadata, metrics, slow_transactions, jobs, slow_jobs)
+        logger.debug("Payload: #{payload}")
+
+        reporter.report(payload, headers)
       rescue => e
-        logger.warn "Error on checkin to #{reporter.uri.to_s}"
+        logger.warn "Error on checkin"
         logger.info e.message
         logger.debug e.backtrace
       end

data/lib/scout_apm/agent.rb

@@ -18,6 +18,7 @@ module ScoutApm
     attr_accessor :options # options passed to the agent when +#start+ is called.
     attr_accessor :metric_lookup # Hash used to lookup metric ids based on their name and scope
     attr_reader :slow_request_policy
+    attr_reader :slow_job_policy
 
     # All access to the agent is thru this class method to ensure multiple Agent instances are not initialized per-Ruby process.
     def self.instance(options = {})
@@ -31,6 +32,7 @@ module ScoutApm
       @started = false
       @options ||= options
       @config = ScoutApm::Config.new(options[:config_path])
+      @slow_job_policy = ScoutApm::SlowJobPolicy.new
 
       @store          = ScoutApm::Store.new
       @layaway        = ScoutApm::Layaway.new
@@ -78,7 +80,7 @@ module ScoutApm
 
       if defined?(::ScoutRails)
         logger.warn "ScoutAPM is incompatible with the old Scout Rails plugin. Please remove scout_rails from your Gemfile"
-        return false
+        return false unless force?
       end
 
       true
@@ -108,15 +110,17 @@ module ScoutApm
 
       app_server_load_hook
 
+      if environment.background_job_integration
+        environment.background_job_integration.install
+        logger.info "Installed Background Job Integration [#{environment.background_job_name}]"
+      end
+
       # start_background_worker? is true on non-forking servers, and directly
       # starts the background worker.  On forking servers, a server-specific
       # hook is inserted to start the background worker after forking.
       if start_background_worker?
         start_background_worker
         logger.info "Scout Agent [#{ScoutApm::VERSION}] Initialized"
-      elsif environment.background_job_integration
-        environment.background_job_integration.install
-        logger.info "Scout Agent [#{ScoutApm::VERSION}] loaded in [#{environment.background_job_name}] master process. Monitoring will start after background job framework forks its workers."
       else
         environment.app_server_integration.install
         logger.info "Scout Agent [#{ScoutApm::VERSION}] loaded in [#{environment.app_server}] master process. Monitoring will start after server forks its workers."
@@ -244,7 +248,7 @@ module ScoutApm
           install_instrument(ScoutApm::Instruments::ActionControllerRails3Rails4)
           install_instrument(ScoutApm::Instruments::MiddlewareSummary)
           install_instrument(ScoutApm::Instruments::RailsRouter)
-        when :sinatra     then install_instrument(ScoutApm::Instruments::Sinatra)
+        # when :sinatra     then install_instrument(ScoutApm::Instruments::Sinatra)
         end
       end
 
@@ -272,7 +276,7 @@ module ScoutApm
 
       # Allow users to skip individual instruments via the config file
       instrument_short_name = instrument_klass.name.split("::").last
-      if (config.value("disabled_instruments") || []).include?(instrument_short_name)
+      if config.value("disabled_instruments").include?(instrument_short_name)
         logger.info "Skipping Disabled Instrument: #{instrument_short_name} - To re-enable, change `disabled_instruments` key in scout_apm.yml"
         return
       end

data/lib/scout_apm/background_job_integrations/sidekiq.rb

@@ -12,7 +12,7 @@ module ScoutApm
       end
 
       def forking?
-        true
+        false
       end
 
       def install
@@ -20,30 +20,41 @@ module ScoutApm
         SidekiqMiddleware.class_eval do
           include ScoutApm::Tracer
         end
+
         ::Sidekiq.configure_server do |config|
           config.server_middleware do |chain|
             chain.add SidekiqMiddleware
           end
         end
+
         require 'sidekiq/processor' # sidekiq v4 has not loaded this file by this point
+
         ::Sidekiq::Processor.class_eval do
-          old = instance_method(:initialize)
-          define_method(:initialize) do |boss|
-            ScoutApm::Agent.instance.start_background_worker
-            old.bind(self).call(boss)
+          def initialize_with_scout(boss)
+            ::ScoutApm::Agent.instance.start_background_worker unless ::ScoutApm::Agent.instance.background_worker_running?
+            initialize_without_scout(boss)
           end
+
+          alias_method :initialize_without_scout, :initialize
+          alias_method :initialize, :initialize_with_scout
         end
       end
     end
 
     class SidekiqMiddleware
       def call(worker, msg, queue)
-        msg_args = msg["args"].first
-        job_class = msg_args["job_class"]
-        latency = (Time.now.to_f - (msg['enqueued_at'] || msg['created_at'])) * 1000
+        job_class = msg["class"] # TODO: Validate this across different versions of Sidekiq
+        if job_class == "ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper" && msg.has_key?("wrapped")
+          job_class = msg["wrapped"]
+        end
+
+        latency = (Time.now.to_f - (msg['enqueued_at'] || msg['created_at']))
 
-        ScoutApm::Agent.instance.store.track_one!("Queue", queue, 0, {:extra_metrics => {:latency => latency}})
         req = ScoutApm::RequestManager.lookup
+        req.job!
+        req.annotate_request(:queue_latency => latency)
+
+        req.start_layer( ScoutApm::Layer.new("Queue", queue) )
         req.start_layer( ScoutApm::Layer.new("Job", job_class) )
 
         begin
@@ -51,9 +62,10 @@ module ScoutApm
         rescue
           req.error!
           raise
-        ensure
-          req.stop_layer
         end
+      ensure
+        req.stop_layer # Job
+        req.stop_layer # Queue
       end
     end
   end

data/lib/scout_apm/call_set.rb

@@ -0,0 +1,61 @@
+module ScoutApm
+  class CallSet
+
+    N_PLUS_ONE_MAGIC_NUMBER = 5 # Fetch backtraces on this number of calls to a layer. The caller data is only collected on this call to limit overhead.
+    N_PLUS_ONE_TIME_THRESHOLD = 150/1000.0 # Minimum time in seconds before we start performing any work. This is to prevent doing a lot of work on already fast calls.
+
+    attr_reader :call_count
+
+    def initialize
+      @items = [] # An array of Layer descriptions that are associated w/a single Layer name (ex: User/find). Note this may contain nil items.
+      @grouped_items = Hash.new { |h, k| h[k] = [] } # items groups by their normalized name since multiple layers could have the same layer name.
+      @call_count = 0
+      @captured = false # cached for performance
+      @start_time = Time.now
+      @past_start_time = false # cached for performance
+    end
+
+    def update!(item = nil)
+      if @captured # No need to do any work if we've already captured a backtrace.
+        return
+      end
+      @call_count += 1
+      @items << item
+      if @grouped_items.any? # lazy grouping as normalizing items can be expensive.
+        @grouped_items[unique_name_for(item)] << item
+      end
+    end
+
+    # Limit our workload if time across this set of calls is small.
+    def past_time_threshold?
+      return true if @past_time_threshold # no need to check again once past
+      @past_time_threshold = (Time.now-@start_time) >= N_PLUS_ONE_TIME_THRESHOLD
+    end
+
+    # We're selective on capturing a backtrace for two reasons:
+    # * Grouping ActiveRecord calls requires us to sanitize the SQL. This isn't cheap.
+    # * Capturing backtraces isn't cheap.
+    def capture_backtrace?
+      if !@captured && @call_count >= N_PLUS_ONE_MAGIC_NUMBER && past_time_threshold? && at_magic_number?
+        @captured = true
+      end
+    end
+
+    def at_magic_number?
+      grouped_items[unique_name_for(@items.last)].size >= N_PLUS_ONE_MAGIC_NUMBER
+    end
+
+    def grouped_items
+      if @grouped_items.any? 
+        @grouped_items
+      else
+        @grouped_items.merge!(@items.group_by { |item| unique_name_for(item) })
+      end
+    end
+
+    # Determine this items' "hash key"
+    def unique_name_for(item)
+      item.to_s
+    end
+  end
+end

data/lib/scout_apm/config.rb

@@ -27,7 +27,8 @@ module ScoutApm
         'stackprof_interval' => 20000, # microseconds, 1000 = 1 millisecond, so 20k == 20 milliseconds
         'uri_reporting' => 'full_path',
         'report_format' => 'json',
-        'disabled_instruments' => []
+        'disabled_instruments' => [],
+        'enable_background_jobs' => true,
     }.freeze
 
     def initialize(config_path = nil)

data/lib/scout_apm/environment.rb

@@ -25,7 +25,7 @@ module ScoutApm
 
     BACKGROUND_JOB_INTEGRATIONS = [
       ScoutApm::BackgroundJobIntegrations::Sidekiq.new,
-      ScoutApm::BackgroundJobIntegrations::DelayedJob.new
+      # ScoutApm::BackgroundJobIntegrations::DelayedJob.new
     ]
 
     FRAMEWORK_INTEGRATIONS = [
@@ -130,9 +130,11 @@ module ScoutApm
     end
 
     def background_job_integration
-      @background_job_integration ||= BACKGROUND_JOB_INTEGRATIONS.detect {|integration| integration.present?}
-      #### Temporary Disable
-      nil
+      if Agent.instance.config.value("enable_background_jobs", !Agent.instance.config.config_file_exists?)
+        @background_job_integration ||= BACKGROUND_JOB_INTEGRATIONS.detect {|integration| integration.present?}
+      else
+        nil
+      end
     end
 
     def background_job_name
@@ -158,11 +160,15 @@ module ScoutApm
     end
 
     def ruby_19?
-      defined?(RUBY_ENGINE) && RUBY_ENGINE == "ruby" && RUBY_VERSION.match(/^1\.9/)
+      @ruby_19 ||= defined?(RUBY_ENGINE) && RUBY_ENGINE == "ruby" && RUBY_VERSION.match(/^1\.9/)
     end
 
     def ruby_187?
-      defined?(RUBY_VERSION) && RUBY_VERSION.match(/^1\.8\.7/)
+      @ruby_187 ||= defined?(RUBY_VERSION) && RUBY_VERSION.match(/^1\.8\.7/)
+    end
+
+    def ruby_2?
+      @ruby_2 ||= defined?(RUBY_VERSION) && RUBY_VERSION.match(/^2/)
     end
 
     ### framework checks
@@ -170,6 +176,5 @@ module ScoutApm
     def sinatra?
       framework_integration.name == :sinatra
     end
-
   end # class Environemnt
 end

data/lib/scout_apm/histogram.rb

@@ -0,0 +1,124 @@
+module ScoutApm
+  HistogramBin = Struct.new(:value, :count)
+
+  class NumericHistogram
+    attr_reader :max_bins
+    attr_reader :bins
+    attr_accessor :total
+
+    def initialize(max_bins)
+      @max_bins = max_bins
+      @bins = []
+      @total = 0
+    end
+
+    def add(new_value)
+      @total += 1
+      create_new_bin(new_value.to_f)
+      trim
+    end
+
+    def quantile(q)
+      return 0 if total == 0
+
+      if q > 1
+        q = q / 100.0
+      end
+
+      count = q.to_f * total.to_f
+
+      bins.each_with_index do |bin, index|
+        count -= bin.count
+
+        if count <= 0
+          return bin.value
+        end
+      end
+
+      # If we fell through, we were asking for the last (max) value
+      return bins[-1].value
+    end
+
+    def mean
+      if total == 0
+        return 0
+      end
+
+      sum = bins.inject(0) { |s, bin| s + (bin.value * bin.count) }
+      return sum.to_f / total.to_f
+    end
+
+    def combine!(other)
+      @bins = (other.bins + @bins).sort_by {|b| b.value }
+      @total += other.total
+      trim
+      self
+    end
+
+    def as_json
+      bins.map{|b| [b.value, b.count]}
+    end
+
+    private
+
+    # If we exactly match an existing bin, add to it, otherwise create a new bin holding a count for the new value.
+    def create_new_bin(new_value)
+      bins.each_with_index do |bin, index|
+        # If it matches exactly, increment the bin's count
+        if bin.value == new_value
+          bin.count += 1
+          return
+        end
+
+        # We've gone one bin too far, so insert before the current bin.
+        if bin.value > new_value
+          # Insert at this index
+          new_bin = HistogramBin.new(new_value, 1)
+          bins.insert(index, new_bin)
+          return
+        end
+      end
+
+      # If we get to here, the bin needs to be added to the end.
+      bins << HistogramBin.new(new_value, 1)
+    end
+
+    def trim
+      while bins.length > max_bins
+        trim_one
+      end
+    end
+
+    def trim_one
+      minDelta = Float::MAX
+      minDeltaIndex = 0
+
+      # Which two bins should we merge?
+      bins.each_with_index do |_, index|
+        next if index == 0
+
+        delta = bins[index].value - bins[index - 1].value
+        if delta < minDelta
+          minDelta = delta
+          minDeltaIndex = index
+        end
+      end
+
+      # Create the merged bin with summed count, and weighted value
+      mergedCount = bins[minDeltaIndex - 1].count + bins[minDeltaIndex].count
+      mergedValue = (
+        bins[minDeltaIndex - 1].value * bins[minDeltaIndex - 1].count +
+        bins[minDeltaIndex].value     * bins[minDeltaIndex].count
+        ) / mergedCount
+
+      mergedBin = HistogramBin.new(mergedValue, mergedCount)
+
+      # Remove the two bins we just merged together, then add the merged one
+      bins.slice!(minDeltaIndex - 1, 2)
+      bins.insert(minDeltaIndex - 1, mergedBin)
+    rescue => e
+      ScoutApm::Agent.instance.logger.info("Error in NumericHistogram#trim_one. #{e.message}, #{e.backtrace}, #{self.inspect}")
+      raise
+    end
+  end
+end

data/lib/scout_apm/instruments/action_controller_rails_2.rb

@@ -63,6 +63,7 @@ module ScoutApm
         req.context.add_user(:ip => request.remote_ip)
         req.set_headers(request.headers)
         req.start_layer( ScoutApm::Layer.new("Controller", "#{controller_path}/#{action_name}") )
+        req.web!
 
         begin
           perform_action_without_scout_instruments(*args, &block)

data/lib/scout_apm/instruments/action_controller_rails_3_rails4.rb

@@ -63,6 +63,7 @@ module ScoutApm
         req.annotate_request(:uri => path)
         req.context.add_user(:ip => request.remote_ip)
         req.set_headers(request.headers)
+        req.web!
 
         req.start_layer( ScoutApm::Layer.new("Controller", "#{controller_path}/#{action_name}") )
         begin

data/lib/scout_apm/instruments/delayed_job.rb

@@ -33,6 +33,7 @@ module ScoutApm
 
         ScoutApm::Agent.instance.store.track_one!("Queue", queue, 0, {:extra_metrics => {:latency => latency}})
         req = ScoutApm::RequestManager.lookup
+        req.job!
         req.start_layer( ScoutApm::Layer.new("Job", scout_method_name) )
 
         begin

data/lib/scout_apm/instruments/process/process_memory.rb

@@ -26,7 +26,7 @@ module ScoutApm
             get_mem_from_procfile
           when /darwin9/ # 10.5
             get_mem_from_shell("ps -o rsz")
-          when /darwin1[0123]/ # 10.6 - 10.10
+          when /darwin1[01234]/ # 10.6 - 10.11
             get_mem_from_shell("ps -o rss")
           else
             0 # What default? was nil.

data/lib/scout_apm/instruments/sinatra.rb

@@ -34,7 +34,7 @@ module ScoutApm
         req = ScoutApm::RequestManager.lookup
         req.annotate_request(:uri => @request.path_info)
         req.context.add_user(:ip => @request.ip)
-        req.set_headers(request.headers)
+        # req.set_headers(env) # TODO: Parse headers with name HTTP_*
 
         req.start_layer( ScoutApm::Layer.new("Controller", scout_controller_action) )
         begin

data/lib/scout_apm/job_record.rb

@@ -0,0 +1,76 @@
+# Records details about all runs of a given job.
+#
+# Contains:
+#   Queue Name
+#   Job Name
+#   Job Runtime - histogram
+#   Metrics collected during the run (Database, HTTP, View, etc)
+module ScoutApm
+  class JobRecord
+    attr_reader :queue_name
+    attr_reader :job_name
+    attr_reader :total_time
+    attr_reader :exclusive_time
+    attr_reader :errors
+    attr_reader :metric_set
+
+    def initialize(queue_name, job_name, total_time, exclusive_time, errors, metrics)
+      @queue_name = queue_name
+      @job_name = job_name
+
+      @total_time = NumericHistogram.new(50)
+      @total_time.add(total_time)
+
+      @exclusive_time = NumericHistogram.new(50)
+      @exclusive_time.add(exclusive_time)
+
+      @errors = errors.to_i
+
+      @metric_set = MetricSet.new
+      @metric_set.absorb_all(metrics)
+    end
+
+    # Modifies self and returns self, after merging in `other`.
+    def combine!(other)
+      same_job = queue_name == other.queue_name && job_name == other.job_name
+      raise "Mismatched Merge of Background Job" unless same_job
+
+      @errors += other.errors
+      @metric_set = metric_set.combine!(other.metric_set)
+      @total_time.combine!(other.total_time)
+      @exclusive_time.combine!(other.exclusive_time)
+
+      self
+    end
+
+    def run_count
+      total_time.total
+    end
+
+    def metrics
+      metric_set.metrics
+    end
+
+
+    ######################
+    # Hash Key interface
+    ######################
+
+    def ==(o)
+      self.eql?(o)
+    end
+
+    def hash
+      h = queue_name.downcase.hash
+      h ^= job_name.downcase.hash
+      h
+    end
+
+    def eql?(o)
+     self.class == o.class &&
+       queue_name.downcase == o.queue_name.downcase &&
+       job_name.downcase == o.job_name.downcase
+    end
+  end
+end
+

data/lib/scout_apm/layaway.rb

@@ -18,7 +18,10 @@ module ScoutApm
           new_req = new_val.request_count
           ScoutApm::Agent.instance.logger.debug("Merging Two reporting periods (#{old_val.timestamp.to_s}, #{new_val.timestamp.to_s}): old req #{old_req}, new req #{new_req}")
 
-          old_val.merge_metrics!(new_val.metrics_payload).merge_slow_transactions!(new_val.slow_transactions)
+          old_val.
+            merge_metrics!(new_val.metrics_payload).
+            merge_slow_transactions!(new_val.slow_transactions).
+            merge_jobs!(new_val.jobs)
         }
 
         ScoutApm::Agent.instance.logger.debug("AddReportingPeriod: AfterMerge Timestamps: #{existing_data.keys.map(&:to_s).inspect}")

data/lib/scout_apm/layaway_file.rb

@@ -16,9 +16,9 @@ module ScoutApm
         return nil
       end
       Marshal.load(dump)
-    rescue ArgumentError, TypeError => e
-      ScoutApm::Agent.instance.logger.debug("Error loading data from layaway file: #{e.inspect}")
-      ScoutApm::Agent.instance.logger.debug(e.backtrace.inspect)
+    rescue NameError, ArgumentError, TypeError => e
+      ScoutApm::Agent.instance.logger.info("Unable to load data from Layaway file, resetting.")
+      ScoutApm::Agent.instance.logger.debug("#{e.message}, #{e.backtrace.join("\n\t")}")
       nil
     end
 
@@ -37,7 +37,7 @@ module ScoutApm
         end
       end
     rescue Errno::ENOENT, Exception  => e
-      ScoutApm::Agent.instance.logger.error("Unable to access the layaway file [#{e.message}]. " +
+      ScoutApm::Agent.instance.logger.error("Unable to access the layaway file [#{e.class} - #{e.message}]. " +
                                             "The user running the app must have read & write access. " +
                                             "Change the path by setting the `data_file` key in scout_apm.yml"
                                            )

data/lib/scout_apm/layer.rb

@@ -30,6 +30,7 @@ module ScoutApm
     # backtrace of where it occurred.
     attr_reader :backtrace
 
+    BACKTRACE_CALLER_LIMIT = 30 # maximum number of lines to send thru for backtrace analysis
 
     def initialize(type, name, start_time = Time.now)
       @type = type
@@ -66,10 +67,19 @@ module ScoutApm
       "#{type}/#{name}"
     end
 
-    def store_backtrace(bt)
-      return unless bt.is_a? Array
-      return unless bt.length > 0
-      @backtrace = bt
+    def capture_backtrace!
+      ScoutApm::Agent.instance.logger.debug { "Capturing Backtrace for Layer [#{type}/#{name}]" }
+      @backtrace = caller_array
+    end
+
+    # In Ruby 2.0+, we can pass the range directly to the caller to reduce the memory footprint.
+    def caller_array
+      # omits the first several callers which are in the ScoutAPM stack.
+      if ScoutApm::Environment.instance.ruby_2?
+        caller(3...BACKTRACE_CALLER_LIMIT)
+      else
+        caller[3...BACKTRACE_CALLER_LIMIT]
+      end
     end
 
     ######################################

data/lib/scout_apm/layer_converters/converter_base.rb

@@ -0,0 +1,30 @@
+module ScoutApm
+  module LayerConverters
+    class ConverterBase
+      attr_reader :walker
+      attr_reader :request
+      attr_reader :root_layer
+
+      def initialize(request)
+        @request = request
+        @root_layer = request.root_layer
+        @walker = DepthFirstWalker.new(root_layer)
+      end
+
+      # Scope is determined by the first Controller we hit.  Most of the time
+      # there will only be 1 anyway.  But if you have a controller that calls
+      # another controller method, we may pick that up:
+      #     def update
+      #       show
+      #       render :update
+      #     end
+      def scope_layer
+        @scope_layer ||= walker.walk do |layer|
+          if layer.type == "Controller"
+            break layer
+          end
+        end
+      end
+    end
+  end
+end