scout_apm 2.1.8 → 2.1.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a8c4c269396ae187fe901317425d0c984e2906bb
-  data.tar.gz: 3b2c16163875d6b9dd4fb5b54a3485fa4fc27353
+  metadata.gz: b39d56f8f0d746e68ea6efb1563b03a9f61e42a3
+  data.tar.gz: 07a2274f0a5951b4d5d33321ee736dfea10e6f20
 SHA512:
-  metadata.gz: e8793cf1a126e0e8c7294c633a06648352d301a7214f31aa7b863b9f1d3862c0148585eb711ce671312d17d697e92f834886c630e1974b62370dea03aaaf7aab
-  data.tar.gz: 84db938d6d9972e14901a3c7b13095a70de32ad0b9dc8c2b688a302bf96c04bb176a0086783c01cc99e47b8c12906d2d05d6ea1a84940f3e8b01c177deb9dcb5
+  metadata.gz: bbc3e9460c9e91c4cbbf88fc79612115b399da95f97844dc0719694cab3651a151d31f550f0c2ba6e93c1f23e4868c17a3dc0aa4485c5a1133ec441a4e5b5a3e
+  data.tar.gz: feae03802789c2b9de7d62dd63b191474323e84c9474a13d7e5c49d3409e71bd952e75c1f10b43540488a39b761135a4da1078e5467274e43b3c41be5b4c0af2

data/CHANGELOG.markdown

@@ -1,3 +1,12 @@
+# 2.1.9
+
+* Send raw histograms of response time, enabling more accurate 95th %iles
+* Gzip payloads
+* Fix Mongoid (5.0) + Mongo (2.1) support
+* Initial Delayed Job support
+* Limit max metric size of a trace to 500.
+
+
 # 2.1.8
 
 * Adds Git revision detection, which is reported on app load and associated with transaction traces

data/lib/scout_apm.rb

@@ -94,6 +94,8 @@ require 'scout_apm/utils/null_logger'
 require 'scout_apm/utils/sql_sanitizer'
 require 'scout_apm/utils/time'
 require 'scout_apm/utils/unique_id'
+require 'scout_apm/utils/numbers'
+require 'scout_apm/utils/gzip_helper'
 
 require 'scout_apm/config'
 require 'scout_apm/environment'
@@ -132,6 +134,7 @@ require 'scout_apm/serializers/payload_serializer_to_json'
 require 'scout_apm/serializers/jobs_serializer_to_json'
 require 'scout_apm/serializers/slow_jobs_serializer_to_json'
 require 'scout_apm/serializers/metrics_to_json_serializer'
+require 'scout_apm/serializers/histograms_serializer_to_json'
 require 'scout_apm/serializers/directive_serializer'
 require 'scout_apm/serializers/app_server_load_serializer'
 

data/lib/scout_apm/agent.rb

@@ -130,7 +130,7 @@ module ScoutApm
 
       [ ScoutApm::Instruments::Process::ProcessCpu.new(environment.processors, logger),
         ScoutApm::Instruments::Process::ProcessMemory.new(logger),
-        ScoutApm::Instruments::PercentileSampler.new(logger, 95),
+        ScoutApm::Instruments::PercentileSampler.new(logger, request_histograms_by_time),
       ].each { |s| store.add_sampler(s) }
 
       app_server_load_hook
@@ -231,7 +231,10 @@ module ScoutApm
     end
 
     def background_worker_running?
-      !! @background_worker_thread
+      @background_worker_thread          &&
+        @background_worker_thread.alive? &&
+        @background_worker               &&
+        @background_worker.running?
     end
 
     # Creates the worker thread. The worker thread is a loop that runs continuously. It sleeps for +Agent#period+ and when it wakes,
@@ -273,10 +276,10 @@ module ScoutApm
     # Loads the instrumention logic.
     def load_instruments
       if !background_job_missing?
-        case environment.background_job_name
-        when :delayed_job
-          install_instrument(ScoutApm::Instruments::DelayedJob)
-        end
+#        case environment.background_job_name
+#        when :delayed_job
+#          install_instrument(ScoutApm::Instruments::DelayedJob)
+#        end
       else
         case environment.framework
         when :rails       then install_instrument(ScoutApm::Instruments::ActionControllerRails2)

data/lib/scout_apm/agent/reporting.rb

@@ -58,6 +58,7 @@ module ScoutApm
         slow_transactions = reporting_period.slow_transactions_payload
         jobs = reporting_period.jobs
         slow_jobs = reporting_period.slow_jobs_payload
+        histograms = reporting_period.histograms
 
         metadata = {
           :app_root      => ScoutApm::Environment.instance.root.to_s,
@@ -68,9 +69,9 @@ module ScoutApm
           :platform      => "ruby",
         }
 
-        log_deliver(metrics, slow_transactions, metadata, slow_jobs)
+        log_deliver(metrics, slow_transactions, metadata, slow_jobs, histograms)
 
-        payload = ScoutApm::Serializers::PayloadSerializer.serialize(metadata, metrics, slow_transactions, jobs, slow_jobs)
+        payload = ScoutApm::Serializers::PayloadSerializer.serialize(metadata, metrics, slow_transactions, jobs, slow_jobs, histograms)
         logger.debug("Payload: #{payload}")
 
         reporter.report(payload, headers)
@@ -80,7 +81,7 @@ module ScoutApm
         logger.debug e.backtrace
       end
 
-      def log_deliver(metrics, slow_transactions, metadata, jobs_traces)
+      def log_deliver(metrics, slow_transactions, metadata, jobs_traces, histograms)
         total_request_count = metrics.
           select { |meta,stats| meta.metric_name =~ /\AController/ }.
           inject(0) {|sum, (_, stat)| sum + stat.call_count }
@@ -97,6 +98,7 @@ module ScoutApm
         metrics_clause    = "#{metrics.length} Metrics for #{total_request_count} requests"
         slow_trans_clause = "#{slow_transactions.length} Slow Transaction Traces"
         job_clause        = "#{jobs_traces.length} Job Traces"
+        histogram_clause  = "#{histograms.length} Histograms"
 
         logger.info "#{time_clause} Delivering #{metrics_clause} and #{slow_trans_clause} and #{job_clause}, #{process_log_str}."
         logger.debug("Metrics: #{metrics.pretty_inspect}\nSlowTrans: #{slow_transactions.pretty_inspect}\nMetadata: #{metadata.inspect.pretty_inspect}")

data/lib/scout_apm/background_job_integrations/delayed_job.rb

@@ -8,12 +8,58 @@ module ScoutApm
       end
 
       def present?
-        defined?(::Delayed::Job) && (File.basename($0) =~ /\Adelayed_job/)
+        defined?(::Delayed::Job) && File.basename($PROGRAM_NAME).start_with?('delayed_job')
       end
 
       def forking?
         false
       end
+
+      def install
+        plugin = Class.new(Delayed::Plugin) do
+          require 'delayed_job'
+
+          callbacks do |lifecycle|
+            lifecycle.around(:invoke_job) do |job, *args, &block|
+              ScoutApm::Agent.instance.start_background_worker unless ScoutApm::Agent.instance.background_worker_running?
+
+              name = job.name
+              queue = job.queue || "default"
+
+              req = ScoutApm::RequestManager.lookup
+              req.job!
+
+              begin
+                latency = Time.now - job.created_at
+                req.annotate_request(:queue_latency => latency)
+              rescue
+              end
+
+              queue_layer = ScoutApm::Layer.new('Queue', queue)
+              job_layer = ScoutApm::Layer.new('Job', name)
+
+              begin
+                req.start_layer(queue_layer)
+                started_queue = true
+                req.start_layer(job_layer)
+                started_job = true
+
+                # Call the job itself.
+                block.call(job, *args)
+              rescue
+                req.error!
+                raise
+              ensure
+                req.stop_layer if started_job
+                req.stop_layer if started_queue
+              end
+            end
+          end
+        end
+
+        Delayed::Worker.plugins << plugin # ScoutApm::BackgroundJobIntegrations::DelayedJobPlugin
+      end
     end
   end
 end
+

data/lib/scout_apm/background_worker.rb

@@ -11,6 +11,10 @@ module ScoutApm
       @keep_running = true
     end
 
+    def running?
+      @keep_running
+    end
+
     def stop
       ScoutApm::Agent.instance.logger.debug "Background Worker: stop requested"
       @keep_running = false

data/lib/scout_apm/config.rb

@@ -159,7 +159,8 @@ module ScoutApm
         'enable_background_jobs' => true,
         'ignore'                 => [],
         'dev_trace'              => false,
-        'profile'                => true # for scoutprof
+        'profile'                => true, # for scoutprof
+        'compress_payload'       => true,
       }.freeze
 
       def value(key)

data/lib/scout_apm/environment.rb

@@ -25,7 +25,7 @@ module ScoutApm
 
     BACKGROUND_JOB_INTEGRATIONS = [
       ScoutApm::BackgroundJobIntegrations::Sidekiq.new,
-      # ScoutApm::BackgroundJobIntegrations::DelayedJob.new
+      ScoutApm::BackgroundJobIntegrations::DelayedJob.new,
     ]
 
     FRAMEWORK_INTEGRATIONS = [

data/lib/scout_apm/histogram.rb

@@ -90,7 +90,11 @@ module ScoutApm
     def combine!(other)
       mutex.synchronize do
         other.mutex.synchronize do
-          @bins = (other.bins + @bins).sort_by {|b| b.value }
+          @bins = (other.bins + @bins).
+            group_by {|b| b.value }.
+            map {|val, bs| [val, bs.inject(0) {|sum, b| sum + b.count }] }.
+            map {|val, sum| HistogramBin.new(val,sum) }.
+            sort_by { |b| b.value }
           @total += other.total
           trim
           self
@@ -100,7 +104,12 @@ module ScoutApm
 
     def as_json
       mutex.synchronize do
-        bins.map{|b| [b.value, b.count]}
+        bins.map{ |b|
+          [
+            ScoutApm::Utils::Numbers.round(b.value, 4),
+            b.count
+          ]
+        }
       end
     end
 

data/lib/scout_apm/instruments/mongoid.rb

@@ -50,11 +50,24 @@ module ScoutApm
                 with_scout_instruments = %Q[
                 def #{method}_with_scout_instruments(*args, &block)
 
+
                   req = ScoutApm::RequestManager.lookup
                   *db, collection = view.collection.namespace.split(".")
 
                   name = collection + "/#{method}"
-                  filter = ScoutApm::Instruments::Mongoid.anonymize_filter(view.filter)
+
+                  # Between Mongo gem version 2.1 and 2.3, this method name was
+                  # changed. Accomodate both.  If for some reason neither is
+                  # there, try to continue with an empty "filter" hash.
+                  raw_filter = if view.respond_to?(:selector)
+                    view.selector
+                  elsif view.respond_to?(:filter)
+                    view.filter
+                  else
+                    {}
+                  end
+
+                  filter = ScoutApm::Instruments::Mongoid.anonymize_filter(raw_filter)
 
                   layer = ScoutApm::Layer.new("MongoDB", name)
                   layer.desc = filter.inspect

data/lib/scout_apm/instruments/percentile_sampler.rb

@@ -1,36 +1,53 @@
 module ScoutApm
   module Instruments
+
+    class HistogramReport
+      attr_reader :name
+      attr_reader :histogram
+
+      def initialize(name, histogram)
+        @name = name
+        @histogram = histogram
+      end
+
+      def combine!(other)
+        raise "Mismatched Histogram Names" unless name == other.name
+        histogram.combine!(other.histogram)
+        self
+      end
+    end
+
     class PercentileSampler
       attr_reader :logger
 
-      attr_reader :percentiles
+      # A hash of { time => RequestHistograms }
+      attr_reader :histograms
 
-      def initialize(logger, percentiles)
+      def initialize(logger, histograms)
         @logger = logger
-        @percentiles = Array(percentiles)
+        @histograms = histograms
       end
 
       def human_name
-        "Percentiles"
+        'Percentiles'
       end
 
-      # Gets the 95th%ile for the time requested
-      def metrics(time)
-        ms = {}
-        histos = ScoutApm::Agent.instance.request_histograms_by_time[time]
-        histos.each_name do |name|
-          percentiles.each do |percentile|
-            meta = MetricMeta.new("Percentile/#{percentile}/#{name}")
-            stat = MetricStats.new
-            stat.update!(histos.quantile(name, percentile))
-            ms[meta] = stat
-          end
-        end
+      def metrics(timestamp, store)
+        store.track_histograms!(percentiles(timestamp), :timestamp => timestamp)
+      end
+
+      def percentiles(time)
+        result = []
 
-        # Wipe the histograms we just collected data on
-        ScoutApm::Agent.instance.request_histograms_by_time.delete(time)
+        histogram = histograms.delete(time)
+
+        return result unless histogram
+
+        histogram.each_name do |name|
+          result << HistogramReport.new(name, histogram.raw(name))
+        end
 
-        ms
+        result
       end
     end
   end

data/lib/scout_apm/instruments/process/process_cpu.rb

@@ -29,16 +29,17 @@ module ScoutApm
           "Process CPU"
         end
 
-        def metrics(_time)
+        def metrics(timestamp, store)
           result = run
           if result
             meta = MetricMeta.new("#{metric_type}/#{metric_name}")
             stat = MetricStats.new(false)
             stat.update!(result)
-            { meta => stat }
+            store.track!({ meta => stat }, :timestamp => timestamp)
           else
             {}
           end
+
         end
 
         # TODO: Figure out a good default instead of nil

data/lib/scout_apm/instruments/process/process_memory.rb

@@ -33,20 +33,20 @@ module ScoutApm
           "Process Memory"
         end
 
-        def metrics(_time)
+        def metrics(timestamp, store)
           result = run
           if result
             meta = MetricMeta.new("#{metric_type}/#{metric_name}")
             stat = MetricStats.new(false)
             stat.update!(result)
-            { meta => stat }
+            store.track!({ meta => stat }, :timestamp => timestamp)
           else
             {}
           end
         end
 
         def run
-          self.class.rss_in_mb.tap { |res| logger.debug "#{human_name}: #{res.inspect}" }      
+          self.class.rss_in_mb.tap { |res| logger.debug "#{human_name}: #{res.inspect}" }
         end
       end
     end

data/lib/scout_apm/layer_converters/converter_base.rb

@@ -1,6 +1,7 @@
 module ScoutApm
   module LayerConverters
     class ConverterBase
+
       attr_reader :walker
       attr_reader :request
       attr_reader :root_layer
@@ -8,7 +9,10 @@ module ScoutApm
       def initialize(request)
         @request = request
         @root_layer = request.root_layer
+        @backtraces = []
         @walker = DepthFirstWalker.new(root_layer)
+
+        @limited = false
       end
 
       # Scope is determined by the first Controller we hit.  Most of the time
@@ -27,6 +31,195 @@ module ScoutApm
           return layer if layer.type == layer_type
         end
       end
+
+      ################################################################################
+      # Subscoping
+      ################################################################################
+      #
+      # Keep a list of subscopes, but only ever use the front one.  The rest
+      # get pushed/popped in cases when we have many levels of subscopable
+      # layers.  This lets us push/pop without otherwise keeping track very closely.
+      def setup_subscopable_callbacks
+        @subscope_layers = []
+
+        walker.before do |layer|
+          if layer.subscopable?
+            @subscope_layers.push(layer)
+          end
+        end
+
+        walker.after do |layer|
+          if layer.subscopable?
+            @subscope_layers.pop
+          end
+        end
+      end
+
+      def subscoped?(layer)
+        @subscope_layers.first && layer != @subscope_layers.first # Don't scope under ourself.
+      end
+
+      def subscope_name
+        @subscope_layers.first.legacy_metric_name
+      end
+
+
+      ################################################################################
+      # Backtrace Handling
+      ################################################################################
+      #
+      # Because we get several layers for the same thing if you call an
+      # instrumented thing repeatedly, and only some of them may have
+      # backtraces captured, we store the backtraces off into another spot
+      # during processing, then at the end, we loop over those saved
+      # backtraces, putting them back into the metrics hash.
+      #
+      # This comes up most often when capturing n+1 backtraces. Because the
+      # query may be fast enough to evade our time-limit based backtrace
+      # capture, only the Nth item (see TrackedRequest for more detail) has a
+      # backtrack captured.  This sequence makes sure that we report up that
+      # backtrace in the aggregated set of metrics around that call.
+
+      # Call this as you are processing each layer. It will store off backtraces
+      def store_backtrace(layer, meta)
+        return unless layer.backtrace
+
+        bt = ScoutApm::Utils::BacktraceParser.new(layer.backtrace).call
+        if bt.any?
+          meta.backtrace = bt
+          @backtraces << meta
+        else
+          ScoutApm::Agent.instance.logger.debug { "Unable to capture an app-specific backtrace for #{meta.inspect}\n#{layer.backtrace}" }
+        end
+      end
+
+      # Call this after you finish walking the layers, and want to take the
+      # set-aside backtraces and place them into the metas they match
+      def attach_backtraces(metric_hash)
+        @backtraces.each do |meta_with_backtrace|
+          metric_hash.keys.find { |k| k == meta_with_backtrace }.backtrace = meta_with_backtrace.backtrace
+        end
+        metric_hash
+      end
+
+
+      ################################################################################
+      # Limit Handling
+      ################################################################################
+
+      # To prevent huge traces from being generated, we should stop collecting
+      # detailed metrics as we go beyond some reasonably large count.
+      #
+      # We should still add up the /all aggregates.
+
+      MAX_METRICS = 500
+
+      def over_metric_limit?(metric_hash)
+        if metric_hash.size > MAX_METRICS
+          @limited = true
+        else
+          false
+        end
+      end
+
+      def limited?
+        !! @limited
+      end
+
+      ################################################################################
+      # Meta Scope
+      ################################################################################
+
+      # When we make MetricMeta records, we need to determine a few things from layer.
+      def make_meta_options(layer)
+        scope_hash = make_meta_options_scope(layer)
+        desc_hash = make_meta_options_desc_hash(layer)
+
+        scope_hash.merge(desc_hash)
+      end
+
+      def make_meta_options_scope(layer)
+        # This layer is scoped under another thing. Typically that means this is a layer under a view.
+        # Like: Controller -> View/users/show -> ActiveRecord/user/find
+        #   in that example, the scope is the View/users/show
+        if subscoped?(layer)
+          {:scope => subscope_name}
+
+        # We don't scope the controller under itself
+        elsif layer == scope_layer
+          {}
+
+        # This layer is a top level metric ("ActiveRecord", or "HTTP" or
+        # whatever, directly under the controller), so scope to the
+        # Controller
+        else
+          {:scope => scope_layer.legacy_metric_name}
+        end
+      end
+
+      def make_meta_options_desc_hash(layer)
+        if layer.desc
+          {:desc => layer.desc.to_s}
+        else
+          {}
+        end
+      end
+
+
+      ################################################################################
+      # Storing metrics into the hashes
+      ################################################################################
+
+      # This is the detailed metric - type, name, backtrace, annotations, etc.
+      def store_specific_metric(layer, metric_hash, allocation_metric_hash)
+        return false if over_metric_limit?(metric_hash)
+
+        meta_options = make_meta_options(layer)
+
+        meta = MetricMeta.new(layer.legacy_metric_name, meta_options)
+        meta.extra.merge!(layer.annotations)
+
+        store_backtrace(layer, meta)
+
+        metric_hash[meta] ||= MetricStats.new(meta_options.has_key?(:scope))
+        allocation_metric_hash[meta] ||= MetricStats.new(meta_options.has_key?(:scope))
+
+        # timing
+        stat = metric_hash[meta]
+        stat.update!(layer.total_call_time, layer.total_exclusive_time)
+
+        # allocations
+        stat = allocation_metric_hash[meta]
+        stat.update!(layer.total_allocations, layer.total_exclusive_allocations)
+      end
+
+      # Merged Metric - no specifics, just sum up by type (ActiveRecord, View, HTTP, etc)
+      def store_aggregate_metric(layer, metric_hash, allocation_metric_hash)
+          meta = MetricMeta.new("#{layer.type}/all")
+
+          metric_hash[meta] ||= MetricStats.new(false)
+          allocation_metric_hash[meta] ||= MetricStats.new(false)
+
+          # timing
+          stat = metric_hash[meta]
+          stat.update!(layer.total_call_time, layer.total_exclusive_time)
+
+          # allocations
+          stat = allocation_metric_hash[meta]
+          stat.update!(layer.total_allocations, layer.total_exclusive_allocations)
+      end
+
+      ################################################################################
+      # Misc Helpers
+      ################################################################################
+
+      # Sometimes we start capturing a layer without knowing if we really
+      # want to make an entry for it.  See ActiveRecord instrumentation for
+      # an example. We start capturing before we know if a query is cached
+      # or not, and want to skip any cached queries.
+      def skip_layer?(layer)
+        return true if layer.annotations[:ignorable]
+      end
     end
   end
 end