scout_apm 1.6.8 → 2.0.0.pre

data/lib/scout_apm/layer_converters/slow_job_converter.rb

@@ -4,29 +4,23 @@ module ScoutApm
       def initialize(*)
         @backtraces = []
         super
-
-        # After call to super, so @request is populated
-        @points = if request.job?
-                    ScoutApm::Agent.instance.slow_job_policy.score(request)
-                  else
-                    -1
-                  end
       end
 
-      def name
-        request.unique_name
-      end
+      def call
+        return unless request.job?
 
-      def score
-        @points
-      end
+        job_name = [queue_layer.name, job_layer.name]
 
-      def call
-        return nil unless request.job?
-        return nil unless queue_layer
-        return nil unless job_layer
+        slow_enough = ScoutApm::Agent.instance.slow_job_policy.slow?(job_name, root_layer.total_call_time)
+        return unless slow_enough
+
+        # record the change in memory usage
+        mem_delta = ScoutApm::Instruments::Process::ProcessMemory.rss_to_mb(request.capture_mem_delta!)
 
-        ScoutApm::Agent.instance.slow_job_policy.stored!(request)
+        timing_metrics, allocation_metrics = create_metrics
+        unless ScoutApm::Instruments::Allocations::ENABLED
+          allocation_metrics = {}
+        end
 
         SlowJobRecord.new(
           queue_layer.name,
@@ -35,8 +29,11 @@ module ScoutApm
           job_layer.total_call_time,
           job_layer.total_exclusive_time,
           request.context,
-          create_metrics,
-          score)
+          timing_metrics,
+          allocation_metrics,
+          mem_delta,
+          job_layer.total_allocations
+        )
       end
 
       def queue_layer
@@ -47,8 +44,15 @@ module ScoutApm
         @job_layer ||= find_first_layer_of_type("Job")
       end
 
+      def find_first_layer_of_type(layer_type)
+        walker.walk do |layer|
+          return layer if layer.type == layer_type
+        end
+      end
+
       def create_metrics
         metric_hash = Hash.new
+        allocation_metric_hash = Hash.new
 
         # Keep a list of subscopes, but only ever use the front one.  The rest
         # get pushed/popped in cases when we have many levels of subscopable
@@ -68,6 +72,14 @@ module ScoutApm
         end
 
         walker.walk do |layer|
+          # Sometimes we start capturing a layer without knowing if we really
+          # want to make an entry for it.  See ActiveRecord instrumentation for
+          # an example. We start capturing before we know if a query is cached
+          # or not, and want to skip any cached queries.
+          next if layer.annotations[:ignorable]
+
+          # The queue_layer is useful to capture for other reasons, but doesn't
+          # create a MetricMeta/Stat of its own
           next if layer == queue_layer
 
           meta_options = if subscope_layers.first && layer != subscope_layers.first # Don't scope under ourself.
@@ -82,6 +94,8 @@ module ScoutApm
           # Specific Metric
           meta_options.merge!(:desc => layer.desc.to_s) if layer.desc
           meta = MetricMeta.new(layer.legacy_metric_name, meta_options)
+          meta.extra.merge!(layer.annotations)
+
           if layer.backtrace
             bt = ScoutApm::Utils::BacktraceParser.new(layer.backtrace).call
             if bt.any? # we could walk thru the call stack and not find in-app code
@@ -95,19 +109,26 @@ module ScoutApm
           end
 
           metric_hash[meta] ||= MetricStats.new( meta_options.has_key?(:scope) )
+          allocation_metric_hash[meta] ||= MetricStats.new( meta_options.has_key?(:scope) )
           stat = metric_hash[meta]
           stat.update!(layer.total_call_time, layer.total_exclusive_time)
+          stat = allocation_metric_hash[meta]
+          stat.update!(layer.total_allocations, layer.total_exclusive_allocations)
 
           # Merged Metric (no specifics, just sum up by type)
           meta = MetricMeta.new("#{layer.type}/all")
           metric_hash[meta] ||= MetricStats.new(false)
+          allocation_metric_hash[meta] ||= MetricStats.new(false)
           stat = metric_hash[meta]
           stat.update!(layer.total_call_time, layer.total_exclusive_time)
+          stat = allocation_metric_hash[meta]
+          stat.update!(layer.total_allocations, layer.total_exclusive_allocations)
         end
 
         metric_hash = attach_backtraces(metric_hash)
+        allocation_metric_hash = attach_backtraces(allocation_metric_hash)
 
-        metric_hash
+        [metric_hash,allocation_metric_hash]
       end
 
       def attach_backtraces(metric_hash)

data/lib/scout_apm/layer_converters/slow_request_converter.rb

@@ -4,50 +4,49 @@ module ScoutApm
       def initialize(*)
         @backtraces = [] # An Array of MetricMetas that have a backtrace
         super
-
-        # After call to super, so @request is populated
-        @points = if request.web?
-                    ScoutApm::Agent.instance.slow_request_policy.score(request)
-                  else
-                    -1
-                  end
-      end
-
-      def name
-        request.unique_name
       end
 
-      def score
-        @points
-      end
-
-      # Unconditionally attempts to convert this into a SlowTransaction object.
-      # Can return nil if the request didn't have any scope_layer.
       def call
         scope = scope_layer
-        return nil unless scope
+        return [nil, {}] unless scope
+
+        policy = ScoutApm::Agent.instance.slow_request_policy.capture_type(root_layer.total_call_time)
+        if policy == ScoutApm::SlowRequestPolicy::CAPTURE_NONE
+          return [nil, {}]
+        end
 
-        ScoutApm::Agent.instance.slow_request_policy.stored!(request)
+        # record the change in memory usage
+        mem_delta = ScoutApm::Instruments::Process::ProcessMemory.rss_to_mb(@request.capture_mem_delta!)
+
+        # increment the slow transaction count if this is a slow transaction.
+        meta = MetricMeta.new("SlowTransaction/#{scope.legacy_metric_name}")
+        stat = MetricStats.new
+        stat.update!(1)
 
         uri = request.annotations[:uri] || ""
 
-        (ScoutApm::Agent.instance.config.value("ignore_traces") || []).each do |pattern|
-          if /#{pattern}/ =~ uri
-            ScoutApm::Agent.instance.logger.debug("Skipped recording a trace for #{uri} due to `ignore_traces` pattern: #{pattern}")
-            return nil
-          end
+        timing_metrics, allocation_metrics = create_metrics
+        unless ScoutApm::Instruments::Allocations::ENABLED
+          allocation_metrics = {}
         end
 
-        metrics = create_metrics
-
-        SlowTransaction.new(uri,
-                            scope.legacy_metric_name,
-                            root_layer.total_call_time,
-                            metrics,
-                            request.context,
-                            root_layer.stop_time,
-                            [], # stackprof
-                            @points)
+        # Disable stackprof output for now
+        stackprof = [] # request.stackprof
+
+        [
+          SlowTransaction.new(uri,
+                              scope.legacy_metric_name,
+                              root_layer.total_call_time,
+                              timing_metrics,
+                              allocation_metrics,
+                              request.context,
+                              root_layer.stop_time,
+                              stackprof,
+                              mem_delta,
+                              root_layer.total_allocations
+                              ),
+          { meta => stat }
+        ]
       end
 
       # Iterates over the TrackedRequest's MetricMetas that have backtraces and attaches each to correct MetricMeta in the Metric Hash.
@@ -58,12 +57,14 @@ module ScoutApm
         metric_hash
       end
 
-      # Full metrics from this request. These get aggregated in Store for the
-      # overview metrics, or stored permanently in a SlowTransaction
+      # Full metrics from this request. These get stored permanently in a SlowTransaction.
       # Some merging of metrics will happen here, so if a request calls the same
       # ActiveRecord or View repeatedly, it'll get merged.
+      # 
+      # This returns a 2-element of Metric Hashes (the first element is timing metrics, the second element is allocation metrics)
       def create_metrics
         metric_hash = Hash.new
+        allocation_metric_hash = Hash.new
 
         # Keep a list of subscopes, but only ever use the front one.  The rest
         # get pushed/popped in cases when we have many levels of subscopable
@@ -83,6 +84,14 @@ module ScoutApm
         end
 
         walker.walk do |layer|
+          # Sometimes we start capturing a layer without knowing if we really
+          # want to make an entry for it.  See ActiveRecord instrumentation for
+          # an example. We start capturing before we know if a query is cached
+          # or not, and want to skip any cached queries.
+          if layer.annotations[:ignorable]
+            next
+          end
+
           meta_options = if subscope_layers.first && layer != subscope_layers.first # Don't scope under ourself.
                            subscope_name = subscope_layers.first.legacy_metric_name
                            {:scope => subscope_name}
@@ -95,6 +104,7 @@ module ScoutApm
           # Specific Metric
           meta_options.merge!(:desc => layer.desc.to_s) if layer.desc
           meta = MetricMeta.new(layer.legacy_metric_name, meta_options)
+          meta.extra.merge!(layer.annotations)
           if layer.backtrace
             bt = ScoutApm::Utils::BacktraceParser.new(layer.backtrace).call
             if bt.any? # we could walk thru the call stack and not find in-app code
@@ -109,19 +119,30 @@ module ScoutApm
             end
           end
           metric_hash[meta] ||= MetricStats.new( meta_options.has_key?(:scope) )
+          allocation_metric_hash[meta] ||= MetricStats.new( meta_options.has_key?(:scope) )
+          # timing
           stat = metric_hash[meta]
           stat.update!(layer.total_call_time, layer.total_exclusive_time)
+          # allocations
+          stat = allocation_metric_hash[meta]
+          stat.update!(layer.total_allocations, layer.total_exclusive_allocations)
 
           # Merged Metric (no specifics, just sum up by type)
           meta = MetricMeta.new("#{layer.type}/all")
           metric_hash[meta] ||= MetricStats.new(false)
+          allocation_metric_hash[meta] ||= MetricStats.new(false)
+          # timing
           stat = metric_hash[meta]
           stat.update!(layer.total_call_time, layer.total_exclusive_time)
+          # allocations
+          stat = allocation_metric_hash[meta]
+          stat.update!(layer.total_allocations, layer.total_exclusive_allocations)
         end
 
         metric_hash = attach_backtraces(metric_hash)
+        allocation_metric_hash = attach_backtraces(allocation_metric_hash)
 
-        metric_hash
+        [metric_hash,allocation_metric_hash]
       end
     end
   end

data/lib/scout_apm/metric_meta.rb

@@ -17,11 +17,7 @@ class MetricMeta
 
   # Unsure if type or bucket is a better name.
   def type
-    bucket_type
-  end
-
-  def name
-    bucket_name
+    bucket
   end
 
   # A key metric is the "core" of a request - either the Rails controller reached, or the background Job executed

data/lib/scout_apm/metric_set.rb

@@ -2,7 +2,7 @@ module ScoutApm
   class MetricSet
     # We can't aggregate CPU, Memory, Capacity, or Controller, so pass through these metrics directly
     # TODO: Figure out a way to not have this duplicate what's in Samplers, and also on server's ingest
-    PASSTHROUGH_METRICS = ["CPU", "Memory", "Instance", "Controller", "SlowTransaction", "Percentile", "Job"]
+    PASSTHROUGH_METRICS = ["CPU", "Memory", "Instance", "Controller", "SlowTransaction"]
 
     attr_reader :metrics
 
@@ -23,15 +23,11 @@ module ScoutApm
         @metrics[meta].combine!(stat)
 
       elsif meta.type == "Errors" # Sadly special cased, we want both raw and aggregate values
-        # When combining MetricSets between different 
-          @metrics[meta] ||= MetricStats.new
-          @metrics[meta].combine!(stat)
-
-        if !@combine_in_progress
-          agg_meta = MetricMeta.new("Errors/Request", :scope => meta.scope)
-          @metrics[agg_meta] ||= MetricStats.new
-          @metrics[agg_meta].combine!(stat)
-        end
+        @metrics[meta] ||= MetricStats.new
+        @metrics[meta].combine!(stat)
+        agg_meta = MetricMeta.new("Errors/Request", :scope => meta.scope)
+        @metrics[agg_meta] ||= MetricStats.new
+        @metrics[agg_meta].combine!(stat)
 
       else # Combine down to a single /all key
         agg_meta = MetricMeta.new("#{meta.type}/all", :scope => meta.scope)
@@ -40,13 +36,8 @@ module ScoutApm
       end
     end
 
-    # Sets a combine_in_progress flag to prevent double-counting Error metrics.
-    # Without it, the Errors/Request number would be increasingly off as
-    # metric_sets get merged in.
     def combine!(other)
-      @combine_in_progress = true
       absorb_all(other.metrics)
-      @combine_in_progress = false
       self
     end
   end

data/lib/scout_apm/reporter.rb

@@ -17,10 +17,8 @@ module ScoutApm
 
     # TODO: Parse & return a real response object, not the HTTP Response object
     def report(payload, headers = {})
-      # Some posts (typically ones under development) bypass the ingestion pipeline and go directly to the webserver. They use direct_host instead of host
-      hosts = [:deploy_hook, :instant_trace].include?(type) ? config.value('direct_host') : config.value('host')
+      Array(config.value('host')).each do |host|
 
-      Array(hosts).each do |host|
         full_uri = uri(host)
         response = post(full_uri, payload, headers)
         unless response && response.is_a?(Net::HTTPSuccess)
@@ -36,7 +34,7 @@ module ScoutApm
       when :app_server_load
         URI.parse("#{host}/apps/app_server_load.scout?key=#{config.value('key')}&name=#{CGI.escape(Environment.instance.application_name)}")
       when :deploy_hook
-        URI.parse("#{host}/apps/deploy.scout?key=#{config.value('key')}&name=#{CGI.escape(config.value('name'))}")
+        URI.parse("https://apm.scoutapp.com/apps/deploy.scout?key=#{config.value('key')}&name=#{CGI.escape(config.value('name'))}")
       end.tap{|u| logger.debug("Posting to #{u.to_s}")}
     end
 
@@ -58,7 +56,7 @@ module ScoutApm
     private
 
     def post(uri, body, headers = Hash.new)
-      response = nil
+      response = :connection_failed
       request(uri) do |connection|
         post = Net::HTTP::Post.new( uri.path +
                                     (uri.query ? ('?' + uri.query) : ''),
@@ -84,7 +82,7 @@ module ScoutApm
         logger.debug "/#{type} FAILED: #{response.inspect}"
       end
     rescue Exception
-      logger.debug "Exception sending request to server: \n#{$!.message}\n\t#{$!.backtrace.join("\n\t")}"
+      logger.info "Exception sending request to server: \n#{$!.message}\n\t#{$!.backtrace.join("\n\t")}"
     ensure
       response
     end

data/lib/scout_apm/serializers/metrics_to_json_serializer.rb

@@ -9,7 +9,11 @@ module ScoutApm
       end
 
       def as_json
-        metrics.map{|meta, stat| metric_as_json(meta, stat) }
+        if metrics
+          metrics.map{|meta, stat| metric_as_json(meta, stat) }
+        else
+          nil
+        end
       end
 
       # Children metrics is a hash of meta=>stat pairs. Leave empty for no children.

data/lib/scout_apm/serializers/payload_serializer_to_json.rb

@@ -22,7 +22,7 @@ module ScoutApm
 
         def rearrange_the_slow_transactions(slow_transactions)
           slow_transactions.to_a.map do |slow_t|
-            slow_t.as_json.merge(:metrics => rearrange_the_metrics(slow_t.metrics))
+            slow_t.as_json.merge(:metrics => rearrange_the_metrics(slow_t.metrics), :allocation_metrics => rearrange_the_metrics(slow_t.allocation_metrics))
           end
         end
 
@@ -59,8 +59,6 @@ module ScoutApm
             "[#{all_the_elements.join(",")}]"
           when Numeric
             formatee
-          when Time
-            %Q["#{formatee.iso8601}"]
           when nil
             "null"
           else # strings and everything

data/lib/scout_apm/serializers/slow_jobs_serializer_to_json.rb

@@ -17,11 +17,13 @@ module ScoutApm
             "time" => job.time,
             "total_time" => job.total_time,
             "exclusive_time" => job.exclusive_time,
-
+            "mem_delta" => job.mem_delta,
+            "allocations" => job.allocations,
+            "seconds_since_startup" => job.seconds_since_startup,
+            "hostname" => job.hostname,
             "metrics" => MetricsToJsonSerializer.new(job.metrics).as_json, # New style of metrics
+            "allocation_metrics" => MetricsToJsonSerializer.new(job.allocation_metrics).as_json, # New style of metrics
             "context" => job.context.to_hash,
-
-            "score" => job.score,
           }
         end
       end

data/lib/scout_apm/slow_job_policy.rb

@@ -1,99 +1,29 @@
-# Long running class that determines if, and in how much detail a potentially
-# slow job should be recorded in
-
+# Create one of these at startup time, and ask it if a certain worker's
+# processing time is slow enough for us to collect a slow trace.
+#
+# Keeps track of a histogram of times for each worker class (spearately), and
+# uses a percentile of normal to mark individual runs as "slow".
+#
+# This assumes that all worker calls will be requested once to `slow?`, so that
+# the data can be stored
 module ScoutApm
   class SlowJobPolicy
-    CAPTURE_TYPES = [
-      CAPTURE_DETAIL  = "capture_detail",
-      CAPTURE_NONE    = "capture_none",
-    ]
-
-    # Adjust speed points. See the function
-    POINT_MULTIPLIER_SPEED = 0.25
-
-    # For each minute we haven't seen an endpoint
-    POINT_MULTIPLIER_AGE = 0.25
-
-    # Outliers are worth up to "1000ms" of weight
-    POINT_MULTIPLIER_PERCENTILE = 1.0
-
-    # A hash of Job Names to the last time we stored a slow trace for it.
-    #
-    # Defaults to a start time that is pretty close to application boot time.
-    # So the "age" of an endpoint we've never seen is the time the application
-    # has been running.
-    attr_reader :last_seen
+    DEFAULT_HISTOGRAM_SIZE = 50
 
+    QUANTILE = 95
 
-    def initialize
-      zero_time = Time.now
-      @last_seen = Hash.new { |h, k| h[k] = zero_time }
+    def initialize(histogram_size = DEFAULT_HISTOGRAM_SIZE)
+      @histograms = Hash.new { |h, k| h[k] = NumericHistogram.new(histogram_size) }
     end
 
-    def stored!(request)
-      last_seen[unique_name_for(request)] = Time.now
-    end
-
-    # Determine if this job trace should be fully analyzed by scoring it
-    # across several metrics, and then determining if that's good enough to
-    # make it into this minute's payload.
-    #
-    # Due to the combining nature of the agent & layaway file, there's no
-    # guarantee that a high scoring local champion will still be a winner when
-    # they go up to "regionals" and are compared against the other processes
-    # running on a node.
-    def score(request)
-      unique_name = request.unique_name
-      if unique_name == :unknown
-        return -1 # A negative score, should never be good enough to store.
-      end
-
-      total_time = request.root_layer.total_call_time
-
-      # How long has it been since we've seen this?
-      age = Time.now - last_seen[unique_name]
-
-      # What approximate percentile was this request?
-      percentile = ScoutApm::Agent.instance.request_histograms.approximate_quantile_of_value(unique_name, total_time)
-
-      return speed_points(total_time) + percentile_points(percentile) + age_points(age)
-    end
-
-    private
-
-    def unique_name_for(request)
-      scope_layer = LayerConverters::ConverterBase.new(request).scope_layer
-      if scope_layer
-        scope_layer.legacy_metric_name
-      else
-        :unknown
-      end
-    end
-
-    # Time in seconds
-    # Logarithm keeps huge times from swamping the other metrics.
-    # 1+ is necessary to keep the log function in positive territory.
-    def speed_points(time)
-      Math.log(1 + time) * POINT_MULTIPLIER_SPEED
-    end
-
-    def percentile_points(percentile)
-      if percentile < 40
-        0.4 # Don't put much emphasis on capturing low percentiles.
-      elsif percentile < 60
-        1.4 # Highest here to get mean traces
-      elsif percentile < 90
-        0.7 # Between 60 & 90% is fine.
-      elsif percentile >= 90
-        1.4 # Highest here to get 90+%ile traces
-      else
-        # impossible.
-        percentile
-      end
-    end
+    # worker: just the worker class name. "PasswordResetJob" or similar
+    # total_time: runtime of the job in seconds
+    # returns true if this request should be stored in higher trace detail, false otherwise
+    def slow?(worker, total_time)
+      @histograms[worker].add(total_time)
+      return false if @histograms[worker].total == 1 # First call is never slow
 
-    def age_points(age)
-      age / 60.0 * POINT_MULTIPLIER_AGE
+      total_time >= @histograms[worker].quantile(QUANTILE)
     end
   end
 end

data/lib/scout_apm/slow_job_record.rb

@@ -14,10 +14,13 @@ module ScoutApm
     alias_method :total_call_time, :total_time
 
     attr_reader :metrics
+    attr_reader :allocation_metrics
+    attr_reader :mem_delta
+    attr_reader :allocations
+    attr_reader :hostname
+    attr_reader :seconds_since_startup
 
-    attr_reader :score
-
-    def initialize(queue_name, job_name, time, total_time, exclusive_time, context, metrics, score)
+    def initialize(queue_name, job_name, time, total_time, exclusive_time, context, metrics, allocation_metrics, mem_delta, allocations)
       @queue_name = queue_name
       @job_name = job_name
       @time = time
@@ -25,28 +28,17 @@ module ScoutApm
       @exclusive_time = exclusive_time
       @context = context
       @metrics = metrics
-      @score = score
+      @allocation_metrics = allocation_metrics
+      @mem_delta = mem_delta
+      @allocations = allocations
+      @seconds_since_startup = (Time.now - ScoutApm::Agent.instance.process_start_time)
+      @hostname = ScoutApm::Environment.instance.hostname
+      ScoutApm::Agent.instance.logger.debug { "Slow Job [#{metric_name}] - Call Time: #{total_call_time} Mem Delta: #{mem_delta}"}
     end
 
     def metric_name
       "Job/#{queue_name}/#{job_name}"
     end
 
-    ########################
-    # Scorable interface
-    #
-    # Needed so we can merge ScoredItemSet instances
-    def call
-      self
-    end
-
-    def name
-      metric_name
-    end
-
-    def score
-      @score
-    end
-
   end
 end