scout_apm 2.2.0.pre3 → 2.3.0.pre

data/lib/scout_apm/layer_children_set.rb

@@ -0,0 +1,77 @@
+module ScoutApm
+  # A set of children records for any given Layer.  This implements some
+  # rate-limiting logic.
+  #
+  # We store the first `unique_cutoff` count of each layer type. So if cutoff
+  # is 1000, we'd store 1000 HTTP layers, and 1000 ActiveRecord calls, and 1000
+  # of each other layer type. After that, make a LimitedLayer object and store
+  # only aggregate counts and times of future layers of that type. (So the
+  # 1001st an onward of ActiveRecord would get only aggregate times, and
+  # counts, without any detail about the SQL called)
+  #
+  # When the set of children is small, keep them unique
+  # When the set of children gets large enough, stop keeping details
+  #
+  # The next optimization, which is not yet implemented:
+  #   when the set of children gets larger, attempt to merge them without data loss
+  class LayerChildrenSet
+    include Enumerable
+
+    # By default, how many unique children of a type do we store before
+    # flipping over to storing only aggregate info.
+    DEFAULT_UNIQUE_CUTOFF = 2000
+    attr_reader :unique_cutoff
+
+    # The Set of children objects
+    attr_reader :children
+    private :children
+
+    def initialize(unique_cutoff = DEFAULT_UNIQUE_CUTOFF)
+      @children = Hash.new
+      @limited_layers = nil # populated when needed
+      @unique_cutoff = unique_cutoff
+    end
+
+    def child_set(metric_type)
+      children[metric_type] = Set.new if !children.has_key?(metric_type)
+      children[metric_type]
+    end
+
+    # Add a new layer into this set
+    # Only add completed layers - otherwise this will collect up incorrect info
+    # into the created LimitedLayer, since it will "freeze" any current data for
+    # total_call_time and similar methods.
+    def <<(child)
+      metric_type = child.type
+      set = child_set(metric_type)
+
+      if set.size >= unique_cutoff
+        # find limited_layer
+        @limited_layers || init_limited_layers
+        @limited_layers[metric_type].absorb(child)
+      else
+        # we have space just add it
+        set << child
+      end
+    end
+
+    def each
+      children.each do |_type, set|
+        set.each do |child_layer|
+          yield child_layer
+        end
+      end
+
+      if @limited_layers
+        @limited_layers.each do |_type, limited_layer|
+          yield limited_layer
+        end
+      end
+    end
+
+    # hold off initializing this until we know we need it
+    def init_limited_layers
+      @limited_layers ||= Hash.new { |hash, key| hash[key] = LimitedLayer.new(key) }
+    end
+  end
+end

data/lib/scout_apm/layer_converters/allocation_metric_converter.rb

@@ -1,16 +1,15 @@
 module ScoutApm
   module LayerConverters
     class AllocationMetricConverter < ConverterBase
-      def call
-        scope = scope_layer
-        return {} unless scope
-        return {} unless ScoutApm::Instruments::Allocations::ENABLED
+      def record!
+        return unless scope_layer
+        return unless ScoutApm::Instruments::Allocations::ENABLED
 
-        meta = MetricMeta.new("ObjectAllocations", {:scope => scope.legacy_metric_name})
+        meta = MetricMeta.new("ObjectAllocations", {:scope => scope_layer.legacy_metric_name})
         stat = MetricStats.new
         stat.update!(root_layer.total_allocations)
 
-        { meta => stat }
+        @store.track!({ meta => stat })
       end
     end
   end

data/lib/scout_apm/layer_converters/converter_base.rb

@@ -1,31 +1,218 @@
 module ScoutApm
   module LayerConverters
     class ConverterBase
-      attr_reader :walker
+
       attr_reader :request
       attr_reader :root_layer
+      attr_reader :layer_finder
 
-      def initialize(request)
+      def initialize(request, layer_finder, store=nil)
         @request = request
+        @layer_finder = layer_finder
+        @store = store
+
         @root_layer = request.root_layer
-        @walker = DepthFirstWalker.new(root_layer)
+        @backtraces = []
+        @limited = false
       end
 
-      # Scope is determined by the first Controller we hit.  Most of the time
-      # there will only be 1 anyway.  But if you have a controller that calls
-      # another controller method, we may pick that up:
-      #     def update
-      #       show
-      #       render :update
-      #     end
       def scope_layer
-        @scope_layer ||= find_first_layer_of_type("Controller") || find_first_layer_of_type("Job")
+        layer_finder.scope
+      end
+
+      ################################################################################
+      # Subscoping
+      ################################################################################
+      #
+      # Keep a list of subscopes, but only ever use the front one.  The rest
+      # get pushed/popped in cases when we have many levels of subscopable
+      # layers.  This lets us push/pop without otherwise keeping track very closely.
+      def register_hooks(walker)
+        @subscope_layers = []
+
+        walker.before do |layer|
+          if layer.subscopable?
+            @subscope_layers.push(layer)
+          end
+        end
+
+        walker.after do |layer|
+          if layer.subscopable?
+            @subscope_layers.pop
+          end
+        end
+      end
+
+      def subscoped?(layer)
+        @subscope_layers.first && layer != @subscope_layers.first # Don't scope under ourself.
+      end
+
+      def subscope_name
+        @subscope_layers.first.legacy_metric_name
+      end
+
+
+      ################################################################################
+      # Backtrace Handling
+      ################################################################################
+      #
+      # Because we get several layers for the same thing if you call an
+      # instrumented thing repeatedly, and only some of them may have
+      # backtraces captured, we store the backtraces off into another spot
+      # during processing, then at the end, we loop over those saved
+      # backtraces, putting them back into the metrics hash.
+      #
+      # This comes up most often when capturing n+1 backtraces. Because the
+      # query may be fast enough to evade our time-limit based backtrace
+      # capture, only the Nth item (see TrackedRequest for more detail) has a
+      # backtrack captured.  This sequence makes sure that we report up that
+      # backtrace in the aggregated set of metrics around that call.
+
+      # Call this as you are processing each layer. It will store off backtraces
+      def store_backtrace(layer, meta)
+        return unless layer.backtrace
+
+        bt = ScoutApm::Utils::BacktraceParser.new(layer.backtrace).call
+        if bt.any?
+          meta.backtrace = bt
+          @backtraces << meta
+        end
       end
 
-      def find_first_layer_of_type(layer_type)
-        walker.walk do |layer|
-          return layer if layer.type == layer_type
+      # Call this after you finish walking the layers, and want to take the
+      # set-aside backtraces and place them into the metas they match
+      def attach_backtraces(metric_hash)
+        @backtraces.each do |meta_with_backtrace|
+          metric_hash.keys.find { |k| k == meta_with_backtrace }.backtrace = meta_with_backtrace.backtrace
         end
+        metric_hash
+      end
+
+
+      ################################################################################
+      # Limit Handling
+      ################################################################################
+
+      # To prevent huge traces from being generated, we should stop collecting
+      # detailed metrics as we go beyond some reasonably large count.
+      #
+      # We should still add up the /all aggregates.
+
+      MAX_METRICS = 500
+
+      def over_metric_limit?(metric_hash)
+        if metric_hash.size > MAX_METRICS
+          @limited = true
+        else
+          false
+        end
+      end
+
+      def limited?
+        !! @limited
+      end
+
+      ################################################################################
+      # Meta Scope
+      ################################################################################
+
+      # When we make MetricMeta records, we need to determine a few things from layer.
+      def make_meta_options(layer)
+        scope_hash = make_meta_options_scope(layer)
+        desc_hash = make_meta_options_desc_hash(layer)
+
+        scope_hash.merge(desc_hash)
+      end
+
+      def make_meta_options_scope(layer)
+        # This layer is scoped under another thing. Typically that means this is a layer under a view.
+        # Like: Controller -> View/users/show -> ActiveRecord/user/find
+        #   in that example, the scope is the View/users/show
+        if subscoped?(layer)
+          {:scope => subscope_name}
+
+        # We don't scope the controller under itself
+        elsif layer == scope_layer
+          {}
+
+        # This layer is a top level metric ("ActiveRecord", or "HTTP" or
+        # whatever, directly under the controller), so scope to the
+        # Controller
+        else
+          {:scope => scope_layer.legacy_metric_name}
+        end
+      end
+
+      def make_meta_options_desc_hash(layer, max_desc_length=1000)
+        if layer.desc
+          desc_s = layer.desc.to_s
+          trimmed_desc = desc_s[0 .. max_desc_length]
+          {:desc => trimmed_desc}
+        else
+          {}
+        end
+      end
+
+
+      ################################################################################
+      # Storing metrics into the hashes
+      ################################################################################
+
+      # This is the detailed metric - type, name, backtrace, annotations, etc.
+      def store_specific_metric(layer, metric_hash, allocation_metric_hash)
+        return false if over_metric_limit?(metric_hash)
+
+        meta_options = make_meta_options(layer)
+
+        meta = MetricMeta.new(layer.legacy_metric_name, meta_options)
+        meta.extra.merge!(layer.annotations) if layer.annotations
+
+        store_backtrace(layer, meta)
+
+        metric_hash[meta] ||= MetricStats.new(meta_options.has_key?(:scope))
+        allocation_metric_hash[meta] ||= MetricStats.new(meta_options.has_key?(:scope))
+
+        # timing
+        stat = metric_hash[meta]
+        stat.update!(layer.total_call_time, layer.total_exclusive_time)
+
+        # allocations
+        stat = allocation_metric_hash[meta]
+        stat.update!(layer.total_allocations, layer.total_exclusive_allocations)
+
+        if LimitedLayer === layer
+          metric_hash[meta].call_count = layer.count
+          allocation_metric_hash[meta].call_count = layer.count
+        end
+      end
+
+      # Merged Metric - no specifics, just sum up by type (ActiveRecord, View, HTTP, etc)
+      def store_aggregate_metric(layer, metric_hash, allocation_metric_hash)
+          meta = MetricMeta.new("#{layer.type}/all")
+
+          metric_hash[meta] ||= MetricStats.new(false)
+          allocation_metric_hash[meta] ||= MetricStats.new(false)
+
+          # timing
+          stat = metric_hash[meta]
+          stat.update!(layer.total_call_time, layer.total_exclusive_time)
+
+          # allocations
+          stat = allocation_metric_hash[meta]
+          stat.update!(layer.total_allocations, layer.total_exclusive_allocations)
+      end
+
+      ################################################################################
+      # Misc Helpers
+      ################################################################################
+
+      # Sometimes we start capturing a layer without knowing if we really
+      # want to make an entry for it.  See ActiveRecord instrumentation for
+      # an example. We start capturing before we know if a query is cached
+      # or not, and want to skip any cached queries.
+      def skip_layer?(layer)
+        return false if layer.annotations.nil?
+        return true  if layer.annotations[:ignorable]
       end
     end
   end

data/lib/scout_apm/layer_converters/database_converter.rb

@@ -0,0 +1,55 @@
+module ScoutApm
+  module LayerConverters
+    class DatabaseConverter < ConverterBase
+      def initialize(*)
+        super
+        @db_query_metric_set = DbQueryMetricSet.new
+      end
+
+      def register_hooks(walker)
+        super
+
+        return unless scope_layer
+
+        walker.on do |layer|
+          next if skip_layer?(layer)
+
+          stat = DbQueryMetricStats.new(
+            layer.name.model,
+            layer.name.normalized_operation,
+            scope_layer.legacy_metric_name, # controller_scope
+            1,                              # count, this is a single query, so 1
+            layer.total_call_time,
+            records_returned(layer)
+          )
+          @db_query_metric_set << stat
+        end
+      end
+
+      def skip_layer?(layer)
+        layer.type != 'ActiveRecord'            ||
+        layer.limited?                          ||
+        ( ! layer.name.respond_to?(:model))     ||
+        ( ! layer.name.respond_to?(:normalized_operation)) ||
+        layer.name.model.nil?                   ||
+        layer.name.normalized_operation.nil?
+      end
+
+      def record!
+        # Everything in the metric set here is from a single transaction, which
+        # we want to keep track of. (One web call did a User#find 10 times, but
+        # only due to 1 http request)
+        @db_query_metric_set.increment_transaction_count!
+        @store.track_db_query_metrics!(@db_query_metric_set)
+      end
+
+      def records_returned(layer)
+        if layer.annotations
+          layer.annotations.fetch(:record_count, 0)
+        else
+          0
+        end
+      end
+    end
+  end
+end

data/lib/scout_apm/layer_converters/depth_first_walker.rb

@@ -5,30 +5,42 @@ module ScoutApm
 
       def initialize(root_layer)
         @root_layer = root_layer
+
+        @on_blocks = []
+        @before_blocks = []
+        @after_blocks = []
       end
 
       def before(&block)
-        @before_block = block
+        @before_blocks << block
       end
 
       def after(&block)
-        @after_block = block
+        @after_blocks << block
+      end
+
+      def on(&block)
+        @on_blocks << block
       end
 
-      def walk(layer=root_layer, &block)
+      def walk(layer=root_layer)
         # Need to run this for the root layer the first time through.
         if layer == root_layer
-          @before_block.call(layer) if @before_block
-          yield layer
-          @after_block.call(layer) if @after_block
+          @before_blocks.each{|b| b.call(layer) }
+          @on_blocks.each{|b| b.call(layer) }
         end
 
         layer.children.each do |child|
-          @before_block.call(child) if @before_block
-          yield child
-          walk(child, &block)
-          @after_block.call(child) if @after_block
+          @before_blocks.each{|b| b.call(child) }
+          @on_blocks.each{|b| b.call(child) }
+          walk(child)
+          @after_blocks.each{|b| b.call(child) }
         end
+
+        if layer == root_layer
+          @after_blocks.each{|b| b.call(layer) }
+        end
+
         nil
       end
     end

data/lib/scout_apm/layer_converters/error_converter.rb

@@ -1,19 +1,17 @@
 module ScoutApm
   module LayerConverters
     class ErrorConverter < ConverterBase
-      def call
-        scope = scope_layer
-
+      def record!
         # Should we mark a request as errored out if a middleware raises?
         # How does that interact w/ a tool like Sentry or Honeybadger?
-        return {} unless scope
-        return {} unless request.error?
+        return unless scope_layer
+        return unless request.error?
 
-        meta = MetricMeta.new("Errors/#{scope.legacy_metric_name}", {})
+        meta = MetricMeta.new("Errors/#{scope_layer.legacy_metric_name}", {})
         stat = MetricStats.new
         stat.update!(1)
 
-        { meta => stat }
+        @store.track!({ meta => stat })
       end
     end
   end