scout_apm 2.1.14 → 2.1.15

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: faa58ca3a2800e8aade5ade08a31d2c5db1e5a7d
-  data.tar.gz: ceb42f746d3b72eb84610522eeee9247fd68cf81
+  metadata.gz: 2432d4b3a94011da560df94d67bfda1c0e58ed80
+  data.tar.gz: bafd8c126d8ea5e1329d7d9e119ce24cacea9af3
 SHA512:
-  metadata.gz: b692efc8a697ffb71657d9c2493c1c4fd2bc003f5970155339654679832d6b7fd9da95d26bc0d2b46c153980da697532376556556ff8b2ba10140cc5fb398180
-  data.tar.gz: 3ae789a0000e598a39bce414c4955d13fa354e38f716d53ddb6041160a89fc9a500c3f1d1144095e0487722eedc2951cb6681e8db4f8917bf162adbc52112b51
+  metadata.gz: 766e322ec27e6ba797cb7756e67975e2b4c41b3cc5c1ad957054445e798b05c75878e28d46641f6d6fc5ff86dfb8351d4500a5264f5aa7d3f9f778715b8c7517
+  data.tar.gz: 62a7ce7ee17fcfed6011fba1b62fb40795ba7c5b31fc6a7bd2d20f17c9af2c84ffe3c9c3bd56a513b6561f8cb3170fe078c76e1dd6892df22a03d1facc0555e0

data/CHANGELOG.markdown

@@ -1,3 +1,7 @@
+# 2.1.15
+
+* Limit memory usage for very long running requests.
+
 # 2.1.14
 
 * Add TrackedRequest#ignore_request! to entirely ignore and stop capturing a

data/lib/scout_apm.rb

@@ -27,6 +27,8 @@ require 'scout_apm/version'
 
 require 'scout_apm/tracked_request'
 require 'scout_apm/layer'
+require 'scout_apm/limited_layer'
+require 'scout_apm/layer_children_set'
 require 'scout_apm/request_manager'
 require 'scout_apm/call_set'
 

data/lib/scout_apm/layer.rb

@@ -13,13 +13,17 @@ module ScoutApm
     #   instrumentation for an example of how this is useful
     attr_accessor :name
 
-    # An array of children layers, in call order.
+    # An array of children layers
     # For instance, if we are in a middleware, there will likely be only a single
     # child, which is another middleware.  In a Controller, we may have a handful
     # of children: [ActiveRecord, ActiveRecord, View, HTTP Call].
     #
     # This useful to get actual time spent in this layer vs. children time
-    attr_reader :children
+    #
+    # TODO: Check callers for compatibility w/ nil to avoid making an empty array
+    def children
+      @children || LayerChildrenSet.new
+    end
 
     # Time objects recording the start & stop times of this layer
     attr_reader :start_time, :stop_time
@@ -38,6 +42,8 @@ module ScoutApm
     # Known Keys:
     #   :record_count - The number of rows returned by an AR query (From notification instantiation.active_record)
     #   :class_name   - The ActiveRecord class name (From notification instantiation.active_record)
+    #
+    # If no annotations are ever set, this will return nil
     attr_reader :annotations
 
     BACKTRACE_CALLER_LIMIT = 50 # maximum number of lines to send thru for backtrace analysis
@@ -45,15 +51,18 @@ module ScoutApm
     def initialize(type, name, start_time = Time.now)
       @type = type
       @name = name
-      @annotations = {}
       @start_time = start_time
       @allocations_start = ScoutApm::Instruments::Allocations.count
       @allocations_stop = 0
-      @children = [] # In order of calls
+
+      # initialize these only on first use
+      @children = nil
+      @annotations = nil
       @desc = nil
     end
 
     def add_child(child)
+      @children ||= LayerChildrenSet.new
       @children << child
     end
 
@@ -72,6 +81,7 @@ module ScoutApm
 
     # This data is internal to ScoutApm, to add custom information, use the Context api.
     def annotate_layer(hsh)
+      @annotations ||= {}
       @annotations.merge!(hsh)
     end
 
@@ -144,6 +154,7 @@ module ScoutApm
         map { |child| child.total_call_time }.
         inject(0) { |sum, time| sum + time }
     end
+    private :child_time
 
     ######################################
     # Allocation Calculations
@@ -169,5 +180,6 @@ module ScoutApm
         map { |child| child.total_allocations }.
         inject(0) { |sum, obj| sum + obj }
     end
+    private :child_allocations
   end
 end

data/lib/scout_apm/layer_children_set.rb

@@ -0,0 +1,72 @@
+module ScoutApm
+  # A set of children records for any given Layer.  This implements some
+  # rate-limiting logic.
+  #
+  # We store the first `unique_cutoff` count of each layer type. So if cutoff
+  # is 1000, we'd store 1000 HTTP layers, and 1000 ActiveRecord calls, and 1000
+  # of each other layer type. After that, make a LimitedLayer object and store
+  # only aggregate counts and times of future layers of that type. (So the
+  # 1001st an onward of ActiveRecord would get only aggregate times, and
+  # counts, without any detail about the SQL called)
+  #
+  # When the set of children is small, keep them unique
+  # When the set of children gets large enough, stop keeping details
+  #
+  # The next optimization, which is not yet implemented:
+  #   when the set of children gets larger, attempt to merge them without data loss
+  class LayerChildrenSet
+    include Enumerable
+
+    # By default, how many unique children of a type do we store before
+    # flipping over to storing only aggregate info.
+    DEFAULT_UNIQUE_CUTOFF = 2000
+    attr_reader :unique_cutoff
+
+    # The Set of children objects
+    attr_reader :children
+    private :children
+
+    def initialize(unique_cutoff = DEFAULT_UNIQUE_CUTOFF)
+      @children = Hash.new { |hash, key| hash[key] = Set.new }
+      @limited_layers = nil # populated when needed
+      @unique_cutoff = unique_cutoff
+    end
+
+    # Add a new layer into this set
+    # Only add completed layers - otherwise this will collect up incorrect info
+    # into the created LimitedLayer, since it will "freeze" any current data for
+    # total_call_time and similar methods.
+    def <<(child)
+      metric_type = child.type
+      set = children[metric_type]
+
+      if set.size >= unique_cutoff
+        # find limited_layer
+        @limited_layers || init_limited_layers
+        @limited_layers[metric_type].absorb(child)
+      else
+        # we have space just add it
+        set << child
+      end
+    end
+
+    def each
+      children.each do |_type, set|
+        set.each do |child_layer|
+          yield child_layer
+        end
+      end
+
+      if @limited_layers
+        @limited_layers.each do |_type, limited_layer|
+          yield limited_layer
+        end
+      end
+    end
+
+    # hold off initializing this until we know we need it
+    def init_limited_layers
+      @limited_layers ||= Hash.new { |hash, key| hash[key] = LimitedLayer.new(key) }
+    end
+  end
+end

data/lib/scout_apm/layer_converters/converter_base.rb

@@ -177,7 +177,7 @@ module ScoutApm
         meta_options = make_meta_options(layer)
 
         meta = MetricMeta.new(layer.legacy_metric_name, meta_options)
-        meta.extra.merge!(layer.annotations)
+        meta.extra.merge!(layer.annotations) if layer.annotations
 
         store_backtrace(layer, meta)
 
@@ -191,6 +191,11 @@ module ScoutApm
         # allocations
         stat = allocation_metric_hash[meta]
         stat.update!(layer.total_allocations, layer.total_exclusive_allocations)
+
+        if LimitedLayer === layer
+          metric_hash[meta].call_count = layer.count
+          allocation_metric_hash[meta].call_count = layer.count
+        end
       end
 
       # Merged Metric - no specifics, just sum up by type (ActiveRecord, View, HTTP, etc)
@@ -218,7 +223,8 @@ module ScoutApm
       # an example. We start capturing before we know if a query is cached
       # or not, and want to skip any cached queries.
       def skip_layer?(layer)
-        return true if layer.annotations[:ignorable]
+        return false if layer.annotations.nil?
+        return true  if layer.annotations[:ignorable]
       end
     end
   end

data/lib/scout_apm/layer_converters/job_converter.rb

@@ -57,7 +57,7 @@ module ScoutApm
         walker.walk do |layer|
           next if layer == job_layer
           next if layer == queue_layer
-          next if layer.annotations[:ignorable]
+          next if skip_layer?(layer)
 
           # we don't need to use the full metric name for scoped metrics as we
           # only display metrics aggregrated by type, just use "ActiveRecord"

data/lib/scout_apm/layer_converters/metric_converter.rb

@@ -21,7 +21,7 @@ module ScoutApm
         metric_hash = Hash.new
 
         walker.walk do |layer|
-          next if layer.annotations[:ignorable]
+          next if skip_layer?(layer)
 
           meta_options = if layer == scope_layer # We don't scope the controller under itself
                           {}

data/lib/scout_apm/limited_layer.rb

@@ -0,0 +1,122 @@
+module ScoutApm
+  # A LimitedLayer is a lossy-compression approach to fall back on once we max out
+  # the number of detailed layer objects we store.  See LayerChildrenSet for the
+  # logic on when that change over happens
+  #
+  # QUESTION: What do we do if we attempt to merge an item that has children?
+  class LimitedLayer
+    attr_reader :type
+
+    def initialize(type)
+      @type = type
+
+      @total_call_time = 0
+      @total_exclusive_time = 0
+      @total_allocations = 0
+      @total_exclusive_allocations = 0
+      @total_layers = 0
+    end
+
+    def absorb(layer)
+      @total_layers += 1
+
+      @total_call_time += layer.total_call_time
+      @total_exclusive_time += layer.total_exclusive_time
+
+      @total_allocations += layer.total_allocations
+      @total_exclusive_allocations += layer.total_exclusive_allocations
+    end
+
+    def total_call_time
+      @total_call_time
+    end
+
+    def total_exclusive_time
+      @total_exclusive_time
+    end
+
+    def total_allocations
+      @total_allocations
+    end
+
+    def total_exclusive_allocations
+      @total_exclusive_allocations
+    end
+
+    def count
+      @total_layers
+    end
+
+    # This is the old style name. This function is used for now, but should be
+    # removed, and the new type & name split should be enforced through the
+    # app.
+    def legacy_metric_name
+      "#{type}/Limited"
+    end
+
+    def children
+      Set.new
+    end
+
+    def annotations
+      nil
+    end
+
+    def to_s
+      "<LimitedLayer type=#{type} count=#{count}>"
+    end
+
+    ######################################################
+    #  Stub out some methods with static default values  #
+    ######################################################
+    def subscopable?
+      false
+    end
+
+    def desc
+      nil
+    end
+
+    def backtrace
+      nil
+    end
+
+
+    #######################################################################
+    #  Many methods don't make any sense on a limited layer. Raise errors  #
+    #  aggressively for now to detect mistaken calls                      #
+    #######################################################################
+
+    def add_child
+      raise "Should never call add_child on a limited_layer"
+    end
+
+    def record_stop_time!(*)
+      raise "Should never call record_stop_time! on a limited_layer"
+    end
+
+    def record_allocations!
+      raise "Should never call record_allocations! on a limited_layer"
+    end
+
+    def desc=(*)
+      raise "Should never call desc on a limited_layer"
+    end
+
+    def annotate_layer(*)
+      raise "Should never call annotate_layer on a limited_layer"
+    end
+
+    def subscopable!
+      raise "Should never call subscopable! on a limited_layer"
+    end
+
+    def capture_backtrace!
+      raise "Should never call capture_backtrace on a limited_layer"
+    end
+
+    def caller_array
+      raise "Should never call caller_array on a limited_layer"
+    end
+  end
+end

data/lib/scout_apm/tracked_request.rb

@@ -62,7 +62,6 @@ module ScoutApm
       return ignoring_start_layer if ignoring_request?
 
       start_request(layer) unless @root_layer
-      @layers[-1].add_child(layer) if @layers.any?
       @layers.push(layer)
     end
 
@@ -86,6 +85,8 @@ module ScoutApm
       layer.record_stop_time!
       layer.record_allocations!
 
+      @layers[-1].add_child(layer) if @layers.any?
+
       # This must be called before checking if a backtrace should be collected as the call count influences our capture logic.
       # We call `#update_call_counts in stop layer to ensure the layer has a final desc. Layer#desc is updated during the AR instrumentation flow.
       update_call_counts!(layer)

data/lib/scout_apm/version.rb

@@ -1,4 +1,4 @@
 module ScoutApm
-  VERSION = "2.1.14"
+  VERSION = "2.1.15"
 end
 

data/test/unit/layer_children_set_test.rb

@@ -0,0 +1,88 @@
+require 'test_helper'
+require 'scout_apm/layer_children_set'
+
+class LayerChildrenSetTest < Minitest::Test
+  SET = ScoutApm::LayerChildrenSet
+
+  def test_limit_default
+    assert_equal SET::DEFAULT_UNIQUE_CUTOFF, SET.new.unique_cutoff
+  end
+
+  # Add 5, make sure they're all in the children list we get back.
+  def test_add_layer_before_limit
+    s = SET.new(5)
+
+    5.times do
+      s << make_layer("LayerType", "LayerName")
+    end
+
+    children = s.to_a
+    assert_equal 5, children.size
+
+    # Don't care about order
+    (0..4).each do |i|
+      assert children.include?(lookup_layer(i))
+    end
+  end
+
+  def test_add_layer_after_limit
+    s = SET.new(5)
+
+    10.times do
+      s << make_layer("LayerType", "LayerName")
+    end
+
+    children = s.to_a
+    # 6 = 5 real ones + 1 merged.
+    assert_equal 6, children.size
+
+    # Don't care about order
+    (0..4).each do |i|
+      assert children.include?(lookup_layer(i))
+    end
+
+    # Don't care about order
+    (5..9).each do |i|
+      assert ! children.include?(lookup_layer(i))
+    end
+
+    limited_layer = children.last
+    assert_equal ScoutApm::LimitedLayer, limited_layer.class
+    assert_equal 5, limited_layer.count
+  end
+
+  def test_add_layer_with_different_type_after_limit
+    s = SET.new(5)
+
+    # Add 20 items
+    10.times do
+      s << make_layer("LayerType", "LayerName")
+      s << make_layer("DifferentLayerType", "LayerName")
+    end
+
+    children = s.to_a
+
+    # Tyo types, so 2 distinct limitdlayer objects
+    limited_layers = children.select{ |l| ScoutApm::LimitedLayer === l }
+    assert_equal 2, limited_layers.length
+
+    # 5 unchanged children each for the two layer types, plus the 2 limitd when each overran their limit
+    assert_equal 12, children.length
+    limited_layers.each { |ml| assert_equal 5, ml.count }
+  end
+
+  #############
+  #  Helpers  #
+  #############
+
+  def make_layer(type, name)
+    @made_layers ||= []
+    l = ScoutApm::Layer.new(type, name)
+    @made_layers << l
+    l
+  end
+
+  def lookup_layer(i)
+    @made_layers[i]
+  end
+end

data/test/unit/limited_layer_test.rb

@@ -0,0 +1,53 @@
+require 'test_helper'
+require 'ostruct'
+
+class LimitedLayerTest < Minitest::Test
+
+  def test_counts_while_absorbing
+    ll = ScoutApm::LimitedLayer.new("ActiveRecord")
+    assert_equal 0, ll.count
+
+    ll.absorb faux_layer("ActiveRecord", "User#Find", 2, 1, 200, 100)
+    assert_equal 1, ll.count
+
+    ll.absorb faux_layer("ActiveRecord", "User#Find", 2, 1, 200, 100)
+    assert_equal 2, ll.count
+  end
+
+  def test_sums_values_while_absorbing
+    ll = ScoutApm::LimitedLayer.new("ActiveRecord")
+
+    ll.absorb faux_layer("ActiveRecord", "User#Find", 2, 1, 200, 100)
+    assert_equal 1, ll.total_exclusive_time
+    assert_equal 2, ll.total_call_time
+    assert_equal 100, ll.total_exclusive_allocations
+    assert_equal 200, ll.total_allocations
+
+
+    ll.absorb faux_layer("ActiveRecord", "User#Find", 4, 3, 400, 300)
+    assert_equal 4, ll.total_exclusive_time           # 3 + 1
+    assert_equal 6, ll.total_call_time                # 4 + 2
+    assert_equal 400, ll.total_exclusive_allocations  # 300 + 100
+    assert_equal 600, ll.total_allocations            # 400 + 200
+  end
+
+  def test_the_name
+    ll = ScoutApm::LimitedLayer.new("ActiveRecord")
+    assert_equal "ActiveRecord/Limited", ll.legacy_metric_name
+  end
+
+  #############
+  #  Helpers  #
+  #############
+
+  def faux_layer(type, name, tct, tet, a_tct, a_tet)
+    OpenStruct.new(
+      :type => type,
+      :name => name,
+      :total_call_time => tct,
+      :total_exclusive_time => tet,
+      :total_allocations => a_tct,
+      :total_exclusive_allocations => a_tet,
+    )
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: scout_apm
 version: !ruby/object:Gem::Version
-  version: 2.1.14
+  version: 2.1.15
 platform: ruby
 authors:
 - Derek Haynes
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-11-01 00:00:00.000000000 Z
+date: 2016-11-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: minitest
@@ -180,6 +180,7 @@ files:
 - lib/scout_apm/layaway.rb
 - lib/scout_apm/layaway_file.rb
 - lib/scout_apm/layer.rb
+- lib/scout_apm/layer_children_set.rb
 - lib/scout_apm/layer_converters/allocation_metric_converter.rb
 - lib/scout_apm/layer_converters/converter_base.rb
 - lib/scout_apm/layer_converters/depth_first_walker.rb
@@ -189,6 +190,7 @@ files:
 - lib/scout_apm/layer_converters/request_queue_time_converter.rb
 - lib/scout_apm/layer_converters/slow_job_converter.rb
 - lib/scout_apm/layer_converters/slow_request_converter.rb
+- lib/scout_apm/limited_layer.rb
 - lib/scout_apm/metric_meta.rb
 - lib/scout_apm/metric_set.rb
 - lib/scout_apm/metric_stats.rb
@@ -251,6 +253,8 @@ files:
 - test/unit/instruments/net_http_test.rb
 - test/unit/instruments/percentile_sampler_test.rb
 - test/unit/layaway_test.rb
+- test/unit/layer_children_set_test.rb
+- test/unit/limited_layer_test.rb
 - test/unit/metric_set_test.rb
 - test/unit/scored_item_set_test.rb
 - test/unit/serializers/payload_serializer_test.rb
@@ -301,6 +305,8 @@ test_files:
 - test/unit/instruments/net_http_test.rb
 - test/unit/instruments/percentile_sampler_test.rb
 - test/unit/layaway_test.rb
+- test/unit/layer_children_set_test.rb
+- test/unit/limited_layer_test.rb
 - test/unit/metric_set_test.rb
 - test/unit/scored_item_set_test.rb
 - test/unit/serializers/payload_serializer_test.rb