scout_apm 1.2.4.pre → 1.2.4.pre1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: fa64d725755deae00527598cd770bf40a773fcf8
-  data.tar.gz: 17b868c65b2b6bd7749fb14dd7164a773248a26b
+  metadata.gz: f7457a6c1ecec53bd454c6e9ccc2d21fc21d6c61
+  data.tar.gz: 46579e9b5c9e9b948c9828d5552df54275d92d07
 SHA512:
-  metadata.gz: 0ba12e4e11843ef25e3005d51c6e7e9f0d38a3db2a15f3c2e5bce014bc7ceaadf68e33f1b5443535cb938936232800755e5b20f3e21e3183d26598062fc9f018
-  data.tar.gz: eaf8da676edae1147d935594e7ffdfa070bfcbd7fcf3c7bce18c835262a9e98121ae21e5fd916bd391325f4d85f8fcc4975b6f18949b7dac54e246b8d24a9f8d
+  metadata.gz: 915274645119dc9afabe6ce236774a19c41db1464f82f2d393541b29dfe8f27ce3d12bcb0087aca6bfdd5079420772341b7c1a7679b05062982d4ef7df78e462
+  data.tar.gz: b760b06a1cf2b6c9a29d9543af9784888016b324d328dd19ec6d36585d6f0066976018d34939d4966ed2ffbfc77fd9f964c9579dfa12e1a7fefe9397e8dec514

data/CHANGELOG.markdown

@@ -2,6 +2,7 @@
 
 * Removing layaway file validation in main thread
 * Fixing :force so agent will start in tests
+* Rate-limiting slow transactions to 10 per-reporting period
 
 # 1.2.3
 

data/Rakefile

@@ -1,6 +1,8 @@
 require "bundler/gem_tasks"
 
 task :default => :test
+
+desc "Run Unit Tests"
 task :test do
   $: << File.expand_path(File.dirname(__FILE__) + "/test")
   Dir.glob('./test/**/*_test.rb').each { |file| require file }

data/lib/scout_apm.rb

@@ -98,6 +98,7 @@ require 'scout_apm/context'
 require 'scout_apm/stackprof_tree_collapser'
 require 'scout_apm/slow_transaction'
 require 'scout_apm/slow_request_policy'
+require 'scout_apm/slow_transaction_set'
 require 'scout_apm/capacity'
 require 'scout_apm/attribute_arranger'
 

data/lib/scout_apm/agent/reporting.rb

@@ -65,7 +65,15 @@ module ScoutApm
           select { |meta,stats| meta.metric_name =~ /\AController/ }.
           inject(0) {|sum, (_, stat)| sum + stat.call_count }
 
-        logger.info "[#{Time.parse(metadata[:agent_time]).strftime("%H:%M")}] Delivering #{metrics.length} Metrics for #{total_request_count} requests and #{slow_transactions.length} Slow Transaction Traces"
+        memory = metrics.
+          find {|meta,stats| meta.metric_name =~ /\AMemory/ }
+        process_log_str = if memory
+                            "Recorded from #{memory.last.call_count} processes"
+                          else
+                            "Recorded across (unknown) processes"
+                          end
+
+        logger.info "[#{Time.parse(metadata[:agent_time]).strftime("%H:%M")}] Delivering #{metrics.length} Metrics for #{total_request_count} requests and #{slow_transactions.length} Slow Transaction Traces, #{process_log_str}."
         logger.debug("Metrics: #{metrics.pretty_inspect}\nSlowTrans: #{slow_transactions.pretty_inspect}\nMetadata: #{metadata.inspect.pretty_inspect}")
       end
 

data/lib/scout_apm/layer_converter.rb

@@ -86,42 +86,42 @@ module ScoutApm
     end
   end
 
-  # Take a TrackedRequest and turn it into either nil, or a SlowTransaction record
+  # Take a TrackedRequest and turn it into a slow transaction if needed
+  # return a 2 element array, [ Slow Transaction or Nil ,  Hash of metrics to store ]
   class LayerSlowTransactionConverter < LayerConverterBase
     def call
       policy = ScoutApm::Agent.instance.slow_request_policy.capture_type(root_layer.total_call_time)
 
-      if policy == ScoutApm::SlowRequestPolicy::CAPTURE_NONE
-        return nil
+      case policy
+      when ScoutApm::SlowRequestPolicy::CAPTURE_SUMMARY
+        return [nil, {}]
+      when ScoutApm::SlowRequestPolicy::CAPTURE_NONE
+        return [nil, {}]
       end
 
-
       scope = scope_layer
-      return nil unless scope
+      return [nil, {}] unless scope
+
       uri = request.annotations[:uri] || ""
 
-      metrics = case policy
-                when ScoutApm::SlowRequestPolicy::CAPTURE_SUMMARY
-                  {}
-                when ScoutApm::SlowRequestPolicy::CAPTURE_DETAIL
-                  create_metrics
-                end
-
-      stackprof = case policy
-                  when ScoutApm::SlowRequestPolicy::CAPTURE_SUMMARY
-                    []
-                  when ScoutApm::SlowRequestPolicy::CAPTURE_DETAIL
-                    # Disable stackprof output for now
-                    [] # request.stackprof
-                  end
-
-      SlowTransaction.new(uri,
-                          scope.legacy_metric_name,
-                          root_layer.total_call_time,
-                          metrics,
-                          request.context,
-                          root_layer.stop_time,
-                          stackprof)
+      metrics = create_metrics
+      # Disable stackprof output for now
+      stackprof = [] # request.stackprof
+
+      meta = MetricMeta.new("SlowTransaction/#{scope_layer.legacy_metric_name}")
+      stat = MetricStats.new
+      stat.update!(1)
+
+      [
+        SlowTransaction.new(uri,
+                            scope.legacy_metric_name,
+                            root_layer.total_call_time,
+                            metrics,
+                            request.context,
+                            root_layer.stop_time,
+                            stackprof),
+        { meta => stat }
+      ]
 
     end
 

data/lib/scout_apm/slow_request_policy.rb

@@ -1,9 +1,8 @@
-# Long running class that provides a yes/no answer to see if a transaction counts as a "slow transaction"
+# Long running class that determines if, and in how much detail a potentially
+# slow transaction should be recorded in
 #
 # Rules:
 #   - Runtime must be slower than a threshold
-#   - Log detailed metrics only for the first X
-#   - Stop logging anything after a maximum
 
 module ScoutApm
   class SlowRequestPolicy
@@ -16,68 +15,15 @@ module ScoutApm
     # It's not slow unless it's at least this slow
     SLOW_REQUEST_TIME_THRESHOLD = 2.0 # seconds
 
-    # Stop recording detailed metrics after this count.  Still record the fact
-    # a slow request happened though
-    MAX_DETAIL_PER_MINUTE = 10
-
-    # Stop recording anything after this number of slow transactions in a
-    # minute.  Will also log a message once per minute that it is stopping
-    # recording.
-    MAX_PER_MINUTE = 500
-
-    def initialize
-      @minute_count = 0
-      @detailed_count = 0 # How many detailed slow transactions have we captured this minute?
-      @minute = Time.now.min
-      @clipped_recording = false
-    end
-
     def capture_type(time)
-      reset_counters
-
       return CAPTURE_NONE unless slow_enough?(time)
-      return CAPTURE_NONE if clip_recording?
-
-      @minute_count += 1
-
-      if @detailed_count < MAX_DETAIL_PER_MINUTE
-        @detailed_count += 1
-        return CAPTURE_DETAIL
-      else
-        return CAPTURE_SUMMARY
-      end
-
+      return CAPTURE_DETAIL
     end
 
     private
 
-    def reset_counters
-      t = Time.now.min
-      return if t == @minute
-
-      @minute_count = 0
-      @detailed_count = 0
-      @minute = t
-      @clipped_recording = false
-    end
-
     def slow_enough?(time)
       time > SLOW_REQUEST_TIME_THRESHOLD
     end
-
-    # Breaker for rapid-fire excessive slow requests.
-    # If we trip this breaker continually, it will log once per minute that it is broken
-    def clip_recording?
-      if @minute_count > MAX_PER_MINUTE
-        if !@clipped_recording
-          ScoutApm::Agent.instance.logger.info("Skipping future slow requests this minute, reached limit of #{MAX_PER_MINUTE}")
-        end
-
-        @clipped_recording = true
-        true
-      else
-        false
-      end
-    end
   end
 end

data/lib/scout_apm/slow_transaction_set.rb

@@ -0,0 +1,67 @@
+# In order to keep load down, only record a sample of Slow Transactions.  In
+# order to make that sampling as fair as possible, follow a basic algorithm:
+#
+# When adding a new SlowTransaction:
+#  * Just add it if there is an open spot
+#  * If there isn't an open spot, attempt to remove an over-represented
+#    endpoint instead ("attempt_to_evict"). Overrepresented is simply "has more
+#    than @fair number of SlowTransactions for that end point"
+#  * If there isn't an open spot, and nobody is valid to evict, drop the
+#    incoming SlowTransaction without adding.
+#
+# There is no way to remove SlowTransactions from this set, create a new object
+# for each reporting period.
+module ScoutApm
+  class SlowTransactionSet
+    include Enumerable
+
+    DEFAULT_TOTAL = 10
+    DEFAULT_FAIR = 1
+
+    attr_reader :total, :fair
+
+    def initialize(total=DEFAULT_TOTAL, fair=DEFAULT_FAIR)
+      @total = total
+      @fair = fair
+      @slow_transactions = []
+    end
+
+    def each
+      @slow_transactions.each { |s| yield s }
+    end
+
+    def <<(slow_transaction)
+      return if attempt_append(slow_transaction)
+      attempt_to_evict
+      attempt_append(slow_transaction)
+    end
+
+    def empty_slot?
+      @slow_transactions.length < total
+    end
+
+    def attempt_append(slow_transaction)
+      if empty_slot?
+        @slow_transactions.push(slow_transaction)
+        true
+      else
+        false
+      end
+    end
+
+    def attempt_to_evict
+      return if @slow_transactions.length == 0
+
+      overrepresented = @slow_transactions.
+        group_by { |st| st.metric_name }.
+        to_a.
+        sort_by { |(_, sts)| sts.length }.
+        last
+
+      if overrepresented[1].length > fair
+        fastest = overrepresented[1].sort_by { |st| st.total_call_time }.first
+        @slow_transactions.delete(fastest)
+      end
+    end
+  end
+end

data/lib/scout_apm/stackprof_tree_collapser.rb

@@ -11,7 +11,8 @@ module ScoutApm
         begin
           ScoutApm::Agent.instance.logger.debug("StackProf - Samples: #{raw_stackprof[:samples]}, GC: #{raw_stackprof[:gc_samples]}, missed: #{raw_stackprof[:missed_samples]}, Interval: #{raw_stackprof[:interval]}")
         rescue
-          ScoutApm::Agent.instance.logger.debug("StackProf Raw - #{raw_stackprof.inspect}")
+          # Not a useful log message, currently stackprof is disabled
+          # ScoutApm::Agent.instance.logger.debug("StackProf Raw - #{raw_stackprof.inspect}")
         end
       end
     end

data/lib/scout_apm/store.rb

@@ -31,6 +31,7 @@ module ScoutApm
 
     # Save a new slow transaction
     def track_slow_transaction!(slow_transaction)
+      return unless slow_transaction
       @mutex.synchronize {
         reporting_periods[current_timestamp].merge_slow_transactions!(slow_transaction)
       }
@@ -91,7 +92,7 @@ module ScoutApm
     def initialize(timestamp)
       @timestamp = timestamp
 
-      @slow_transactions = Array.new
+      @slow_transactions = SlowTransactionSet.new
       @aggregate_metrics = Hash.new
     end
 
@@ -103,9 +104,11 @@ module ScoutApm
       self
     end
 
-    def merge_slow_transactions!(slow_transactions)
-      @slow_transactions += Array(slow_transactions)
-      trim_slow_transaction_metrics
+    def merge_slow_transactions!(new_transactions)
+      Array(new_transactions).each do |one_transaction|
+        @slow_transactions << one_transaction
+      end
+
       self
     end
 
@@ -117,7 +120,7 @@ module ScoutApm
     end
 
     def slow_transactions_payload
-      @slow_transactions
+      @slow_transactions.to_a
     end
 
     private
@@ -139,7 +142,7 @@ module ScoutApm
 
     # We can't aggregate CPU, Memory, Capacity, or Controller, so pass through these metrics directly
     # TODO: Figure out a way to not have this duplicate what's in Samplers, and also on server's ingest
-    PASSTHROUGH_METRICS = ["CPU", "Memory", "Instance", "Controller"]
+    PASSTHROUGH_METRICS = ["CPU", "Memory", "Instance", "Controller", "SlowTransaction"]
 
     # Absorbs a single new metric into the aggregates
     def absorb(metric)

data/lib/scout_apm/tracked_request.rb

@@ -130,8 +130,9 @@ module ScoutApm
       metrics = LayerMetricConverter.new(self).call
       ScoutApm::Agent.instance.store.track!(metrics)
 
-      slow = LayerSlowTransactionConverter.new(self).call
+      slow, slow_metrics = LayerSlowTransactionConverter.new(self).call
       ScoutApm::Agent.instance.store.track_slow_transaction!(slow)
+      ScoutApm::Agent.instance.store.track!(slow_metrics)
 
       error_metrics = LayerErrorConverter.new(self).call
       ScoutApm::Agent.instance.store.track!(error_metrics)

data/lib/scout_apm/version.rb

@@ -1,4 +1,4 @@
 module ScoutApm
-  VERSION = "1.2.4.pre"
+  VERSION = "1.2.4.pre1"
 end
 

data/test/unit/layaway_test.rb

@@ -13,7 +13,7 @@ class LayawayTest < Minitest::Test
     data = ScoutApm::Layaway.new
     t = ScoutApm::StoreReportingPeriodTimestamp.new
     data.add_reporting_period(t,ScoutApm::StoreReportingPeriod.new(t))
-    assert_equal [TIMESTAMP,t], Marshal.load(File.read(DATA_FILE_PATH)).keys
+    assert_equal [TIMESTAMP,t].sort_by(&:timestamp), Marshal.load(File.read(DATA_FILE_PATH)).keys.sort_by(&:timestamp)
   end
 
   def test_merge_reporting_period

data/test/unit/slow_transaction_set_test.rb

@@ -0,0 +1,93 @@
+require 'test_helper'
+
+require 'scout_apm/slow_transaction_set'
+require 'scout_apm/slow_transaction'
+
+class SlowTransactionSetTest < Minitest::Test
+  def test_adding_to_empty_set
+    set = ScoutApm::SlowTransactionSet.new(3, 1)
+    set << make_slow("Controller/Foo")
+    assert_equal 1, set.count
+  end
+
+  def test_adding_to_partially_full_set
+    set = ScoutApm::SlowTransactionSet.new(3, 1)
+    set << make_slow("Controller/Foo")
+    set << make_slow("Controller/Foo")
+    assert_equal 2, set.count
+  end
+
+  def test_overflow_of_one_type
+    max_size = 3
+    set = ScoutApm::SlowTransactionSet.new(max_size, 1)
+    set << make_slow("Controller/Foo")
+    set << make_slow("Controller/Foo")
+    set << make_slow("Controller/Foo")
+    set << make_slow("Controller/Foo")
+    set << make_slow("Controller/Foo")
+    set << make_slow("Controller/Foo")
+    assert_equal max_size, set.count
+  end
+
+  def test_eviction_of_overrepresented
+    max_size = 3
+    set = ScoutApm::SlowTransactionSet.new(max_size, 1)
+    set << make_slow("Controller/Foo")
+    set << make_slow("Controller/Foo")
+    set << make_slow("Controller/Foo")
+    set << make_slow("Controller/Foo")
+    set << make_slow("Controller/Foo")
+    set << make_slow("Controller/Bar")
+
+    # 3 total
+    assert_equal max_size, set.count
+    assert_equal 1, set.select{|sl| sl.metric_name == "Controller/Bar"}.length
+    assert_equal 2, set.select{|sl| sl.metric_name == "Controller/Foo"}.length
+  end
+
+  # Fill the set with /Foo records, then add a /Bar to evict. Check that the
+  # evicted one was the fastest of the Foos
+  def test_eviction_of_fastest
+    max_size = 3
+    set = ScoutApm::SlowTransactionSet.new(max_size, 1)
+
+    [1,2,3].shuffle.each do |seconds| # Shuffle to remove any assumptions on order
+      set << make_slow("Controller/Foo", seconds)
+    end
+    set << make_slow("Controller/Bar", 8)
+
+    # The foo taking 1 second should be evicted
+    assert_equal 2, set.select{|sl| sl.metric_name == "Controller/Foo"}.map{ |sl| sl.total_call_time}.min
+  end
+
+  def test_eviction_when_no_overrepresented
+    max_size = 4
+    fair = 2
+    set = ScoutApm::SlowTransactionSet.new(max_size, fair)
+
+    # Full, but each is at fair level
+    set << make_slow("Controller/Bar")
+    set << make_slow("Controller/Bar")
+    set << make_slow("Controller/Foo")
+    set << make_slow("Controller/Foo")
+
+    set << make_slow("Controller/Quux")
+    assert_equal max_size, set.count
+    assert_equal 0, set.select{|sl| sl.metric_name == "Controller/Quux" }.length
+  end
+
+  ##############
+  #### Helpers
+  ##############
+
+  def make_slow(metric, time=5)
+    ScoutApm::SlowTransaction.new(
+      "http://foo.app/#{metric}",
+      metric,
+      time,
+      {}, # metrics
+      {}, # context
+      Time.now, # end time
+      []) # stackprof
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: scout_apm
 version: !ruby/object:Gem::Version
-  version: 1.2.4.pre
+  version: 1.2.4.pre1
 platform: ruby
 authors:
 - Derek Haynes
@@ -127,6 +127,7 @@ files:
 - lib/scout_apm/server_integrations/webrick.rb
 - lib/scout_apm/slow_request_policy.rb
 - lib/scout_apm/slow_transaction.rb
+- lib/scout_apm/slow_transaction_set.rb
 - lib/scout_apm/stack_item.rb
 - lib/scout_apm/stackprof_tree_collapser.rb
 - lib/scout_apm/store.rb
@@ -150,6 +151,7 @@ files:
 - test/unit/instruments/active_record_instruments_test.rb
 - test/unit/layaway_test.rb
 - test/unit/serializers/payload_serializer_test.rb
+- test/unit/slow_transaction_set_test.rb
 - test/unit/sql_sanitizer_test.rb
 homepage: https://github.com/scoutapp/scout_apm_ruby
 licenses: []
@@ -171,7 +173,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: 1.3.1
 requirements: []
 rubyforge_project: scout_apm
-rubygems_version: 2.4.8
+rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
 summary: Ruby application performance monitoring
@@ -184,4 +186,5 @@ test_files:
 - test/unit/instruments/active_record_instruments_test.rb
 - test/unit/layaway_test.rb
 - test/unit/serializers/payload_serializer_test.rb
+- test/unit/slow_transaction_set_test.rb
 - test/unit/sql_sanitizer_test.rb