scout_apm 2.0.0.pre2 → 2.0.0.pre3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 119d6658eaefd7e3b177c1e982006f0d59dd208b
-  data.tar.gz: 5585bca449804241d8778a1ff716444c50ac86d8
+  metadata.gz: d49313e6479d1530b66fcc237fbe105646dd33d8
+  data.tar.gz: bfc290ad58b83f3c3509cc81ffb14d9d08ab1237
 SHA512:
-  metadata.gz: 5e6ad957c5948b239d530e9cc4e35445a37abb1331ef3c20357371a1c5da9286e4b9d38e84df2aecedd23a39e1a5d0163d76dd76549e41c700b78e15f770338d
-  data.tar.gz: 060b1d7a09e6e5c37cd1a087bd7bf13faaab3e3b5b3aaf8d7f4173a50fa97c76d39c4ccaf299b05cdd48232a6b7638caeab88674110072a26692ad77fad2533e
+  metadata.gz: 420825d2d15e709e0d07209ff6d32c03fb51400123b31b880de67dd1c1890e4ab23421a05830d1821569db135aa4f30ab9e1b9b82e1112b713d0b09a797c8f5b
+  data.tar.gz: 21d53aadf8fbb4584e5c5a75437f6b1433f28d7792c4715435b49f7f0170b619fc38246bae2e8b64b3e6acd546c57b7eead417d8ddafaafa1a34abd3c2f8f777

data/CHANGELOG.markdown

@@ -8,6 +8,15 @@
   * seconds_since_startup (larger memory increases and other other odd behavior more common when close to startup)
 * Initial support for instant traces
 * Collect 95th percentiles
+* Remove unused & old references to Stackprof
+
+# 1.6.0
+
+* Dynamic algorithm for selecting when to collect traces. Now, we will collect a
+  more complete cross-section of your application's performance, dynamically
+  tuned as your application runs.
+* Record and report 95th percentiles for each action
+* A variety of bug fixes
 
 # 1.5.5
 

data/lib/scout_apm/agent.rb

@@ -24,8 +24,9 @@ module ScoutApm
     # Histogram of the cumulative requests since the start of the process
     attr_reader :request_histograms
 
-    # Histogram of the requests since last reset. Reset by the sampler, so once per minutes.
-    attr_reader :request_histograms_resettable
+    # Histogram of the requests, distinct by reporting period (minute)
+    # { StoreReportingPeriodTimestamp => RequestHistograms }
+    attr_reader :request_histograms_by_time
 
     # All access to the agent is thru this class method to ensure multiple Agent instances are not initialized per-Ruby process.
     def self.instance(options = {})
@@ -50,7 +51,7 @@ module ScoutApm
       @slow_request_policy = ScoutApm::SlowRequestPolicy.new
       @slow_job_policy = ScoutApm::SlowJobPolicy.new
       @request_histograms = ScoutApm::RequestHistograms.new
-      @request_histograms_resettable = ScoutApm::RequestHistograms.new
+      @request_histograms_by_time = Hash.new { |h, k| h[k] = ScoutApm::RequestHistograms.new }
 
       @store          = ScoutApm::Store.new
       @layaway        = ScoutApm::Layaway.new
@@ -124,11 +125,10 @@ module ScoutApm
 
       load_instruments if should_load_instruments?(options)
 
-      @samplers = [
-        ScoutApm::Instruments::Process::ProcessCpu.new(environment.processors, logger),
+      [ ScoutApm::Instruments::Process::ProcessCpu.new(environment.processors, logger),
         ScoutApm::Instruments::Process::ProcessMemory.new(logger),
         ScoutApm::Instruments::PercentileSampler.new(logger, 95),
-      ]
+      ].each { |s| store.add_sampler(s) }
 
       app_server_load_hook
 
@@ -242,16 +242,19 @@ module ScoutApm
       @background_worker = ScoutApm::BackgroundWorker.new
       @background_worker_thread = Thread.new do
         @background_worker.start {
-          # First, run periodic samplers. These should run once a minute,
-          # rather than per-request. "CPU Load" and similar.
-          run_samplers
-          capacity.process
-
           ScoutApm::Agent.instance.process_metrics
+          clean_old_percentiles
         }
       end
     end
 
+    def clean_old_percentiles
+      request_histograms_by_time.
+        keys.
+        select {|timestamp| timestamp.age_in_seconds > 60 * 10 }.
+        each {|old_timestamp| request_histograms_by_time.delete(old_timestamp) }
+    end
+
     # If we want to skip the app_server_check, then we must load it.
     def should_load_instruments?(options={})
       return true if options[:skip_app_server_check]
@@ -284,10 +287,6 @@ module ScoutApm
       install_instrument(ScoutApm::Instruments::Redis)
       install_instrument(ScoutApm::Instruments::InfluxDB)
       install_instrument(ScoutApm::Instruments::Elasticsearch)
-
-      if StackProf.respond_to?(:fake?) && StackProf.fake?
-        logger.info 'StackProf not found - add `gem "stackprof"` to your Gemfile to enable advanced code profiling (only for Ruby 2.1+)'
-      end
     rescue
       logger.warn "Exception loading instruments:"
       logger.warn $!.message
@@ -314,24 +313,6 @@ module ScoutApm
       environment.deploy_integration
     end
 
-    # TODO: Extract a proper class / registery for these. They don't really belong here
-    def run_samplers
-      @samplers.each do |sampler|
-        begin
-          if sampler.respond_to? :metrics
-            store.track!(sampler.metrics)
-          else
-            result = sampler.run
-            store.track_one!(sampler.metric_type, sampler.metric_name, result) if result
-          end
-        rescue => e
-          logger.info "Error reading #{sampler.human_name}"
-          logger.debug e.message
-          logger.debug e.backtrace.join("\n")
-        end
-      end
-    end
-
     def app_server_missing?(options = {})
       !environment.app_server_integration(true).found? && !options[:skip_app_server_check]
     end

data/lib/scout_apm/config.rb

@@ -68,7 +68,6 @@ module ScoutApm
         'host'                   => 'https://checkin.scoutapp.com',
         'direct_host'            => 'https://apm.scoutapp.com',
         'log_level'              => 'info',
-        'stackprof_interval'     => 20000, # microseconds, 1000 = 1 millisecond, so 20k == 20 milliseconds
         'uri_reporting'          => 'full_path',
         'report_format'          => 'json',
         'disabled_instruments'   => [],

data/lib/scout_apm/instruments/percentile_sampler.rb

@@ -14,25 +14,24 @@ module ScoutApm
         "Percentiles"
       end
 
-      def metrics
+      # Gets the 95th%ile for the time requested
+      def metrics(time)
         ms = {}
-
-        ScoutApm::Agent.instance.request_histograms_resettable.each_name do |name|
+        histos = ScoutApm::Agent.instance.request_histograms_by_time[time]
+        histos.each_name do |name|
           percentiles.each do |percentile|
             meta = MetricMeta.new("Percentile/#{percentile}/#{name}")
             stat = MetricStats.new
-            stat.update!(ScoutApm::Agent.instance.request_histograms_resettable.quantile(name, percentile))
+            stat.update!(histos.quantile(name, percentile))
             ms[meta] = stat
           end
         end
 
-        # Wipe the histograms, get ready for the next minute's worth of data.
-        ScoutApm::Agent.instance.request_histograms_resettable.reset_all!
+        # Wipe the histograms we just collected data on
+        ScoutApm::Agent.instance.request_histograms_by_time.delete(time)
 
         ms
       end
-
-      private
     end
   end
 end

data/lib/scout_apm/instruments/process/process_cpu.rb

@@ -29,6 +29,18 @@ module ScoutApm
           "Process CPU"
         end
 
+        def metrics(_time)
+          result = run
+          if result
+            meta = MetricMeta.new("#{metric_type}/#{metric_name}")
+            stat = MetricStats.new(false)
+            stat.update!(result)
+            { meta => stat }
+          else
+            {}
+          end
+        end
+
         # TODO: Figure out a good default instead of nil
         def run
           res = nil

data/lib/scout_apm/instruments/process/process_memory.rb

@@ -33,6 +33,18 @@ module ScoutApm
           "Process Memory"
         end
 
+        def metrics(_time)
+          result = run
+          if result
+            meta = MetricMeta.new("#{metric_type}/#{metric_name}")
+            stat = MetricStats.new(false)
+            stat.update!(result)
+            { meta => stat }
+          else
+            {}
+          end
+        end
+
         def run
           self.class.rss_in_mb.tap { |res| logger.debug "#{human_name}: #{res.inspect}" }      
         end

data/lib/scout_apm/layer_converters/slow_request_converter.rb

@@ -46,9 +46,6 @@ module ScoutApm
           end
         end
 
-        # Disable stackprof output for now
-        stackprof = [] # request.stackprof
-
         SlowTransaction.new(uri,
                             scope.legacy_metric_name,
                             root_layer.total_call_time,
@@ -56,7 +53,7 @@ module ScoutApm
                             allocation_metrics,
                             request.context,
                             root_layer.stop_time,
-                            stackprof,
+                            [], # stackprof, now unused.
                             mem_delta,
                             root_layer.total_allocations,
                             @points)

data/lib/scout_apm/slow_transaction.rb

@@ -11,7 +11,6 @@ module ScoutApm
     attr_reader :context
     attr_reader :time
     attr_reader :prof
-    attr_reader :raw_prof
     attr_reader :mem_delta
     attr_reader :allocations
     attr_accessor :hostname # hack - we need to reset these server side.
@@ -25,9 +24,7 @@ module ScoutApm
       @allocation_metrics = allocation_metrics
       @context = context
       @time = time
-      @prof = ScoutApm::StackprofTreeCollapser.new(raw_stackprof).call
-      @raw_prof = raw_stackprof # Send whole data up to server
-
+      @prof = []
       @mem_delta = mem_delta
       @allocations = allocations
       @seconds_since_startup = (Time.now - ScoutApm::Agent.instance.process_start_time)

data/lib/scout_apm/store.rb

@@ -6,9 +6,13 @@ module ScoutApm
     # A hash of reporting periods. { StoreReportingPeriodTimestamp => StoreReportingPeriod }
     attr_reader :reporting_periods
 
+    # Used to pull metrics into each reporting period, as that reporting period is finished.
+    attr_reader :samplers
+
     def initialize
       @mutex = Mutex.new
       @reporting_periods = Hash.new { |h,k| h[k] = StoreReportingPeriod.new(k) }
+      @samplers = []
     end
 
     def current_timestamp
@@ -66,12 +70,32 @@ module ScoutApm
       @mutex.synchronize {
         reporting_periods.select { |time, rp| force || time.timestamp < current_timestamp.timestamp}.
                           each   { |time, rp|
+                                   collect_samplers(rp)
                                    layaway.add_reporting_period(time, rp)
                                    reporting_periods.delete(time)
                                  }
       }
       ScoutApm::Agent.instance.logger.debug("Finished writing to layaway")
     end
+
+    ######################################
+    # Sampler support
+    def add_sampler(sampler)
+      @samplers << sampler
+    end
+
+    def collect_samplers(rp)
+      @samplers.each do |sampler|
+        begin
+          metrics = sampler.metrics(rp.timestamp)
+          rp.absorb_metrics!(metrics)
+        rescue => e
+          ScoutApm::Agent.instance.logger.info "Error reading #{sampler.human_name} for period: #{rp}"
+          ScoutApm::Agent.instance.logger.debug e.message
+          ScoutApm::Agent.instance.logger.debug e.backtrace.join("\n")
+        end
+      end
+    end
   end
 
   # A timestamp, normalized to the beginning of a minute. Used as a hash key to

data/lib/scout_apm/tracked_request.rb

@@ -22,10 +22,6 @@ module ScoutApm
     #   :queue_latency - how long a background Job spent in the queue before starting processing
     attr_reader :annotations
 
-    # Nil until the request is finalized, at which point it will hold the
-    # entire raw stackprof output for this request
-    attr_reader :stackprof
-
     # Headers as recorded by rails
     # Can be nil if we never reach a Rails Controller
     attr_reader :headers
@@ -52,7 +48,6 @@ module ScoutApm
       @ignoring_children = false
       @context = Context.new
       @root_layer = nil
-      @stackprof = nil
       @error = false
       @instant_key = nil
       @mem_start = mem_usage
@@ -158,21 +153,14 @@ module ScoutApm
     # Run at the beginning of the whole request
     #
     # * Capture the first layer as the root_layer
-    # * Start Stackprof (disabling to avoid conflicts if stackprof is included as middleware since we aren't sending this up to server now)
     def start_request(layer)
       @root_layer = layer unless @root_layer # capture root layer
-      #StackProf.start(:mode => :wall, :interval => ScoutApm::Agent.instance.config.value("stackprof_interval"))
     end
 
     # Run at the end of the whole request
     #
-    # * Collect stackprof info
     # * Send the request off to be stored
     def stop_request
-      # ScoutApm::Agent.instance.logger.debug("stop_request: #{annotations[:uri]}" )
-      #StackProf.stop # disabling to avoid conflicts if stackprof is included as middleware since we aren't sending this up to server now
-      #@stackprof = StackProf.results
-
       record!
     end
 
@@ -234,7 +222,8 @@ module ScoutApm
       # Update immediate and long-term histograms for both job and web requests
       if unique_name != :unknown
         ScoutApm::Agent.instance.request_histograms.add(unique_name, root_layer.total_call_time)
-        ScoutApm::Agent.instance.request_histograms_resettable.add(unique_name, root_layer.total_call_time)
+        ScoutApm::Agent.instance.request_histograms_by_time[ScoutApm::Agent.instance.store.current_timestamp].
+          add(unique_name, root_layer.total_call_time)
       end
 
       metrics = LayerConverters::MetricConverter.new(self).call

data/lib/scout_apm/version.rb

@@ -1,4 +1,4 @@
 module ScoutApm
-  VERSION = "2.0.0.pre2"
+  VERSION = "2.0.0.pre3"
 end
 

data/lib/scout_apm.rb

@@ -18,11 +18,6 @@ require 'yaml'
 #####################################
 # Gem Requires
 #####################################
-begin
-  require 'stackprof'
-rescue LoadError
-  require 'scout_apm/utils/fake_stack_prof'
-end
 require 'rusage'
 
 #####################################
@@ -116,7 +111,6 @@ require 'scout_apm/metric_set'
 require 'scout_apm/store'
 require 'scout_apm/tracer'
 require 'scout_apm/context'
-require 'scout_apm/stackprof_tree_collapser'
 require 'scout_apm/instant_reporting'
 
 require 'scout_apm/metric_meta'

data/test/unit/serializers/payload_serializer_test.rb

@@ -6,7 +6,6 @@ require 'scout_apm/serializers/payload_serializer_to_json'
 require 'scout_apm/slow_transaction'
 require 'scout_apm/metric_meta'
 require 'scout_apm/metric_stats'
-require 'scout_apm/stackprof_tree_collapser'
 require 'scout_apm/utils/fake_stack_prof'
 require 'scout_apm/context'
 require 'ostruct'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: scout_apm
 version: !ruby/object:Gem::Version
-  version: 2.0.0.pre2
+  version: 2.0.0.pre3
 platform: ruby
 authors:
 - Derek Haynes
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-06-01 00:00:00.000000000 Z
+date: 2016-06-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rusage
@@ -200,13 +200,11 @@ files:
 - lib/scout_apm/slow_request_policy.rb
 - lib/scout_apm/slow_transaction.rb
 - lib/scout_apm/stack_item.rb
-- lib/scout_apm/stackprof_tree_collapser.rb
 - lib/scout_apm/store.rb
 - lib/scout_apm/tracer.rb
 - lib/scout_apm/tracked_request.rb
 - lib/scout_apm/utils/active_record_metric_name.rb
 - lib/scout_apm/utils/backtrace_parser.rb
-- lib/scout_apm/utils/fake_stack_prof.rb
 - lib/scout_apm/utils/installed_gems.rb
 - lib/scout_apm/utils/klass_helper.rb
 - lib/scout_apm/utils/null_logger.rb

data/lib/scout_apm/stackprof_tree_collapser.rb

@@ -1,103 +0,0 @@
-module ScoutApm
-  class StackprofTreeCollapser
-    attr_reader :raw_stackprof
-    attr_reader :nodes
-
-    def initialize(raw_stackprof)
-      @raw_stackprof = raw_stackprof
-
-      # Log the raw stackprof info
-      #unless StackProf.respond_to?(:fake?) && StackProf.fake?
-      #  begin
-      #    ScoutApm::Agent.instance.logger.debug("StackProf - Samples: #{raw_stackprof[:samples]}, GC: #{raw_stackprof[:gc_samples]}, missed: #{raw_stackprof[:missed_samples]}, Interval: #{raw_stackprof[:interval]}")
-      #  rescue
-      #    ScoutApm::Agent.instance.logger.debug("StackProf Raw - #{raw_stackprof.inspect}")
-      #  end
-      #end
-    end
-
-    def call
-      build_tree
-      connect_children
-      total_samples_of_app_nodes
-    rescue
-      []
-    end
-
-    private
-
-    def build_tree
-      @nodes = raw_stackprof[:frames].map do |(frame_id, frame_data)|
-        TreeNode.new(frame_id,                     # frame_id
-                     frame_data[:name],            # name
-                     frame_data[:file],            # file
-                     frame_data[:line],            # line
-                     frame_data[:samples],         # samples
-                     frame_data[:total_samples],   # total_samples
-                     (frame_data[:edges] || {}),   # children_edges [ { id => weight } ]
-                     [],                           # children [ treenode, ... ]
-                     []                            # parents [ [treenode, int (weight) ], [...] ]
-                    )
-      end
-    end
-
-    def connect_children
-      nodes.each do |node|
-        children = nodes.find_all { |n| node.children_edges.keys.include? n.frame_id }
-
-        node.children_edges.each do |(frame_id, weight)|
-          child = children.detect{ |c| c.frame_id == frame_id }
-          child.parents << [node, weight]
-        end
-
-        node.children = children
-      end
-    end
-
-    def in_app_nodes
-      nodes.select {|n| n.app? }
-    end
-
-    def total_samples_of_app_nodes
-      in_app_nodes.reject{|n| n.calls_only_app_nodes? && !n.has_samples? }.
-        map{|n| { :samples => n.total_samples,
-                  :name => n.name,
-                  :file => n.file,
-                  :line => n.line
-                }
-        }
-    end
-
-    ###########################################
-    # TreeNode class represents a single node.
-    ###########################################
-    TreeNode = Struct.new(:frame_id, :name, :file, :line, :samples, :total_samples,
-                          :children_edges, :children, :parents) do
-      def app?
-        @is_app ||= file =~ /^#{ScoutApm::Environment.instance.root}/
-      end
-
-      # Force object_id to be the equality mechanism, rather than struct's
-      # default which delegates to == on each value.  That is wrong because
-      # we want to be able to dup a node in the tree construction process and
-      # not have those compare equal to each other.
-      def ==(other)
-        object_id == other.object_id
-      end
-
-      def inspect
-        "#{frame_id}: #{name} - ##{samples}\n" +
-        "  Parents: #{parents.map{ |(p, w)| "#{p.name}: #{w}"}.join("\n           ") }\n" +
-        "  Children: #{children_edges.inspect} \n"
-      end
-
-      def calls_only_app_nodes?
-        children.all?(&:app?)
-      end
-
-      def has_samples?
-        samples > 0
-      end
-    end
-  end
-end

data/lib/scout_apm/utils/fake_stack_prof.rb

@@ -1,40 +0,0 @@
-# A fake implementation of stackprof, for systems that don't support it.
-module StackProf
-  def self.start(*args)
-    @running = true
-  end
-
-  def self.stop(*args)
-    @running = false
-  end
-
-  def self.running?
-    !!@running
-  end
-
-  def self.run(*args)
-    start
-    yield
-    stop
-    results
-  end
-
-  def self.sample(*args)
-  end
-
-  def self.results(*args)
-    {
-      :version => 0.0,
-      :mode => :wall,
-      :interval => 1000,
-      :samples => 0,
-      :gc_samples => 0,
-      :missed_samples => 0,
-      :frames => {},
-    }
-  end
-
-  def self.fake?
-    true
-  end
-end