scout_apm 1.6.8 → 2.0.0.pre

data/lib/scout_apm/slow_request_policy.rb

@@ -1,5 +1,8 @@
 # Long running class that determines if, and in how much detail a potentially
 # slow transaction should be recorded in
+#
+# Rules:
+#   - Runtime must be slower than a threshold
 
 module ScoutApm
   class SlowRequestPolicy
@@ -8,92 +11,21 @@ module ScoutApm
       CAPTURE_NONE    = "capture_none",
     ]
 
-    # Adjust speed points. See the function
-    POINT_MULTIPLIER_SPEED = 0.25
+    # It's not slow unless it's at least this slow
+    SLOW_REQUEST_TIME_THRESHOLD = 2.0 # seconds
 
-    # For each minute we haven't seen an endpoint
-    POINT_MULTIPLIER_AGE = 0.25
-
-    # Outliers are worth up to "1000ms" of weight
-    POINT_MULTIPLIER_PERCENTILE = 1.0
-
-    # A hash of Endpoint Name to the last time we stored a slow transaction for it.
-    #
-    # Defaults to a start time that is pretty close to application boot time.
-    # So the "age" of an endpoint we've never seen is the time the application
-    # has been running.
-    attr_reader :last_seen
-
-
-    def initialize
-      zero_time = Time.now
-      @last_seen = Hash.new { |h, k| h[k] = zero_time }
-    end
-
-    def stored!(request)
-      last_seen[unique_name_for(request)] = Time.now
-    end
-
-    # Determine if this request trace should be fully analyzed by scoring it
-    # across several metrics, and then determining if that's good enough to
-    # make it into this minute's payload.
-    #
-    # Due to the combining nature of the agent & layaway file, there's no
-    # guarantee that a high scoring local champion will still be a winner when
-    # they go up to "regionals" and are compared against the other processes
-    # running on a node.
-    def score(request)
-      unique_name = request.unique_name
-      if unique_name == :unknown
-        return -1 # A negative score, should never be good enough to store.
-      end
-
-      total_time = request.root_layer.total_call_time
-
-      # How long has it been since we've seen this?
-      age = Time.now - last_seen[unique_name]
-
-      # What approximate percentile was this request?
-      percentile = ScoutApm::Agent.instance.request_histograms.approximate_quantile_of_value(unique_name, total_time)
-
-      return speed_points(total_time) + percentile_points(percentile) + age_points(age)
-    end
-
-    private
-
-    def unique_name_for(request)
-      scope_layer = LayerConverters::ConverterBase.new(request).scope_layer
-      if scope_layer
-        scope_layer.legacy_metric_name
+    def capture_type(time)
+      if !slow_enough?(time)
+        CAPTURE_NONE
       else
-        :unknown
+        CAPTURE_DETAIL
       end
     end
 
-    # Time in seconds
-    # Logarithm keeps huge times from swamping the other metrics.
-    # 1+ is necessary to keep the log function in positive territory.
-    def speed_points(time)
-      Math.log(1 + time) * POINT_MULTIPLIER_SPEED
-    end
-
-    def percentile_points(percentile)
-      if percentile < 40
-        0.4 # Don't put much emphasis on capturing low percentiles.
-      elsif percentile < 60
-        1.4 # Highest here to get mean traces
-      elsif percentile < 90
-        0.7 # Between 60 & 90% is fine.
-      elsif percentile >= 90
-        1.4 # Highest here to get 90+%ile traces
-      else
-        # impossible.
-        percentile
-      end
-    end
+    private
 
-    def age_points(age)
-      age / 60.0 * POINT_MULTIPLIER_AGE
+    def slow_enough?(time)
+      time > SLOW_REQUEST_TIME_THRESHOLD
     end
   end
 end

data/lib/scout_apm/slow_transaction.rb

@@ -5,21 +5,33 @@ module ScoutApm
     attr_reader :metric_name
     attr_reader :total_call_time
     attr_reader :metrics
+    attr_reader :allocation_metrics
     attr_reader :meta
     attr_reader :uri
     attr_reader :context
     attr_reader :time
     attr_reader :prof
+    attr_reader :raw_prof
+    attr_reader :mem_delta
+    attr_reader :allocations
+    attr_accessor :hostname # hack - we need to reset these server side.
+    attr_accessor :seconds_since_startup # hack - we need to reset these server side.
 
-    def initialize(uri, metric_name, total_call_time, metrics, context, time, raw_stackprof, score)
+    def initialize(uri, metric_name, total_call_time, metrics, allocation_metrics, context, time, raw_stackprof, mem_delta, allocations)
       @uri = uri
       @metric_name = metric_name
       @total_call_time = total_call_time
       @metrics = metrics
+      @allocation_metrics = allocation_metrics
       @context = context
       @time = time
-      @prof = []
-      @score = score
+      @prof = ScoutApm::StackprofTreeCollapser.new(raw_stackprof).call
+      @raw_prof = raw_stackprof # Send whole data up to server
+      @mem_delta = mem_delta
+      @allocations = allocations
+      @seconds_since_startup = (Time.now - ScoutApm::Agent.instance.process_start_time)
+      @hostname = ScoutApm::Environment.instance.hostname
+      ScoutApm::Agent.instance.logger.debug { "Slow Request [#{uri}] - Call Time: #{total_call_time} Mem Delta: #{mem_delta}"}
     end
 
     # Used to remove metrics when the payload will be too large.
@@ -33,28 +45,12 @@ module ScoutApm
     end
 
     def as_json
-      json_attributes = [:key, :time, :total_call_time, :uri, [:context, :context_hash], :prof, :score]
+      json_attributes = [:key, :time, :total_call_time, :uri, [:context, :context_hash], :prof, :mem_delta, :allocations, :seconds_since_startup, :hostname]
       ScoutApm::AttributeArranger.call(self, json_attributes)
     end
 
     def context_hash
       context.to_hash
     end
-
-    ########################
-    # Scorable interface
-    #
-    # Needed so we can merge ScoredItemSet instances
-    def call
-      self
-    end
-
-    def name
-      metric_name
-    end
-
-    def score
-      @score
-    end
   end
 end

data/lib/scout_apm/stackprof_tree_collapser.rb

@@ -0,0 +1,103 @@
+module ScoutApm
+  class StackprofTreeCollapser
+    attr_reader :raw_stackprof
+    attr_reader :nodes
+
+    def initialize(raw_stackprof)
+      @raw_stackprof = raw_stackprof
+
+      # Log the raw stackprof info
+      #unless StackProf.respond_to?(:fake?) && StackProf.fake?
+      #  begin
+      #    ScoutApm::Agent.instance.logger.debug("StackProf - Samples: #{raw_stackprof[:samples]}, GC: #{raw_stackprof[:gc_samples]}, missed: #{raw_stackprof[:missed_samples]}, Interval: #{raw_stackprof[:interval]}")
+      #  rescue
+      #    ScoutApm::Agent.instance.logger.debug("StackProf Raw - #{raw_stackprof.inspect}")
+      #  end
+      #end
+    end
+
+    def call
+      build_tree
+      connect_children
+      total_samples_of_app_nodes
+    rescue
+      []
+    end
+
+    private
+
+    def build_tree
+      @nodes = raw_stackprof[:frames].map do |(frame_id, frame_data)|
+        TreeNode.new(frame_id,                     # frame_id
+                     frame_data[:name],            # name
+                     frame_data[:file],            # file
+                     frame_data[:line],            # line
+                     frame_data[:samples],         # samples
+                     frame_data[:total_samples],   # total_samples
+                     (frame_data[:edges] || {}),   # children_edges [ { id => weight } ]
+                     [],                           # children [ treenode, ... ]
+                     []                            # parents [ [treenode, int (weight) ], [...] ]
+                    )
+      end
+    end
+
+    def connect_children
+      nodes.each do |node|
+        children = nodes.find_all { |n| node.children_edges.keys.include? n.frame_id }
+
+        node.children_edges.each do |(frame_id, weight)|
+          child = children.detect{ |c| c.frame_id == frame_id }
+          child.parents << [node, weight]
+        end
+
+        node.children = children
+      end
+    end
+
+    def in_app_nodes
+      nodes.select {|n| n.app? }
+    end
+
+    def total_samples_of_app_nodes
+      in_app_nodes.reject{|n| n.calls_only_app_nodes? && !n.has_samples? }.
+        map{|n| { :samples => n.total_samples,
+                  :name => n.name,
+                  :file => n.file,
+                  :line => n.line
+                }
+        }
+    end
+
+    ###########################################
+    # TreeNode class represents a single node.
+    ###########################################
+    TreeNode = Struct.new(:frame_id, :name, :file, :line, :samples, :total_samples,
+                          :children_edges, :children, :parents) do
+      def app?
+        @is_app ||= file =~ /^#{ScoutApm::Environment.instance.root}/
+      end
+
+      # Force object_id to be the equality mechanism, rather than struct's
+      # default which delegates to == on each value.  That is wrong because
+      # we want to be able to dup a node in the tree construction process and
+      # not have those compare equal to each other.
+      def ==(other)
+        object_id == other.object_id
+      end
+
+      def inspect
+        "#{frame_id}: #{name} - ##{samples}\n" +
+        "  Parents: #{parents.map{ |(p, w)| "#{p.name}: #{w}"}.join("\n           ") }\n" +
+        "  Children: #{children_edges.inspect} \n"
+      end
+
+      def calls_only_app_nodes?
+        children.all?(&:app?)
+      end
+
+      def has_samples?
+        samples > 0
+      end
+    end
+  end
+end

data/lib/scout_apm/store.rb

@@ -6,13 +6,9 @@ module ScoutApm
     # A hash of reporting periods. { StoreReportingPeriodTimestamp => StoreReportingPeriod }
     attr_reader :reporting_periods
 
-    # Used to pull metrics into each reporting period, as that reporting period is finished.
-    attr_reader :samplers
-
     def initialize
       @mutex = Mutex.new
       @reporting_periods = Hash.new { |h,k| h[k] = StoreReportingPeriod.new(k) }
-      @samplers = []
     end
 
     def current_timestamp
@@ -26,7 +22,7 @@ module ScoutApm
     # Save newly collected metrics
     def track!(metrics, options={})
       @mutex.synchronize {
-        current_period.absorb_metrics!(metrics)
+        current_period.merge_metrics!(metrics)
       }
     end
 
@@ -70,30 +66,11 @@ module ScoutApm
       @mutex.synchronize {
         reporting_periods.select { |time, rp| force || time.timestamp < current_timestamp.timestamp}.
                           each   { |time, rp|
-                                   collect_samplers(rp)
-                                   layaway.write_reporting_period(rp)
+                                   layaway.add_reporting_period(time, rp)
                                    reporting_periods.delete(time)
                                  }
       }
-    end
-
-    ######################################
-    # Sampler support
-    def add_sampler(sampler)
-      @samplers << sampler
-    end
-
-    def collect_samplers(rp)
-      @samplers.each do |sampler|
-        begin
-          metrics = sampler.metrics(rp.timestamp)
-          rp.absorb_metrics!(metrics)
-        rescue => e
-          ScoutApm::Agent.instance.logger.info "Error reading #{sampler.human_name} for period: #{rp}"
-          ScoutApm::Agent.instance.logger.debug e.message
-          ScoutApm::Agent.instance.logger.debug e.backtrace.join("\n")
-        end
-      end
+      ScoutApm::Agent.instance.logger.debug("Finished writing to layaway")
     end
   end
 
@@ -107,25 +84,8 @@ module ScoutApm
       @timestamp = @raw_time.to_i - @raw_time.sec # The normalized time (integer) to compare by
     end
 
-    def self.minutes_ago(min, base_time=Time.now)
-      adjusted = base_time - (min * 60)
-      new(adjusted)
-    end
-
     def to_s
-      strftime
-    end
-
-    def strftime(pattern=nil)
-      if pattern.nil?
-        to_time.iso8601
-      else
-        to_time.strftime(pattern)
-      end
-    end
-
-    def to_time
-      Time.at(@timestamp)
+      Time.at(@timestamp).iso8601
     end
 
     def eql?(o)
@@ -147,12 +107,11 @@ module ScoutApm
 
   # One period of Storage. Typically 1 minute
   class StoreReportingPeriod
+    # A SlowItemSet to store slow transactions in
+    attr_reader :slow_transactions
 
-    # A ScoredItemSet holding the "best" traces for the period
-    attr_reader :request_traces
-
-    # A ScoredItemSet holding the "best" traces for the period
-    attr_reader :job_traces
+    # A SlowItemSet to store slow jobs in
+    attr_reader :slow_jobs
 
     # A StoreReportingPeriodTimestamp representing the time that this
     # collection of metrics is for
@@ -163,50 +122,31 @@ module ScoutApm
     def initialize(timestamp)
       @timestamp = timestamp
 
-      @request_traces = ScoredItemSet.new
-      @job_traces = ScoredItemSet.new
+      @slow_transactions = SlowItemSet.new
+      @slow_jobs = SlowItemSet.new
 
       @metric_set = MetricSet.new
       @jobs = Hash.new
     end
 
-    # Merges another StoreReportingPeriod into this one
-    def merge(other)
-      self.
-        merge_metrics!(other.metric_set).
-        merge_slow_transactions!(other.slow_transactions_payload).
-        merge_jobs!(other.jobs).
-        merge_slow_jobs!(other.slow_jobs_payload)
-      self
-    end
-
     #################################
     # Add metrics as they are recorded
     #################################
-
-    # For absorbing an array of metric {Meta => Stat} records
-    def absorb_metrics!(metrics)
+    def merge_metrics!(metrics)
       metric_set.absorb_all(metrics)
       self
     end
 
-    # For merging when you have another metric_set object
-    # Makes sure that you don't duplicate error count records
-    def merge_metrics!(other_metric_set)
-      metric_set.combine!(other_metric_set)
-      self
-    end
-
     def merge_slow_transactions!(new_transactions)
       Array(new_transactions).each do |one_transaction|
-        request_traces << one_transaction
+        slow_transactions << one_transaction
       end
 
       self
     end
 
     def merge_jobs!(jobs)
-      Array(jobs).each do |job|
+      jobs.each do |job|
         if @jobs.has_key?(job)
           @jobs[job].combine!(job)
         else
@@ -219,10 +159,8 @@ module ScoutApm
 
     def merge_slow_jobs!(new_jobs)
       Array(new_jobs).each do |job|
-        job_traces << job
+        slow_jobs << job
       end
-
-      self
     end
 
     #################################
@@ -233,7 +171,7 @@ module ScoutApm
     end
 
     def slow_transactions_payload
-      request_traces.to_a
+      slow_transactions.to_a
     end
 
     def jobs
@@ -241,7 +179,7 @@ module ScoutApm
     end
 
     def slow_jobs_payload
-      job_traces.to_a
+      slow_jobs.to_a
     end
 
     #################################

data/lib/scout_apm/tracked_request.rb

@@ -22,6 +22,10 @@ module ScoutApm
     #   :queue_latency - how long a background Job spent in the queue before starting processing
     attr_reader :annotations
 
+    # Nil until the request is finalized, at which point it will hold the
+    # entire raw stackprof output for this request
+    attr_reader :stackprof
+
     # Headers as recorded by rails
     # Can be nil if we never reach a Rails Controller
     attr_reader :headers
@@ -39,12 +43,14 @@ module ScoutApm
 
     def initialize
       @layers = []
-      @call_counts = Hash.new { |h, k| h[k] = CallSet.new }
+      @call_set = Hash.new { |h, k| h[k] = CallSet.new }
       @annotations = {}
       @ignoring_children = false
       @context = Context.new
       @root_layer = nil
+      @stackprof = nil
       @error = false
+      @mem_start = mem_usage
     end
 
     def start_layer(layer)
@@ -53,7 +59,6 @@ module ScoutApm
       end
 
       start_request(layer) unless @root_layer
-      update_call_counts!(layer)
       @layers[-1].add_child(layer) if @layers.any?
       @layers.push(layer)
     end
@@ -63,7 +68,11 @@ module ScoutApm
 
       layer = @layers.pop
       layer.record_stop_time!
+      layer.record_allocations!
 
+      # This must be called before checking if a backtrace should be collected as the call count influences our capture logic.
+      # We call `#update_call_counts in stop layer to ensure the layer has a final desc. Layer#desc is updated during the AR instrumentation flow.
+      update_call_counts!(layer)
       if capture_backtrace?(layer)
         layer.capture_backtrace!
       end
@@ -73,6 +82,18 @@ module ScoutApm
       end
     end
 
+    # Grab the currently running layer. Useful for adding additional data as we
+    # learn it. This is useful in ActiveRecord instruments, where we start the
+    # instrumentation early, and gradually learn more about the request that
+    # actually happened as we go (for instance, the # of records found, or the
+    # actual SQL generated).
+    # 
+    # Returns nil in the case there is no current layer. That would be normal
+    # for a completed TrackedRequest
+    def current_layer
+      @layers.last
+    end
+
     BACKTRACE_BLACKLIST = ["Controller", "Job"]
     def capture_backtrace?(layer)
       # Never capture backtraces for this kind of layer. The backtrace will
@@ -88,7 +109,7 @@ module ScoutApm
       return true if layer.total_exclusive_time > BACKTRACE_THRESHOLD
 
       # Capture any layer that we've seen many times. Captures n+1 problems
-      return true if @call_counts[layer.name].capture_backtrace?
+      return true if @call_set[layer.name].capture_backtrace?
 
       # Don't capture otherwise
       false
@@ -96,7 +117,16 @@ module ScoutApm
 
     # Maintains a lookup Hash of call counts by layer name. Used to determine if we should capture a backtrace.
     def update_call_counts!(layer)
-      @call_counts[layer.name].update!(layer.desc)
+      @call_set[layer.name].update!(layer.desc)
+    end
+
+    # This may be in bytes or KB based on the OSX. We store this as-is here and only do conversion to MB in Layer Converters.
+    def mem_usage
+      ScoutApm::Instruments::Process::ProcessMemory.rss
+    end
+
+    def capture_mem_delta!
+      @mem_delta = mem_usage - @mem_start
     end
 
     ###################################
@@ -112,14 +142,21 @@ module ScoutApm
     # Run at the beginning of the whole request
     #
     # * Capture the first layer as the root_layer
+    # * Start Stackprof (disabling to avoid conflicts if stackprof is included as middleware since we aren't sending this up to server now)
     def start_request(layer)
       @root_layer = layer unless @root_layer # capture root layer
+      #StackProf.start(:mode => :wall, :interval => ScoutApm::Agent.instance.config.value("stackprof_interval"))
     end
 
     # Run at the end of the whole request
     #
+    # * Collect stackprof info
     # * Send the request off to be stored
     def stop_request
+      # ScoutApm::Agent.instance.logger.debug("stop_request: #{annotations[:uri]}" )
+      #StackProf.stop # disabling to avoid conflicts if stackprof is included as middleware since we aren't sending this up to server now
+      #@stackprof = StackProf.results
+
       record!
     end
 
@@ -174,47 +211,27 @@ module ScoutApm
     def record!
       @recorded = true
 
-      # Update immediate and long-term histograms for both job and web requests
-      if unique_name != :unknown
-        ScoutApm::Agent.instance.request_histograms.add(unique_name, root_layer.total_call_time)
-        ScoutApm::Agent.instance.request_histograms_by_time[ScoutApm::Agent.instance.store.current_timestamp].
-          add(unique_name, root_layer.total_call_time)
-      end
-
       metrics = LayerConverters::MetricConverter.new(self).call
       ScoutApm::Agent.instance.store.track!(metrics)
 
+      slow, slow_metrics = LayerConverters::SlowRequestConverter.new(self).call
+      ScoutApm::Agent.instance.store.track_slow_transaction!(slow)
+      ScoutApm::Agent.instance.store.track!(slow_metrics)
+
       error_metrics = LayerConverters::ErrorConverter.new(self).call
       ScoutApm::Agent.instance.store.track!(error_metrics)
 
-      if web?
-        # Don't #call this - that's the job of the ScoredItemSet later.
-        slow_converter = LayerConverters::SlowRequestConverter.new(self)
-        ScoutApm::Agent.instance.store.track_slow_transaction!(slow_converter)
+      queue_time_metrics = LayerConverters::RequestQueueTimeConverter.new(self).call
+      ScoutApm::Agent.instance.store.track!(queue_time_metrics)
 
-        queue_time_metrics = LayerConverters::RequestQueueTimeConverter.new(self).call
-        ScoutApm::Agent.instance.store.track!(queue_time_metrics)
-      end
-
-      if job?
-        job_metrics = LayerConverters::JobConverter.new(self).call
-        ScoutApm::Agent.instance.store.track_job!(job_metrics)
+      job = LayerConverters::JobConverter.new(self).call
+      ScoutApm::Agent.instance.store.track_job!(job)
 
-        job_converter = LayerConverters::SlowJobConverter.new(self)
-        ScoutApm::Agent.instance.store.track_slow_job!(job_converter)
-      end
-    end
+      slow_job = LayerConverters::SlowJobConverter.new(self).call
+      ScoutApm::Agent.instance.store.track_slow_job!(slow_job)
 
-    # Only call this after the request is complete
-    def unique_name
-      @unique_name ||= begin
-                         scope_layer = LayerConverters::ConverterBase.new(self).scope_layer
-                         if scope_layer
-                           scope_layer.legacy_metric_name
-                         else
-                           :unknown
-                         end
-                       end
+      allocation_metrics = LayerConverters::AllocationMetricConverter.new(self).call
+      ScoutApm::Agent.instance.store.track!(allocation_metrics)
     end
 
     # Have we already persisted this request?

data/lib/scout_apm/utils/active_record_metric_name.rb

@@ -40,6 +40,8 @@ module ScoutApm
         name.downcase == o.name.downcase
       end
 
+      alias_method :==, :eql?
+
       private
 
       def model

data/lib/scout_apm/utils/fake_stack_prof.rb

@@ -0,0 +1,40 @@
+# A fake implementation of stackprof, for systems that don't support it.
+module StackProf
+  def self.start(*args)
+    @running = true
+  end
+
+  def self.stop(*args)
+    @running = false
+  end
+
+  def self.running?
+    !!@running
+  end
+
+  def self.run(*args)
+    start
+    yield
+    stop
+    results
+  end
+
+  def self.sample(*args)
+  end
+
+  def self.results(*args)
+    {
+      :version => 0.0,
+      :mode => :wall,
+      :interval => 1000,
+      :samples => 0,
+      :gc_samples => 0,
+      :missed_samples => 0,
+      :frames => {},
+    }
+  end
+
+  def self.fake?
+    true
+  end
+end