scout_apm 1.1.0.pre1 → 1.2.0.pre1

data/lib/scout_apm/instruments/middleware.rb

@@ -0,0 +1,44 @@
+module ScoutApm
+  module Instruments
+    class Middleware
+      def initalize(logger=ScoutApm::Agent.instance.logger)
+        @logger = logger
+        @installed = false
+      end
+
+      def installed?
+        @installed
+      end
+
+      def install
+        @installed = true
+
+        if defined?(ActionDispatch) && defined?(ActionDispatch::MiddlewareStack) && defined?(ActionDispatch::MiddlewareStack::Middleware)
+          ActionDispatch::MiddlewareStack::Middleware.class_eval do
+            def build(app)
+              ScoutApm::Agent.instance.logger.info("Building Middleware #{klass.name}")
+              new_mw = klass.new(app, *args, &block)
+              MiddlewareWrapper.new(new_mw, klass.name)
+            end
+          end
+        end
+      end
+
+      class MiddlewareWrapper
+        def initialize(app, name)
+          @app = app
+          @type = "Middleware"
+          @name = name
+        end
+
+        def call(env)
+          req = ScoutApm::RequestManager.lookup
+          req.start_layer( ScoutApm::Layer.new(@type, @name) )
+          @app.call(env)
+        ensure
+          req.stop_layer
+        end
+      end
+    end
+  end
+end

data/lib/scout_apm/instruments/mongoid.rb

@@ -22,7 +22,7 @@ module ScoutApm
           ::Mongoid::Collection.class_eval do
             include ScoutApm::Tracer
             (::Mongoid::Collections::Operations::ALL - [:<<, :[]]).each do |method|
-              instrument_method method, :metric_name => "MongoDB/\#{@klass}/#{method}"
+              instrument_method method, :type => "MongoDB", :name => '#{@klass}/#{method}'
             end
           end
         end

data/lib/scout_apm/instruments/moped.rb

@@ -23,8 +23,8 @@ module ScoutApm
             def process_with_scout_instruments(operation, &callback)
               if operation.respond_to?(:collection)
                 collection = operation.collection
-                self.class.instrument("MongoDB/Process/#{collection}/#{operation.class.to_s.split('::').last}",
-                                      :desc => scout_sanitize_log(operation.log_inspect)) do
+                name = "Process/#{collection}/#{operation.class.to_s.split('::').last}"
+                self.class.instrument("MongoDB", name, :annotate_layer => { :query => scout_sanitize_log(operation.log_inspect) }) do
                   process_without_scout_instruments(operation, &callback)
                 end
               end

data/lib/scout_apm/instruments/net_http.rb

@@ -22,7 +22,7 @@ module ScoutApm
             include ScoutApm::Tracer
 
             def request_with_scout_instruments(*args,&block)
-              self.class.instrument("HTTP/request", :desc => "#{(@address+args.first.path.split('?').first)[0..99]}") do
+              self.class.instrument("HTTP", "request", :annotate_layer => { :url => "#{(@address+args.first.path.split('?').first)[0..99]}" } ) do
                 request_without_scout_instruments(*args,&block)
               end
             end
@@ -30,7 +30,6 @@ module ScoutApm
             alias request request_with_scout_instruments
           end
         end
-
       end
     end
   end

data/lib/scout_apm/instruments/process/process_cpu.rb

@@ -17,8 +17,12 @@ module ScoutApm
           @last_stime = t.stime
         end
 
+        def metric_type
+          "CPU"
+        end
+
         def metric_name
-          "CPU/Utilization"
+          "Utilization"
         end
 
         def human_name

data/lib/scout_apm/instruments/process/process_memory.rb

@@ -8,8 +8,12 @@ module ScoutApm
           @logger = logger
         end
 
+        def metric_type
+          "Memory"
+        end
+
         def metric_name
-          "Memory/Physical"
+          "Physical"
         end
 
         def human_name

data/lib/scout_apm/instruments/sinatra.rb

@@ -29,9 +29,21 @@ module ScoutApm
 
     module SinatraInstruments
       def dispatch_with_scout_instruments!
-        scout_controller_action = "Controller/Sinatra/#{scout_sinatra_controller_name(@request)}"
-        self.class.scout_apm_trace(scout_controller_action, :uri => @request.path_info, :ip => @request.ip) do
+        scout_controller_action = "Sinatra/#{scout_sinatra_controller_name(@request)}"
+
+        req = ScoutApm::RequestManager.lookup
+        req.annotate_request(:uri => @request.path_info)
+        req.context.add_user(:ip => @request.ip)
+        req.set_headers(request.headers)
+
+        req.start_layer( ScoutApm::Layer.new("Controller", scout_controller_action) )
+        begin
           dispatch_without_scout_instruments!
+        rescue
+          req.error!
+          raise
+        ensure
+          req.stop_layer
         end
       end
 

data/lib/scout_apm/layaway.rb

@@ -1,9 +1,5 @@
-# Stores metrics in a file before sending them to the server. Two uses:
+# Stores StoreReportingPeriod objects in a file before sending them to the server.
 # 1. A centralized store for multiple Agent processes. This way, only 1 checkin is sent to Scout rather than 1 per-process.
-# 2. Bundling up reports from multiple timeslices to make updates more efficent server-side.
-#
-# Data is stored in a Hash, where the keys are Time.to_i on the minute. The value is a Hash {:metrics => Hash, :slow_transactions => Array}.
-# When depositing data, the new data is either merged with an existing time or placed in a new key.
 module ScoutApm
   class Layaway
     attr_accessor :file
@@ -12,93 +8,51 @@ module ScoutApm
       @file = ScoutApm::LayawayFile.new
     end
 
-    def deposit_and_deliver
-      new_metrics = ScoutApm::Agent.instance.store.metric_hash
-      log_deposited_metrics(new_metrics)
-      log_deposited_slow_transactions(ScoutApm::Agent.instance.store.slow_transactions)
-      to_deliver = {}
-      file.read_and_write do |old_data|
-        old_data ||= Hash.new
-        # merge data
-        # if (1) there's data in the file and (2) there isn't any data yet for the current minute, this means we've
-        # collected all metrics for the previous slots and we're ready to deliver.
-        #
-        # Example w/2 processes:
-        #
-        # 12:00:34 ---
-        # Process 1: old_data.any? => false, so deposits.
-        # Process 2: old_data_any? => true and old_data[12:00].nil? => false, so deposits.
-        #
-        # 12:01:34 ---
-        # Process 1: old_data.any? => true and old_data[12:01].nil? => true, so delivers metrics.
-        # Process 2: old_data.any? => true and old_data[12:01].nil? => false, so deposits.
-        if old_data.any? and old_data[slot].nil?
-          to_deliver = old_data
-          old_data = Hash.new
-        elsif old_data.any?
-          ScoutApm::Agent.instance.logger.debug "Not yet time to deliver payload for slot [#{Utils::Time.to_s(old_data.keys.sort.last)}]"
+    # We're changing the format, so detect if we're loading an old formatted
+    # file, and just drop it if so. There's no important data there, since it's
+    # used mostly for just syncronizing between processes
+    def verify_layaway_file_contents
+      file.read_and_write do |existing_data|
+        existing_data ||= {}
+        if existing_data.keys.all?{|k| k.is_a? StoreReportingPeriodTimestamp } &&
+            existing_data.values.all? {|v| v.is_a? StoreReportingPeriod }
+          existing_data
         else
-          ScoutApm::Agent.instance.logger.debug "There is no data in the layaway file to deliver."
+          {}
         end
-        old_data[slot]=ScoutApm::Agent.instance.store.merge_data_and_clear(old_data[slot] || {:metrics => {}, :slow_transactions => []})
-        log_saved_data(old_data,new_metrics)
-        old_data
       end
-      to_deliver.any? ? validate_data(to_deliver) : {}
     end
 
-    # Ensures the data we're sending to the server isn't stale.
-    # This can occur if the agent is collecting data, and app server goes down w/data in the local storage.
-    # When it is restarted later data will remain in local storage but it won't be for the current reporting interval.
-    #
-    # If the data is stale, an empty Hash is returned. Otherwise, the data from the most recent slot is returned.
-    def validate_data(data)
-      data = data.to_a.sort
-      now = Time.now
-      if (most_recent = data.first.first) < now.to_i - 2*60
-        ScoutApm::Agent.instance.logger.debug "Local Storage is stale (#{Utils::Time.to_s(most_recent)}). Not sending data."
-        {}
-      else
-        data.first.last
+    def add_reporting_period(time, reporting_period)
+      file.read_and_write do |existing_data|
+        existing_data ||= Hash.new
+        existing_data.merge!(time => reporting_period) {|key, old_val, new_val|
+          old_val.merge_metrics!(new_val.metrics).merge_slow_transactions!(new_val.slow_transactions)
+        }
       end
-    rescue
-      ScoutApm::Agent.instance.logger.debug $!.message
-      ScoutApm::Agent.instance.logger.debug $!.backtrace
     end
 
-    # Data is stored under timestamp-keys, aligned to the beginning of the
-    # current minute
-    def slot
-      t = Time.now
-      t -= t.sec
-      t.to_i
-    end
+    REPORTING_INTERVAL = 60 # seconds
+    MAX_INTERVALS = 5
 
-    def log_deposited_metrics(new_metrics)
-      request_count = new_metrics.
-        to_a.
-        select    { |meta, stats| meta.metric_name =~ /\AController/ }.
-        map       { |meta, stats| stats.call_count }.
-        inject(0) { |total, i| total + i }
+    # Returns an array of ReportingPeriod objects that are ready to be pushed to the server
+    def periods_ready_for_delivery
+      ready_for_delivery = []
 
-      ScoutApm::Agent.instance.logger.debug "Depositing #{request_count} requests into #{Utils::Time.to_s(slot)} slot."
-    end
+      file.read_and_write do |existing_data|
+        existing_data ||= {}
+        ready_for_delivery = existing_data.select {|time, rp| should_send?(rp) } # Select off the values we want
+
+        # Rewrite anything not plucked out back to the file
+        existing_data.reject {|k, v| ready_for_delivery.keys.include?(k) }
+      end
 
-    def log_deposited_slow_transactions(new_slow_transactions)
-      ScoutApm::Agent.instance.logger.debug "Depositing #{new_slow_transactions.size} slow transactions into #{Utils::Time.to_s(slot)} slot."
+      return ready_for_delivery.values
     end
 
-    def log_saved_data(old_data,new_metrics)
-      ScoutApm::Agent.instance.logger.debug "Saving the following #{old_data.size} time slots locally:"
-      old_data.each do |k,v|
-        controller_count = 0
-        new_metrics.each do |meta,stats|
-          if meta.metric_name =~ /\AController/
-            controller_count += stats.call_count
-          end
-        end
-        ScoutApm::Agent.instance.logger.debug "#{Utils::Time.to_s(k)} => #{controller_count} requests and #{v[:slow_transactions].size} slow transactions"
-      end
+    # We just want to send anything older than X
+    def should_send?(reporting_period)
+      reporting_period.timestamp.age_in_seconds > REPORTING_INTERVAL
     end
   end
 end

data/lib/scout_apm/layaway_file.rb

@@ -42,8 +42,9 @@ module ScoutApm
                                             "Change the path by setting the `data_file` key in scout_apm.yml"
                                            )
       ScoutApm::Agent.instance.logger.debug(e.backtrace.join("\n\t"))
+
       # ensure the in-memory metric hash is cleared so data doesn't continue to accumulate.
-      ScoutApm::Agent.instance.store.metric_hash = {}
+      # ScoutApm::Agent.instance.store.metric_hash = {}
     end
 
     def get_data(f)

data/lib/scout_apm/layer.rb

@@ -0,0 +1,115 @@
+module ScoutApm
+  class Layer
+    # Type: a general name for the kind of thing being tracked.
+    #   Examples: "Middleware", "ActiveRecord", "Controller", "View"
+    #
+    attr_reader :type
+
+    # Name: a more specific name of this single item
+    #   Examples: "Rack::Cache", "User#find", "users/index", "users/index.html.erb"
+    attr_reader :name
+
+    # An array of children layers, in call order.
+    # For instance, if we are in a middleware, there will likely be only a single
+    # child, which is another middleware.  In a Controller, we may have a handful
+    # of children: [ActiveRecord, ActiveRecord, View, HTTP Call].
+    #
+    # This useful to get actual time spent in this layer vs. children time
+    attr_reader :children
+
+    # Time objects recording the start & stop times of this layer
+    attr_reader :start_time, :stop_time
+
+    # The description of this layer.  Will contain additional details specific to the type of layer.
+    # For an ActiveRecord metric, it will contain the SQL run
+    # For an outoing HTTP call, it will contain the remote URL accessed
+    # Leave blank if there is nothing to note
+    attr_reader :desc
+
+    # If this layer took longer than a fixed amount of time, store the
+    # backtrace of where it occurred.
+    attr_reader :backtrace
+
+
+    def initialize(type, name, start_time = Time.now)
+      @type = type
+      @name = name
+      @start_time = start_time
+      @children = [] # In order of calls
+      @desc = nil
+    end
+
+    def add_child(child)
+      @children << child
+    end
+
+    def record_stop_time!(stop_time = Time.now)
+      @stop_time = stop_time
+    end
+
+    def desc=(desc)
+      @desc = desc
+    end
+
+    def subscopable!
+      @subscopable = true
+    end
+
+    def subscopable?
+      @subscopable
+    end
+
+    # This is the old style name. This function is used for now, but should be
+    # removed, and the new type & name split should be enforced through the
+    # app.
+    def legacy_metric_name
+      "#{type}/#{name}"
+    end
+
+    def store_backtrace(bt)
+      @backtrace = bt
+    end
+
+    ######################################
+    # Debugging Helpers
+    ######################################
+
+    # May not be safe to call in every rails app, relies on Time#iso8601
+    def to_s
+      name_clause = "#{type}/#{name}"
+
+      total_string = total_call_time == 0 ? nil : "Total: #{total_call_time}"
+      self_string = total_exclusive_time == 0 ? nil : "Self: #{total_exclusive_time}"
+      timing_string = [total_string, self_string].compact.join(", ")
+
+      time_clause = "(Start: #{start_time.iso8601} / Stop: #{stop_time.try(:iso8601)} [#{timing_string}])"
+      desc_clause = "Description: #{desc.inspect}"
+      children_clause = "Children: #{children.length}"
+
+      "<Layer: #{name_clause} #{time_clause} #{desc_clause} #{children_clause}>"
+    end
+
+    ######################################
+    # Time Calculations
+    ######################################
+
+    def total_call_time
+      if stop_time
+        stop_time - start_time
+      else
+        # Shouldn't have called this yet. Return 0
+        0
+      end
+    end
+
+    def total_exclusive_time
+      total_call_time - child_time
+    end
+
+    def child_time
+      children.
+        map { |child| child.total_call_time }.
+        inject(0) { |sum, time| sum + time }
+    end
+  end
+end

data/lib/scout_apm/layer_converter.rb

@@ -0,0 +1,196 @@
+module ScoutApm
+  class LayerConverterBase
+    attr_reader :walker
+    attr_reader :request
+    attr_reader :root_layer
+
+    def initialize(request)
+      @request = request
+      @root_layer = request.root_layer
+      @walker = LayerDepthFirstWalker.new(root_layer)
+    end
+
+    # Scope is determined by the first Controller we hit.  Most of the time
+    # there will only be 1 anyway.  But if you have a controller that calls
+    # another controller method, we may pick that up:
+    #     def update
+    #       show
+    #       render :update
+    #     end
+    def scope_layer
+      @scope_layer ||= walker.walk do |layer|
+        if layer.type == "Controller"
+          break layer
+        end
+      end
+    end
+
+  end
+
+  # Take a TrackedRequest and turn it into a hash of:
+  #   MetricMeta => MetricStats
+  # This will do some merging of metrics as duplicate calls are seen.
+  class LayerMetricConverter < LayerConverterBase
+    def call
+      scope = scope_layer
+
+      # TODO: Track requests that never reach a Controller (for example, when
+      # Middleware decides to return rather than passing onward)
+      return {} unless scope
+
+      create_metrics
+    end
+
+    # Full metrics from this request. These get aggregated in Store for the
+    # overview metrics, or stored permanently in a SlowTransaction
+    # Some merging of metrics will happen here, so if a request calls the same
+    # ActiveRecord or View repeatedly, it'll get merged.
+    def create_metrics
+      metric_hash = Hash.new
+
+      walker.walk do |layer|
+        meta_options = if layer == scope_layer # We don't scope the controller under itself
+                         {}
+                       else
+                         {:scope => scope_layer.legacy_metric_name}
+                       end
+
+        meta_options.merge!(:desc => layer.desc) if layer.desc
+
+        meta = MetricMeta.new(layer.legacy_metric_name, meta_options)
+        meta.extra.merge!(:backtrace => layer.backtrace) if layer.backtrace
+        metric_hash[meta] ||= MetricStats.new( meta_options.has_key?(:scope) )
+
+        stat = metric_hash[meta]
+        stat.update!(layer.total_call_time, layer.total_exclusive_time)
+      end
+
+      metric_hash
+    end
+  end
+
+  class LayerErrorConverter < LayerConverterBase
+    def call
+      scope = scope_layer
+
+      # Should we mark a request as errored out if a middleware raises?
+      # How does that interact w/ a tool like Sentry or Honeybadger?
+      return {} unless scope
+      return {} unless request.error?
+
+      meta = MetricMeta.new("Errors/#{scope.legacy_metric_name}", {})
+      stat = MetricStats.new
+      stat.update!(1)
+
+      { meta => stat }
+    end
+  end
+
+  # Take a TrackedRequest and turn it into either nil, or a SlowTransaction record
+  class LayerSlowTransactionConverter < LayerConverterBase
+    SLOW_REQUEST_TIME_THRESHOLD = 2 # seconds
+
+    def call
+      return nil unless should_capture_slow_request?
+
+      scope = scope_layer
+      return nil unless scope
+
+      uri = request.annotations[:uri] || ""
+      SlowTransaction.new(uri,
+                          scope.legacy_metric_name,
+                          root_layer.total_call_time,
+                          create_metrics,
+                          request.context,
+                          root_layer.stop_time,
+                          request.stackprof)
+    end
+
+    # Full metrics from this request. These get aggregated in Store for the
+    # overview metrics, or stored permanently in a SlowTransaction
+    # Some merging of metrics will happen here, so if a request calls the same
+    # ActiveRecord or View repeatedly, it'll get merged.
+    def create_metrics
+      metric_hash = Hash.new
+
+      subscope_layers = []
+
+      walker.before do |layer|
+        if layer.subscopable?
+          subscope_layers.push(layer)
+        end
+      end
+
+      walker.after do |layer|
+        if layer.subscopable?
+          subscope_layers.pop
+        end
+      end
+
+      walker.walk do |layer|
+        meta_options = if subscope_layers.first && layer != subscope_layers.first # Don't scope under ourself.
+                         subscope_name = subscope_layers.first.legacy_metric_name
+                         {:scope => subscope_name}
+                       elsif layer == scope_layer # We don't scope the controller under itself
+                         {}
+                       else
+                         {:scope => scope_layer.legacy_metric_name}
+                       end
+
+        meta_options.merge!(:desc => layer.desc) if layer.desc
+
+        meta = MetricMeta.new(layer.legacy_metric_name, meta_options)
+        meta.extra.merge!(:backtrace => layer.backtrace) if layer.backtrace
+        metric_hash[meta] ||= MetricStats.new( meta_options.has_key?(:scope) )
+
+        stat = metric_hash[meta]
+        stat.update!(layer.total_call_time, layer.total_exclusive_time)
+      end
+
+      metric_hash
+    end
+
+    def should_capture_slow_request?
+      root_layer.total_call_time > SLOW_REQUEST_TIME_THRESHOLD
+    end
+  end
+
+  class LayerDepthFirstWalker
+    attr_reader :root_layer
+
+    def initialize(root_layer)
+      @root_layer = root_layer
+    end
+
+    def before(&block)
+      @before_block = block
+    end
+
+    def after(&block)
+      @after_block = block
+    end
+
+    def walk(layer=root_layer, &block)
+      layer.children.each do |child|
+        @before_block.call(child) if @before_block
+        yield child
+        walk(child, &block)
+        @after_block.call(child) if @after_block
+      end
+      nil
+    end
+
+    # Do this w/o using recursion, since it's prone to stack overflows
+    # Takes a block to run over each layer
+    # def walk
+      # layer_stack = [root_layer]
+
+      # while layer_stack.any?
+        # current_layer = layer_stack.pop
+        # current_layer.children.reverse.each { |child| layer_stack.push(child) }
+
+        # yield current_layer
+      # end
+    # end
+  end
+end

data/lib/scout_apm/metric_meta.rb

@@ -1,11 +1,12 @@
 # Contains the meta information associated with a metric. Used to lookup Metrics in to Store's metric_hash.
-class ScoutApm::MetricMeta
+module ScoutApm
+class MetricMeta
   include ScoutApm::BucketNameSplitter
 
   def initialize(metric_name, options = {})
     @metric_name = metric_name
     @metric_id = nil
-    @scope = Thread::current[:scout_apm_sub_scope] || Thread::current[:scout_apm_scope_name]
+    @scope = options[:scope]
     @desc = options[:desc]
     @extra = {}
   end
@@ -14,6 +15,20 @@ class ScoutApm::MetricMeta
   attr_accessor :client_id
   attr_accessor :desc, :extra
 
+  # Unsure if type or bucket is a better name.
+  def type
+    bucket
+  end
+
+  # A key metric is the "core" of a request - either the Rails controller reached, or the background Job executed
+  def key_metric?
+    self.class.key_metric?(metric_name)
+  end
+
+  def self.key_metric?(metric_name)
+    !!(metric_name =~ /\A(Controller|Job)\//)
+  end
+
   # To avoid conflicts with different JSON libaries
   def to_json(*a)
      %Q[{"metric_id":#{metric_id || 'null'},"metric_name":#{metric_name.to_json},"scope":#{scope.to_json || 'null'}}]
@@ -31,7 +46,11 @@ class ScoutApm::MetricMeta
   end
 
   def eql?(o)
-   self.class == o.class && metric_name.downcase.eql?(o.metric_name.downcase) && scope == o.scope && client_id == o.client_id && desc == o.desc
+   self.class             == o.class                &&
+     metric_name.downcase == o.metric_name.downcase &&
+     scope                == o.scope                &&
+     client_id            == o.client_id            &&
+     desc                 == o.desc
   end
 
   def as_json
@@ -39,4 +58,5 @@ class ScoutApm::MetricMeta
     # query, stack_trace
     ScoutApm::AttributeArranger.call(self, json_attributes)
   end
-end # class MetricMeta
+end
+end