scout_apm 2.2.0.pre3 → 2.3.0.pre

data/lib/scout_apm/instruments/net_http.rb

@@ -22,12 +22,17 @@ module ScoutApm
             include ScoutApm::Tracer
 
             def request_with_scout_instruments(*args,&block)
-              url = (@address + args.first.path.split('?').first)[0..99]
-              self.class.instrument("HTTP", "request", :desc => url) do
+              self.class.instrument("HTTP", "request", :desc => request_scout_description(args.first)) do
                 request_without_scout_instruments(*args, &block)
               end
             end
 
+            def request_scout_description(req)
+              path = req.path
+              path = path.path if path.respond_to?(:path)
+              (@address + path.split('?').first)[0..99]
+            end
+
             alias request_without_scout_instruments request
             alias request request_with_scout_instruments
           end

data/lib/scout_apm/instruments/percentile_sampler.rb

@@ -1,36 +1,53 @@
 module ScoutApm
   module Instruments
+
+    class HistogramReport
+      attr_reader :name
+      attr_reader :histogram
+
+      def initialize(name, histogram)
+        @name = name
+        @histogram = histogram
+      end
+
+      def combine!(other)
+        raise "Mismatched Histogram Names" unless name == other.name
+        histogram.combine!(other.histogram)
+        self
+      end
+    end
+
     class PercentileSampler
       attr_reader :logger
 
-      attr_reader :percentiles
+      # A hash of { time => RequestHistograms }
+      attr_reader :histograms
 
-      def initialize(logger, percentiles)
+      def initialize(logger, histograms)
         @logger = logger
-        @percentiles = Array(percentiles)
+        @histograms = histograms
       end
 
       def human_name
-        "Percentiles"
+        'Percentiles'
       end
 
-      # Gets the 95th%ile for the time requested
-      def metrics(time)
-        ms = {}
-        histos = ScoutApm::Agent.instance.request_histograms_by_time[time]
-        histos.each_name do |name|
-          percentiles.each do |percentile|
-            meta = MetricMeta.new("Percentile/#{percentile}/#{name}")
-            stat = MetricStats.new
-            stat.update!(histos.quantile(name, percentile))
-            ms[meta] = stat
-          end
-        end
+      def metrics(timestamp, store)
+        store.track_histograms!(percentiles(timestamp), :timestamp => timestamp)
+      end
+
+      def percentiles(time)
+        result = []
 
-        # Wipe the histograms we just collected data on
-        ScoutApm::Agent.instance.request_histograms_by_time.delete(time)
+        histogram = histograms.delete(time)
+
+        return result unless histogram
+
+        histogram.each_name do |name|
+          result << HistogramReport.new(name, histogram.raw(name))
+        end
 
-        ms
+        result
       end
     end
   end

data/lib/scout_apm/instruments/process/process_cpu.rb

@@ -29,16 +29,17 @@ module ScoutApm
           "Process CPU"
         end
 
-        def metrics(_time)
+        def metrics(timestamp, store)
           result = run
           if result
             meta = MetricMeta.new("#{metric_type}/#{metric_name}")
             stat = MetricStats.new(false)
             stat.update!(result)
-            { meta => stat }
+            store.track!({ meta => stat }, :timestamp => timestamp)
           else
             {}
           end
+
         end
 
         # TODO: Figure out a good default instead of nil

data/lib/scout_apm/instruments/process/process_memory.rb

@@ -33,20 +33,20 @@ module ScoutApm
           "Process Memory"
         end
 
-        def metrics(_time)
+        def metrics(timestamp, store)
           result = run
           if result
             meta = MetricMeta.new("#{metric_type}/#{metric_name}")
             stat = MetricStats.new(false)
             stat.update!(result)
-            { meta => stat }
+            store.track!({ meta => stat }, :timestamp => timestamp)
           else
             {}
           end
         end
 
         def run
-          self.class.rss_in_mb.tap { |res| logger.debug "#{human_name}: #{res.inspect}" }      
+          self.class.rss_in_mb.tap { |res| logger.debug "#{human_name}: #{res.inspect}" }
         end
       end
     end

data/lib/scout_apm/instruments/resque.rb

@@ -0,0 +1,40 @@
+module ScoutApm
+  module Instruments
+    module Resque
+      def around_perform_with_scout_instruments(*args)
+        job_name = self.to_s
+        queue = find_queue
+
+        if job_name == "ActiveJob::QueueAdapters::ResqueAdapter::JobWrapper"
+          job_name = args.first["job_class"] rescue job_name
+          queue = args.first["queue_name"] rescue queue_name
+        end
+
+        req = ScoutApm::RequestManager.lookup
+        req.job!
+
+        begin
+          req.start_layer(ScoutApm::Layer.new('Queue', queue))
+          started_queue = true
+          req.start_layer(ScoutApm::Layer.new('Job', job_name))
+          started_job = true
+
+          yield
+        rescue => e
+          req.error!
+          raise
+        ensure
+          req.stop_layer if started_job
+          req.stop_layer if started_queue
+        end
+      end
+
+      def find_queue
+        return @queue if @queue
+        return queue if self.respond_to?(:queue)
+        return "unknown"
+      end
+    end
+  end
+end
+

data/lib/scout_apm/layaway.rb

@@ -7,13 +7,13 @@
 #
 module ScoutApm
   class Layaway
-    # How old a file needs to be in Seconds before it gets reported.
-    REPORTING_AGE = 120
-
     # How long to let a stale file sit before deleting it.
     # Letting it sit a bit may be useful for debugging
     STALE_AGE = 10 * 60
 
+    # Failsafe to prevent writing layaway files if for some reason they are not being cleaned up
+    MAX_FILES_LIMIT = 5000
+
     # A strftime format string for how we render timestamps in filenames.
     # Must be sortable as an integer
     TIME_FORMAT = "%Y%m%d%H%M"
@@ -48,7 +48,11 @@ module ScoutApm
       @directory = Pathname.new(found)
     end
 
-    def write_reporting_period(reporting_period)
+    def write_reporting_period(reporting_period, files_limit = MAX_FILES_LIMIT)
+      if at_layaway_file_limit?(files_limit)
+        ScoutApm::Agent.instance.logger.error("Hit layaway file limit. Not writing to layaway file")
+        return false
+      end
       filename = file_for(reporting_period.timestamp)
       layaway_file = LayawayFile.new(filename)
       layaway_file.write(reporting_period)
@@ -60,41 +64,56 @@ module ScoutApm
     def with_claim(timestamp)
       coordinator_file = glob_pattern(timestamp, :coordinator)
 
-
-      # This file gets deleted only by a process that successfully obtained a lock
-      f = File.open(coordinator_file, File::RDWR | File::CREAT)
       begin
-        # Nonblocking, Exclusive lock.
-        if f.flock(File::LOCK_EX | File::LOCK_NB)
-
-          ScoutApm::Agent.instance.logger.debug("Obtained Reporting Lock")
-
-          files = all_files_for(timestamp).reject{|l| l.to_s == coordinator_file.to_s }
-          rps = files.map{ |layaway| LayawayFile.new(layaway).load }.compact
-          if rps.any?
-            yield rps
+        # This file gets deleted only by a process that successfully created and obtained the exclusive lock
+        f = File.open(coordinator_file, File::RDWR | File::CREAT | File::EXCL | File::NONBLOCK)
+      rescue Errno::EEXIST
+        false
+      end
 
-            delete_files_for(timestamp) # also removes the coodinator_file
-            delete_stale_files(timestamp.to_time - STALE_AGE)
-          else
-            File.unlink(coordinator_file)
-            ScoutApm::Agent.instance.logger.debug("No layaway files to report")
+      begin
+        if f
+          begin
+            ScoutApm::Agent.instance.logger.debug("Obtained Reporting Lock")
+
+            log_layaway_file_information
+
+            files = all_files_for(timestamp).reject{|l| l.to_s == coordinator_file.to_s }
+            rps = files.map{ |layaway| LayawayFile.new(layaway).load }.compact
+            if rps.any?
+              yield rps
+
+              ScoutApm::Agent.instance.logger.debug("Deleting the now-reported layaway files for #{timestamp.to_s}")
+              delete_files_for(timestamp) # also removes the coodinator_file
+
+              ScoutApm::Agent.instance.logger.debug("Checking for any Stale layaway files")
+              delete_stale_files(timestamp.to_time - STALE_AGE)
+            else
+              File.unlink(coordinator_file)
+              ScoutApm::Agent.instance.logger.debug("No layaway files to report")
+            end
+
+            true
+          rescue Exception => e
+            ScoutApm::Agent.instance.logger.debug("Caught an exception in with_claim, with the coordination file locked: #{e.message}, #{e.backtrace.inspect}")
+            raise
+          ensure
+            # Unlock the file when done!
+            f.flock(File::LOCK_UN | File::LOCK_NB)
+            f.close
           end
-
-          # Unlock the file when done!
-          f.flock(File::LOCK_UN | File::LOCK_NB)
-          f.close
-          true
         else
           # Didn't obtain lock, another process is reporting. Return false from this function, but otherwise no work
-          f.close
           false
         end
       end
     end
 
     def delete_files_for(timestamp)
-      all_files_for(timestamp).each { |layaway| File.unlink(layaway) }
+      all_files_for(timestamp).each { |layaway|
+        ScoutApm::Agent.instance.logger.debug("Deleting layaway file: #{layaway}")
+        File.unlink(layaway)
+      }
     end
 
     def delete_stale_files(older_than)
@@ -105,6 +124,8 @@ module ScoutApm
         select { |timestamp| timestamp.to_i < older_than.strftime(TIME_FORMAT).to_i }.
           tap  { |timestamps| ScoutApm::Agent.instance.logger.debug("Deleting stale layaway files with timestamps: #{timestamps.inspect}") }.
         map    { |timestamp| delete_files_for(timestamp) }
+    rescue => e
+      ScoutApm::Agent.instance.logger.debug("Problem deleting stale files: #{e.message}, #{e.backtrace.inspect}")
     end
 
     private
@@ -155,6 +176,24 @@ module ScoutApm
         nil
       end
     end
+
+    def at_layaway_file_limit?(files_limit = MAX_FILES_LIMIT)
+      all_files_for(:all).count >= files_limit
+    end
+
+    def log_layaway_file_information
+      files_in_temp = Dir["#{directory}/*"].count
+
+      all_filenames = all_files_for(:all)
+      count_per_timestamp = Hash[
+        all_filenames.
+        group_by {|f| timestamp_from_filename(f) }.
+        map{ |timestamp, list| [timestamp, list.length] }
+      ]
+
+
+      ScoutApm::Agent.instance.logger.debug("Total in #{directory}: #{files_in_temp}. Total Layaway Files: #{all_filenames.size}.  By Timestamp: #{count_per_timestamp.inspect}")
+    end
   end
 end
 

data/lib/scout_apm/layer.rb

@@ -13,13 +13,17 @@ module ScoutApm
     #   instrumentation for an example of how this is useful
     attr_accessor :name
 
-    # An array of children layers, in call order.
+    # An array of children layers
     # For instance, if we are in a middleware, there will likely be only a single
     # child, which is another middleware.  In a Controller, we may have a handful
     # of children: [ActiveRecord, ActiveRecord, View, HTTP Call].
     #
     # This useful to get actual time spent in this layer vs. children time
-    attr_reader :children
+    #
+    # TODO: Check callers for compatibility w/ nil to avoid making an empty array
+    def children
+      @children || LayerChildrenSet.new
+    end
 
     # Time objects recording the start & stop times of this layer
     attr_reader :start_time, :stop_time
@@ -38,41 +42,31 @@ module ScoutApm
     # Known Keys:
     #   :record_count - The number of rows returned by an AR query (From notification instantiation.active_record)
     #   :class_name   - The ActiveRecord class name (From notification instantiation.active_record)
+    #
+    # If no annotations are ever set, this will return nil
     attr_reader :annotations
 
-    # ScoutProf - trace_index is an index into the Stack structure in the C
-    # code, used to store captured traces.
-    attr_reader :trace_index
-
-    # ScoutProf - frame_index is an optimization to not capture a few frames
-    # during scoutprof instrumentation
-    attr_reader :frame_index
-
-    # Captured backtraces from ScoutProf. This is distinct from the backtrace
-    # attribute, which gets the ruby backtrace of any given layer. StackProf
-    # focuses on Controller layers, and requires a native extension and a
-    # reasonably recent Ruby.
-    attr_reader :traces
-
     BACKTRACE_CALLER_LIMIT = 50 # maximum number of lines to send thru for backtrace analysis
 
     def initialize(type, name, start_time = Time.now)
       @type = type
       @name = name
-      @annotations = {}
       @start_time = start_time
       @allocations_start = ScoutApm::Instruments::Allocations.count
       @allocations_stop = 0
-      @children = [] # In order of calls
+
+      # initialize these only on first use
+      @children = nil
+      @annotations = nil
       @desc = nil
+    end
 
-      @traces = ScoutApm::TraceSet.new
-      @raw_frames = []
-      @frame_index = ScoutApm::Instruments::Stacks.current_frame_index # For efficiency sake, try to skip the bottom X frames when collecting traces
-      @trace_index = ScoutApm::Instruments::Stacks.current_trace_index
+    def limited?
+      false
     end
 
     def add_child(child)
+      @children ||= LayerChildrenSet.new
       @children << child
     end
 
@@ -91,6 +85,7 @@ module ScoutApm
 
     # This data is internal to ScoutApm, to add custom information, use the Context api.
     def annotate_layer(hsh)
+      @annotations ||= {}
       @annotations.merge!(hsh)
     end
 
@@ -102,14 +97,6 @@ module ScoutApm
       @subscopable
     end
 
-    def traced!
-      @traced = true
-    end
-
-    def traced?
-      @traced
-    end
-
     # This is the old style name. This function is used for now, but should be
     # removed, and the new type & name split should be enforced through the
     # app.
@@ -131,35 +118,6 @@ module ScoutApm
       end
     end
 
-    # Set the name of the file that this action is coming from.
-    # TraceSet uses this to more accurately filter backtraces
-    def set_root_class(klass_name)
-      @traces.set_root_class(klass_name)
-    end
-
-    def start_sampling
-      if ScoutApm::Agent.instance.config.value('profile') && traced?
-        ScoutApm::Instruments::Stacks.update_indexes(frame_index, trace_index)
-        ScoutApm::Instruments::Stacks.start_sampling
-      else
-        ScoutApm::Instruments::Stacks.stop_sampling(false)
-      end
-    end
-
-    def record_traces!
-      if ScoutApm::Agent.instance.config.value('profile')
-        ScoutApm::Instruments::Stacks.stop_sampling(false)
-        if traced?
-          traces.raw_traces = ScoutApm::Instruments::Stacks.profile_frames
-          traces.skipped_in_gc = ScoutApm::Instruments::Stacks.skipped_in_gc
-          traces.skipped_in_handler = ScoutApm::Instruments::Stacks.skipped_in_handler
-          traces.skipped_in_job_registered = ScoutApm::Instruments::Stacks.skipped_in_job_registered
-          traces.skipped_in_not_running = ScoutApm::Instruments::Stacks.skipped_in_not_running
-        end
-      end
-    end
-
-
     ######################################
     # Debugging Helpers
     ######################################
@@ -200,6 +158,7 @@ module ScoutApm
         map { |child| child.total_call_time }.
         inject(0) { |sum, time| sum + time }
     end
+    private :child_time
 
     ######################################
     # Allocation Calculations
@@ -225,5 +184,6 @@ module ScoutApm
         map { |child| child.total_allocations }.
         inject(0) { |sum, obj| sum + obj }
     end
+    private :child_allocations
   end
 end