scout_apm 2.6.10 → 3.0.0.pre0

data/lib/scout_apm/layaway.rb

@@ -7,24 +7,23 @@
 #
 module ScoutApm
   class Layaway
+    # How old a file needs to be in Seconds before it gets reported.
+    REPORTING_AGE = 120
+
     # How long to let a stale file sit before deleting it.
     # Letting it sit a bit may be useful for debugging
     STALE_AGE = 10 * 60
 
-    # Failsafe to prevent writing layaway files if for some reason they are not being cleaned up
-    MAX_FILES_LIMIT = 5000
-
     # A strftime format string for how we render timestamps in filenames.
     # Must be sortable as an integer
     TIME_FORMAT = "%Y%m%d%H%M"
 
-    attr_reader :context
-    def initialize(context)
-      @context = context
-    end
+    attr_accessor :config
+    attr_reader :environment
 
-    def logger
-      context.logger
+    def initialize(config, environment)
+      @config = config
+      @environment = environment
     end
 
     # Returns a Pathname object with the fully qualified directory where the layaway files can be placed.
@@ -35,54 +34,33 @@ module ScoutApm
     def directory
       return @directory if @directory
 
-      data_file = context.config.value("data_file")
+      data_file = config.value("data_file")
       data_file = File.dirname(data_file) if data_file && !File.directory?(data_file)
 
       candidates = [
         data_file,
-        "#{context.environment.root}/tmp",
+        "#{environment.root}/tmp",
         "/tmp"
       ].compact
 
       found = candidates.detect { |dir| File.writable?(dir) }
-      logger.debug("Storing Layaway Files in #{found}")
+      ScoutApm::Agent.instance.logger.debug("Storing Layaway Files in #{found}")
       @directory = Pathname.new(found)
     end
 
-    def write_reporting_period(reporting_period, files_limit = MAX_FILES_LIMIT)
-      if at_layaway_file_limit?(files_limit)
-        # This will happen constantly once we hit this case, so only log the first time
-        @wrote_layaway_limit_error_message ||= logger.error("Layaway: Hit layaway file limit. Not writing to layaway file")
-        return false
-      end
-      logger.debug("Layaway: wrote time period: #{reporting_period.timestamp}")
+    def write_reporting_period(reporting_period)
       filename = file_for(reporting_period.timestamp)
-      layaway_file = LayawayFile.new(context, filename)
+      layaway_file = LayawayFile.new(filename)
       layaway_file.write(reporting_period)
-    rescue => e
-      logger.debug("Layaway: error writing: #{e.message}, #{e.backtrace.inspect}")
-      raise e
     end
 
-    # Claims a given timestamp by getting an exclusive lock on a timestamped
-    # coordinator file. The coordinator file never contains data, it's just a
-    # syncronization mechanism.
-    #
-    # Once the 'claim' is obtained:
-    #   * load and yield each ReportingPeriod from the layaway files.
-    #   * if there are reporting periods:
-    #     * yields any ReportingPeriods collected up from all the files.
-    #     * deletes all of the layaway files (including the coordinator) for the timestamp
-    #   * if not
-    #     * delete the coordinator
-    #   * remove any stale layaway files that may be hanging around.
-    #   * Finally unlock and ensure the coordinator file is cleared.
-    #
-    # If a claim file can't be obtained, return false without doing any work
-    # Another process is handling the reporting.
+    # Claims a given timestamp (getting a lock on a particular filename),
+    # then yields ReportingPeriods collected up from all the files.
+    # If the yield returns truthy, delete the layaway files that made it up.
     def with_claim(timestamp)
       coordinator_file = glob_pattern(timestamp, :coordinator)
 
+
       begin
         # This file gets deleted only by a process that successfully created and obtained the exclusive lock
         f = File.open(coordinator_file, File::RDWR | File::CREAT | File::EXCL | File::NONBLOCK)
@@ -92,35 +70,25 @@ module ScoutApm
 
       begin
         if f
-          begin
-            logger.debug("Obtained Reporting Lock")
-
-            log_layaway_file_information
 
-            files = all_files_for(timestamp).reject{|l| l.to_s == coordinator_file.to_s }
-            rps = files.map{ |layaway| LayawayFile.new(context, layaway).load }.compact
-            if rps.any?
-              yield rps
+          ScoutApm::Agent.instance.logger.debug("Obtained Reporting Lock")
 
-              logger.debug("Layaway: Deleting the now-reported files for #{timestamp.to_s}")
-              delete_files_for(timestamp) # also removes the coodinator_file
-            else
-              File.unlink(coordinator_file)
-              logger.debug("Layaway: No files to report")
-            end
+          files = all_files_for(timestamp).reject{|l| l.to_s == coordinator_file.to_s }
+          rps = files.map{ |layaway| LayawayFile.new(layaway).load }.compact
+          if rps.any?
+            yield rps
 
-            logger.debug("Layaway: Checking for any stale files")
+            delete_files_for(timestamp) # also removes the coodinator_file
             delete_stale_files(timestamp.to_time - STALE_AGE)
-
-            true
-          rescue Exception => e
-            logger.debug("Layaway: Caught an exception in with_claim, with the coordination file locked: #{e.message}, #{e.backtrace.inspect}")
-            raise
-          ensure
-            # Unlock the file when done!
-            f.flock(File::LOCK_UN | File::LOCK_NB)
-            f.close
+          else
+            File.unlink(coordinator_file)
+            ScoutApm::Agent.instance.logger.debug("No layaway files to report")
           end
+
+          # Unlock the file when done!
+          f.flock(File::LOCK_UN | File::LOCK_NB)
+          f.close
+          true
         else
           # Didn't obtain lock, another process is reporting. Return false from this function, but otherwise no work
           false
@@ -129,10 +97,7 @@ module ScoutApm
     end
 
     def delete_files_for(timestamp)
-      all_files_for(timestamp).each { |layaway|
-        logger.debug("Layaway: Deleting file: #{layaway}")
-        File.unlink(layaway)
-      }
+      all_files_for(timestamp).each { |layaway| File.unlink(layaway) }
     end
 
     def delete_stale_files(older_than)
@@ -141,10 +106,8 @@ module ScoutApm
         compact.
         uniq.
         select { |timestamp| timestamp.to_i < older_than.strftime(TIME_FORMAT).to_i }.
-          tap  { |timestamps| logger.debug("Layaway: Deleting stale files with timestamps: #{timestamps.inspect}") }.
+          tap  { |timestamps| ScoutApm::Agent.instance.logger.debug("Deleting stale layaway files with timestamps: #{timestamps.inspect}") }.
         map    { |timestamp| delete_files_for(timestamp) }
-    rescue => e
-      logger.debug("Layaway: Problem deleting stale files: #{e.message}, #{e.backtrace.inspect}")
     end
 
     private
@@ -188,30 +151,13 @@ module ScoutApm
     end
 
     def timestamp_from_filename(filename)
-      match = filename.match(%r{scout_(\d+)_\d+\.data\z})
+      match = filename.match(%r{scout_(.*)_.*\.data})
       if match
         match[1]
       else
         nil
       end
     end
-
-    def at_layaway_file_limit?(files_limit = MAX_FILES_LIMIT)
-      all_files_for(:all).count >= files_limit
-    end
-
-    def log_layaway_file_information
-      files_in_temp = Dir["#{directory}/*"].count
-
-      all_filenames = all_files_for(:all)
-      count_per_timestamp = Hash[
-        all_filenames.
-        group_by {|f| timestamp_from_filename(f) }.
-        map{ |timestamp, list| [timestamp, list.length] }
-      ]
-
-      logger.debug("Layaway: Total Files in #{directory}: #{files_in_temp}. Total Layaway Files: #{all_filenames.size}.  By Timestamp: #{count_per_timestamp.inspect}")
-    end
   end
 end
 

data/lib/scout_apm/layaway_file.rb

@@ -2,15 +2,9 @@
 module ScoutApm
   class LayawayFile
     attr_reader :path
-    attr_reader :context
 
-    def initialize(context, path)
+    def initialize(path)
       @path = path
-      @context = context
-    end
-
-    def logger
-      context.logger
     end
 
     def load
@@ -18,8 +12,8 @@ module ScoutApm
       deserialize(data)
     rescue NameError, ArgumentError, TypeError => e
       # Marshal error
-      logger.info("LayawayFile: Unable to load data")
-      logger.debug("#{e.message}, #{e.backtrace.join("\n\t")}")
+      ScoutApm::Agent.instance.logger.info("Unable to load data from Layaway file, resetting.")
+      ScoutApm::Agent.instance.logger.debug("#{e.message}, #{e.backtrace.join("\n\t")}")
       nil
     end
 
@@ -30,10 +24,6 @@ module ScoutApm
 
     def serialize(data)
       Marshal.dump(data)
-    rescue
-      ScoutApm::Agent.instance.logger.info("Failed Marshalling LayawayFile")
-      ScoutApm::Agent.instance.logger.info(ScoutApm::Utils::MarshalLogging.new(data).dive) rescue nil
-      raise
     end
 
     def deserialize(data)

data/lib/scout_apm/layer.rb

@@ -13,17 +13,13 @@ module ScoutApm
     #   instrumentation for an example of how this is useful
     attr_accessor :name
 
-    # An array of children layers
+    # An array of children layers, in call order.
     # For instance, if we are in a middleware, there will likely be only a single
     # child, which is another middleware.  In a Controller, we may have a handful
     # of children: [ActiveRecord, ActiveRecord, View, HTTP Call].
     #
     # This useful to get actual time spent in this layer vs. children time
-    #
-    # TODO: Check callers for compatibility w/ nil to avoid making an empty array
-    def children
-      @children || LayerChildrenSet.new
-    end
+    attr_reader :children
 
     # Time objects recording the start & stop times of this layer
     attr_reader :start_time, :stop_time
@@ -36,42 +32,47 @@ module ScoutApm
 
     # If this layer took longer than a fixed amount of time, store the
     # backtrace of where it occurred.
-    attr_accessor :backtrace
-
-    # The file name associated with the layer. Only used for autoinstruments overhead logging.
-    attr_accessor :file_name
+    attr_reader :backtrace
 
     # As we go through a part of a request, instrumentation can store additional data
     # Known Keys:
     #   :record_count - The number of rows returned by an AR query (From notification instantiation.active_record)
     #   :class_name   - The ActiveRecord class name (From notification instantiation.active_record)
-    #
-    # If no annotations are ever set, this will return nil
     attr_reader :annotations
 
-    attr_reader :allocations_start, :allocations_stop
+    # ScoutProf - trace_index is an index into the Stack structure in the C
+    # code, used to store captured traces.
+    attr_reader :trace_index
+
+    # ScoutProf - frame_index is an optimization to not capture a few frames
+    # during scoutprof instrumentation
+    attr_reader :frame_index
+
+    # Captured backtraces from ScoutProf. This is distinct from the backtrace
+    # attribute, which gets the ruby backtrace of any given layer. StackProf
+    # focuses on Controller layers, and requires a native extension and a
+    # reasonably recent Ruby.
+    attr_reader :traces
 
     BACKTRACE_CALLER_LIMIT = 50 # maximum number of lines to send thru for backtrace analysis
 
     def initialize(type, name, start_time = Time.now)
       @type = type
       @name = name
+      @annotations = {}
       @start_time = start_time
       @allocations_start = ScoutApm::Instruments::Allocations.count
       @allocations_stop = 0
-
-      # initialize these only on first use
-      @children = nil
-      @annotations = nil
+      @children = [] # In order of calls
       @desc = nil
-    end
 
-    def limited?
-      false
+      @traces = ScoutApm::TraceSet.new
+      @raw_frames = []
+      @frame_index = ScoutApm::Instruments::Stacks.current_frame_index # For efficiency sake, try to skip the bottom X frames when collecting traces
+      @trace_index = ScoutApm::Instruments::Stacks.current_trace_index
     end
 
     def add_child(child)
-      @children ||= LayerChildrenSet.new
       @children << child
     end
 
@@ -90,7 +91,6 @@ module ScoutApm
 
     # This data is internal to ScoutApm, to add custom information, use the Context api.
     def annotate_layer(hsh)
-      @annotations ||= {}
       @annotations.merge!(hsh)
     end
 
@@ -102,6 +102,14 @@ module ScoutApm
       @subscopable
     end
 
+    def traced!
+      @traced = true
+    end
+
+    def traced?
+      @traced
+    end
+
     # This is the old style name. This function is used for now, but should be
     # removed, and the new type & name split should be enforced through the
     # app.
@@ -116,13 +124,42 @@ module ScoutApm
     # In Ruby 2.0+, we can pass the range directly to the caller to reduce the memory footprint.
     def caller_array
       # omits the first several callers which are in the ScoutAPM stack.
-      if ScoutApm::Agent.instance.context.environment.ruby_2?
+      if ScoutApm::Environment.instance.ruby_2?
         caller(3...BACKTRACE_CALLER_LIMIT)
       else
         caller[3...BACKTRACE_CALLER_LIMIT]
       end
     end
 
+    # Set the name of the file that this action is coming from.
+    # TraceSet uses this to more accurately filter backtraces
+    def set_root_class(klass_name)
+      @traces.set_root_class(klass_name)
+    end
+
+    def start_sampling
+      if ScoutApm::Agent.instance.config.value('profile') && traced?
+        ScoutApm::Instruments::Stacks.update_indexes(frame_index, trace_index)
+        ScoutApm::Instruments::Stacks.start_sampling
+      else
+        ScoutApm::Instruments::Stacks.stop_sampling(false)
+      end
+    end
+
+    def record_traces!
+      if ScoutApm::Agent.instance.config.value('profile')
+        ScoutApm::Instruments::Stacks.stop_sampling(false)
+        if traced?
+          traces.raw_traces = ScoutApm::Instruments::Stacks.profile_frames
+          traces.skipped_in_gc = ScoutApm::Instruments::Stacks.skipped_in_gc
+          traces.skipped_in_handler = ScoutApm::Instruments::Stacks.skipped_in_handler
+          traces.skipped_in_job_registered = ScoutApm::Instruments::Stacks.skipped_in_job_registered
+          traces.skipped_in_not_running = ScoutApm::Instruments::Stacks.skipped_in_not_running
+        end
+      end
+    end
+
+
     ######################################
     # Debugging Helpers
     ######################################
@@ -163,7 +200,6 @@ module ScoutApm
         map { |child| child.total_call_time }.
         inject(0) { |sum, time| sum + time }
     end
-    private :child_time
 
     ######################################
     # Allocation Calculations
@@ -189,6 +225,5 @@ module ScoutApm
         map { |child| child.total_allocations }.
         inject(0) { |sum, obj| sum + obj }
     end
-    private :child_allocations
   end
 end

data/lib/scout_apm/layer_converters/allocation_metric_converter.rb

@@ -1,17 +1,16 @@
 module ScoutApm
   module LayerConverters
     class AllocationMetricConverter < ConverterBase
-      def record!
-        return unless scope_layer
-        return unless ScoutApm::Instruments::Allocations::ENABLED
+      def call
+        scope = scope_layer
+        return {} unless scope
+        return {} unless ScoutApm::Instruments::Allocations::ENABLED
 
-        meta = MetricMeta.new("ObjectAllocations", {:scope => scope_layer.legacy_metric_name})
+        meta = MetricMeta.new("ObjectAllocations", {:scope => scope.legacy_metric_name})
         stat = MetricStats.new
         stat.update!(root_layer.total_allocations)
-        metrics = { meta => stat }
 
-        @store.track!(metrics)
-        nil # not returning anything in the layer results ... not used
+        { meta => stat }
       end
     end
   end

data/lib/scout_apm/layer_converters/converter_base.rb

@@ -1,220 +1,31 @@
 module ScoutApm
   module LayerConverters
     class ConverterBase
-
-      attr_reader :context
+      attr_reader :walker
       attr_reader :request
       attr_reader :root_layer
-      attr_reader :layer_finder
 
-      def initialize(context, request, layer_finder, store=nil)
-        @context = context
+      def initialize(request)
         @request = request
-        @layer_finder = layer_finder
-        @store = store
-
         @root_layer = request.root_layer
-        @backtraces = []
-        @limited = false
+        @walker = DepthFirstWalker.new(root_layer)
       end
 
+      # Scope is determined by the first Controller we hit.  Most of the time
+      # there will only be 1 anyway.  But if you have a controller that calls
+      # another controller method, we may pick that up:
+      #     def update
+      #       show
+      #       render :update
+      #     end
       def scope_layer
-        layer_finder.scope
-      end
-
-      ################################################################################
-      # Subscoping
-      ################################################################################
-      #
-      # Keep a list of subscopes, but only ever use the front one.  The rest
-      # get pushed/popped in cases when we have many levels of subscopable
-      # layers.  This lets us push/pop without otherwise keeping track very closely.
-      def register_hooks(walker)
-        @subscope_layers = []
-
-        walker.before do |layer|
-          if layer.subscopable?
-            @subscope_layers.push(layer)
-          end
-        end
-
-        walker.after do |layer|
-          if layer.subscopable?
-            @subscope_layers.pop
-          end
-        end
-      end
-
-      def subscoped?(layer)
-        @subscope_layers.first && layer != @subscope_layers.first # Don't scope under ourself.
-      end
-
-      def subscope_name
-        @subscope_layers.first.legacy_metric_name
-      end
-
-
-      ################################################################################
-      # Backtrace Handling
-      ################################################################################
-      #
-      # Because we get several layers for the same thing if you call an
-      # instrumented thing repeatedly, and only some of them may have
-      # backtraces captured, we store the backtraces off into another spot
-      # during processing, then at the end, we loop over those saved
-      # backtraces, putting them back into the metrics hash.
-      #
-      # This comes up most often when capturing n+1 backtraces. Because the
-      # query may be fast enough to evade our time-limit based backtrace
-      # capture, only the Nth item (see TrackedRequest for more detail) has a
-      # backtrack captured.  This sequence makes sure that we report up that
-      # backtrace in the aggregated set of metrics around that call.
-
-      # Call this as you are processing each layer. It will store off backtraces
-      def store_backtrace(layer, meta)
-        return unless layer.backtrace
-
-        bt = ScoutApm::Utils::BacktraceParser.new(layer.backtrace).call
-        if bt.any?
-          meta.backtrace = bt
-          @backtraces << meta
-        end
+        @scope_layer ||= find_first_layer_of_type("Controller") || find_first_layer_of_type("Job")
       end
 
-      # Call this after you finish walking the layers, and want to take the
-      # set-aside backtraces and place them into the metas they match
-      def attach_backtraces(metric_hash)
-        @backtraces.each do |meta_with_backtrace|
-          metric_hash.keys.find { |k| k == meta_with_backtrace }.backtrace = meta_with_backtrace.backtrace
+      def find_first_layer_of_type(layer_type)
+        walker.walk do |layer|
+          return layer if layer.type == layer_type
         end
-        metric_hash
-      end
-
-
-      ################################################################################
-      # Limit Handling
-      ################################################################################
-
-      # To prevent huge traces from being generated, we should stop collecting
-      # detailed metrics as we go beyond some reasonably large count.
-      #
-      # We should still add up the /all aggregates.
-
-      MAX_METRICS = 500
-
-      def over_metric_limit?(metric_hash)
-        if metric_hash.size > MAX_METRICS
-          @limited = true
-        else
-          false
-        end
-      end
-
-      def limited?
-        !! @limited
-      end
-
-      ################################################################################
-      # Meta Scope
-      ################################################################################
-
-      # When we make MetricMeta records, we need to determine a few things from layer.
-      def make_meta_options(layer)
-        scope_hash = make_meta_options_scope(layer)
-        desc_hash = make_meta_options_desc_hash(layer)
-
-        scope_hash.merge(desc_hash)
-      end
-
-      def make_meta_options_scope(layer)
-        # This layer is scoped under another thing. Typically that means this is a layer under a view.
-        # Like: Controller -> View/users/show -> ActiveRecord/user/find
-        #   in that example, the scope is the View/users/show
-        if subscoped?(layer)
-          {:scope => subscope_name}
-
-        # We don't scope the controller under itself
-        elsif layer == scope_layer
-          {}
-
-        # This layer is a top level metric ("ActiveRecord", or "HTTP" or
-        # whatever, directly under the controller), so scope to the
-        # Controller
-        else
-          {:scope => scope_layer.legacy_metric_name}
-        end
-      end
-
-      def make_meta_options_desc_hash(layer, max_desc_length=32768)
-        if layer.desc
-          desc_s = layer.desc.to_s
-          trimmed_desc = desc_s[0 .. max_desc_length]
-          {:desc => trimmed_desc}
-        else
-          {}
-        end
-      end
-
-
-      ################################################################################
-      # Storing metrics into the hashes
-      ################################################################################
-
-      # This is the detailed metric - type, name, backtrace, annotations, etc.
-      def store_specific_metric(layer, metric_hash, allocation_metric_hash)
-        return false if over_metric_limit?(metric_hash)
-
-        meta_options = make_meta_options(layer)
-
-        meta = MetricMeta.new(layer.legacy_metric_name, meta_options)
-        meta.extra.merge!(layer.annotations) if layer.annotations
-
-        store_backtrace(layer, meta)
-
-        metric_hash[meta] ||= MetricStats.new(meta_options.has_key?(:scope))
-        allocation_metric_hash[meta] ||= MetricStats.new(meta_options.has_key?(:scope))
-
-        # timing
-        stat = metric_hash[meta]
-        stat.update!(layer.total_call_time, layer.total_exclusive_time)
-
-        # allocations
-        stat = allocation_metric_hash[meta]
-        stat.update!(layer.total_allocations, layer.total_exclusive_allocations)
-
-        if LimitedLayer === layer
-          metric_hash[meta].call_count = layer.count
-          allocation_metric_hash[meta].call_count = layer.count
-        end
-      end
-
-      # Merged Metric - no specifics, just sum up by type (ActiveRecord, View, HTTP, etc)
-      def store_aggregate_metric(layer, metric_hash, allocation_metric_hash)
-          meta = MetricMeta.new("#{layer.type}/all")
-
-          metric_hash[meta] ||= MetricStats.new(false)
-          allocation_metric_hash[meta] ||= MetricStats.new(false)
-
-          # timing
-          stat = metric_hash[meta]
-          stat.update!(layer.total_call_time, layer.total_exclusive_time)
-
-          # allocations
-          stat = allocation_metric_hash[meta]
-          stat.update!(layer.total_allocations, layer.total_exclusive_allocations)
-      end
-
-      ################################################################################
-      # Misc Helpers
-      ################################################################################
-
-      # Sometimes we start capturing a layer without knowing if we really
-      # want to make an entry for it.  See ActiveRecord instrumentation for
-      # an example. We start capturing before we know if a query is cached
-      # or not, and want to skip any cached queries.
-      def skip_layer?(layer)
-        return false if layer.annotations.nil?
-        return true  if layer.annotations[:ignorable]
       end
     end
   end