scout_apm 2.6.10 → 3.0.0.pre0

data/lib/scout_apm/layer_converters/depth_first_walker.rb

@@ -5,42 +5,30 @@ module ScoutApm
 
       def initialize(root_layer)
         @root_layer = root_layer
-
-        @on_blocks = []
-        @before_blocks = []
-        @after_blocks = []
       end
 
       def before(&block)
-        @before_blocks << block
+        @before_block = block
       end
 
       def after(&block)
-        @after_blocks << block
-      end
-
-      def on(&block)
-        @on_blocks << block
+        @after_block = block
       end
 
-      def walk(layer=root_layer)
+      def walk(layer=root_layer, &block)
         # Need to run this for the root layer the first time through.
         if layer == root_layer
-          @before_blocks.each{|b| b.call(layer) }
-          @on_blocks.each{|b| b.call(layer) }
+          @before_block.call(layer) if @before_block
+          yield layer
+          @after_block.call(layer) if @after_block
         end
 
         layer.children.each do |child|
-          @before_blocks.each{|b| b.call(child) }
-          @on_blocks.each{|b| b.call(child) }
-          walk(child)
-          @after_blocks.each{|b| b.call(child) }
+          @before_block.call(child) if @before_block
+          yield child
+          walk(child, &block)
+          @after_block.call(child) if @after_block
         end
-
-        if layer == root_layer
-          @after_blocks.each{|b| b.call(layer) }
-        end
-
         nil
       end
     end

data/lib/scout_apm/layer_converters/error_converter.rb

@@ -1,19 +1,19 @@
 module ScoutApm
   module LayerConverters
     class ErrorConverter < ConverterBase
-      def record!
+      def call
+        scope = scope_layer
+
         # Should we mark a request as errored out if a middleware raises?
         # How does that interact w/ a tool like Sentry or Honeybadger?
-        return unless scope_layer
-        return unless request.error?
+        return {} unless scope
+        return {} unless request.error?
 
-        meta = MetricMeta.new("Errors/#{scope_layer.legacy_metric_name}", {})
+        meta = MetricMeta.new("Errors/#{scope.legacy_metric_name}", {})
         stat = MetricStats.new
         stat.update!(1)
-        metrics = { meta => stat }
-        
-        @store.track!(metrics)
-        metrics # this result must be returned so it can be accessed by transaction callback extensions
+
+        { meta => stat }
       end
     end
   end

data/lib/scout_apm/layer_converters/job_converter.rb

@@ -10,20 +10,54 @@
 module ScoutApm
   module LayerConverters
     class JobConverter < ConverterBase
-      attr_reader :meta_options
-
-      def register_hooks(walker)
+      def call
         return unless request.job?
 
-        super
+        JobRecord.new(
+          queue_layer.name,
+          job_layer.name,
+          job_layer.total_call_time,
+          job_layer.total_exclusive_time,
+          errors,
+          create_metrics
+        )
+      end
+
+      def queue_layer
+        @queue_layer ||= find_first_layer_of_type("Queue")
+      end
+
+      def job_layer
+        @job_layer ||= find_first_layer_of_type("Job")
+      end
+
+      def errors
+        if request.error?
+          1
+        else
+          0
+        end
+      end
+
+      def find_first_layer_of_type(layer_type)
+        walker.walk do |layer|
+          return layer if layer.type == layer_type
+        end
+      end
+
+      # Full metrics from this request. These get aggregated in Store for the
+      # overview metrics, or stored permanently in a SlowTransaction
+      # Some merging of metrics will happen here, so if a request calls the same
+      # ActiveRecord or View repeatedly, it'll get merged.
+      def create_metrics
+        metric_hash = Hash.new
 
-        @metrics = Hash.new
-        @meta_options = {:scope => layer_finder.job.legacy_metric_name}
+        meta_options = {:scope => job_layer.legacy_metric_name}
 
-        walker.on do |layer|
-          next if layer == layer_finder.job
-          next if layer == layer_finder.queue
-          next if skip_layer?(layer)
+        walker.walk do |layer|
+          next if layer == job_layer
+          next if layer == queue_layer
+          next if layer.annotations[:ignorable]
 
           # we don't need to use the full metric name for scoped metrics as we
           # only display metrics aggregrated by type, just use "ActiveRecord"
@@ -31,41 +65,20 @@ module ScoutApm
           metric_name = layer.type
 
           meta = MetricMeta.new(metric_name, meta_options)
-          @metrics[meta] ||= MetricStats.new( meta_options.has_key?(:scope) )
+          metric_hash[meta] ||= MetricStats.new( meta_options.has_key?(:scope) )
 
-          stat = @metrics[meta]
+          stat = metric_hash[meta]
           stat.update!(layer.total_call_time, layer.total_exclusive_time)
         end
 
-      end
-
-      def record!
-        return unless request.job?
-
-        errors = request.error? ? 1 : 0
-        add_latency_metric!
-
-        record = JobRecord.new(
-          layer_finder.queue.name,
-          layer_finder.job.name,
-          layer_finder.job.total_call_time,
-          layer_finder.job.total_exclusive_time,
-          errors,
-          @metrics
-        )
-
-        @store.track_job!(record)
-        record  # this result must be returned so it can be accessed by transaction callback extensions
-      end
-
-      # This isn't stored as a specific layer, so grabbing it doesn't use the
-      # walker callbacks
-      def add_latency_metric!
+        # Add the latency metric, which wasn't stored as a distinct layer
         latency = request.annotations[:queue_latency] || 0
         meta = MetricMeta.new("Latency", meta_options)
         stat = MetricStats.new
         stat.update!(latency)
-        @metrics[meta] = stat
+        metric_hash[meta] = stat
+
+        metric_hash
       end
     end
   end

data/lib/scout_apm/layer_converters/metric_converter.rb

@@ -1,22 +1,27 @@
 # Take a TrackedRequest and turn it into a hash of:
 #   MetricMeta => MetricStats
-
-# Full metrics from this request. These get aggregated in Store for the
-# overview metrics, or stored permanently in a SlowTransaction
-# Some merging of metrics will happen here, so if a request calls the same
-# ActiveRecord or View repeatedly, it'll get merged.
 module ScoutApm
   module LayerConverters
     class MetricConverter < ConverterBase
-      def register_hooks(walker)
-        super
+      def call
+        scope = scope_layer
+
+        # TODO: Track requests that never reach a Controller (for example, when
+        # Middleware decides to return rather than passing onward)
+        return {} unless scope
 
-        @metrics = {}
+        create_metrics
+      end
 
-        return unless scope_layer
+      # Full metrics from this request. These get aggregated in Store for the
+      # overview metrics, or stored permanently in a SlowTransaction
+      # Some merging of metrics will happen here, so if a request calls the same
+      # ActiveRecord or View repeatedly, it'll get merged.
+      def create_metrics
+        metric_hash = Hash.new
 
-        walker.on do |layer|
-          next if skip_layer?(layer)
+        walker.walk do |layer|
+          next if layer.annotations[:ignorable]
 
           meta_options = if layer == scope_layer # We don't scope the controller under itself
                           {}
@@ -29,16 +34,12 @@ module ScoutApm
           metric_name = meta_options.has_key?(:scope) ? layer.type : layer.legacy_metric_name
 
           meta = MetricMeta.new(metric_name, meta_options)
-          @metrics[meta] ||= MetricStats.new( meta_options.has_key?(:scope) )
+          metric_hash[meta] ||= MetricStats.new( meta_options.has_key?(:scope) )
 
-          stat = @metrics[meta]
+          stat = metric_hash[meta]
           stat.update!(layer.total_call_time, layer.total_exclusive_time)
         end
-      end
-
-      def record!
-        @store.track!(@metrics)
-        @metrics  # this result must be returned so it can be accessed by transaction callback extensions
+        metric_hash
       end
     end
   end

data/lib/scout_apm/layer_converters/request_queue_time_converter.rb

@@ -4,38 +4,38 @@ module ScoutApm
 
       HEADERS = %w(X-Queue-Start X-Request-Start X-QUEUE-START X-REQUEST-START x-queue-start x-request-start)
 
-      def headers
-        request.headers
+      # Headers is a hash of request headers.  In Rails, request.headers would be appropriate
+      def initialize(request)
+        super(request)
+        @headers = request.headers
       end
 
-      def record!
-        return unless request.web?
-
-        return unless headers
+      def call
+        return {} unless headers
 
         raw_start = locate_timestamp
-        return unless raw_start
+        return {} unless raw_start
 
         parsed_start = parse(raw_start)
-        return unless parsed_start
+        return {} unless parsed_start
 
         request_start = root_layer.start_time
         queue_time = (request_start - parsed_start).to_f
 
         # If we end up with a negative value, just bail out and don't report anything
-        return if queue_time < 0
+        return {} if queue_time < 0
 
         meta = MetricMeta.new("QueueTime/Request", {:scope => scope_layer.legacy_metric_name})
         stat = MetricStats.new(true)
         stat.update!(queue_time)
-        metrics = { meta => stat }
-        
-        @store.track!(metrics)
-        metrics  # this result must be returned so it can be accessed by transaction callback extensions
+
+        { meta => stat }
       end
 
       private
 
+      attr_reader :headers
+
       # Looks through the possible headers with this data, and extracts the raw
       # value of the header
       # Returns nil if not found

data/lib/scout_apm/layer_converters/slow_job_converter.rb

@@ -1,49 +1,42 @@
-# Uses a different workflow than normal metrics. We ignore the shared walk of
-# the layer tree, and instead wait until we're sure we even want to do any
-# work. Only then do we go realize all the SlowJobRecord & metrics associated.
-#
 module ScoutApm
   module LayerConverters
     class SlowJobConverter < ConverterBase
-      ###################
-      #  Converter API  #
-      ###################
-      def record!
-        return nil unless request.job?
-        @points = context.slow_job_policy.score(request)
-
-        # Let the store know we're here, and if it wants our data, it will call
-        # back into #call
-        @store.track_slow_job!(self)
+      def initialize(*)
+        @backtraces = []
+        super
+
+        # After call to super, so @request is populated
+        @points = if request.job?
+                    ScoutApm::Agent.instance.slow_job_policy.score(request)
+                  else
+                    -1
+                  end
+      end
 
-        nil # not returning anything in the layer results ... not used
+      def name
+        request.unique_name
       end
 
-      #####################
-      #  ScoreItemSet API #
-      #####################
-      def name; request.unique_name; end
-      def score; @points; end
+      def score
+        @points
+      end
 
-      # Called by the set to force this to actually be created.
       def call
         return nil unless request.job?
-        return nil unless layer_finder.queue
-        return nil unless layer_finder.job
+        return nil unless queue_layer
+        return nil unless job_layer
 
-        context.slow_job_policy.stored!(request)
+        ScoutApm::Agent.instance.slow_job_policy.stored!(request)
 
         # record the change in memory usage
-        mem_delta = ScoutApm::Instruments::Process::ProcessMemory.new(context).rss_to_mb(request.capture_mem_delta!)
+        mem_delta = ScoutApm::Instruments::Process::ProcessMemory.rss_to_mb(request.capture_mem_delta!)
 
         timing_metrics, allocation_metrics = create_metrics
-
         unless ScoutApm::Instruments::Allocations::ENABLED
           allocation_metrics = {}
         end
 
         SlowJobRecord.new(
-          context,
           queue_layer.name,
           job_layer.name,
           root_layer.stop_time,
@@ -54,48 +47,119 @@ module ScoutApm
           allocation_metrics,
           mem_delta,
           job_layer.total_allocations,
-          score,
-          limited?,
-          span_trace
-        )
+          score)
       end
 
-      def create_metrics
-        # Create a new walker, and wire up the subscope stuff
-        walker = LayerConverters::DepthFirstWalker.new(self.root_layer)
-        register_hooks(walker)
+      def queue_layer
+        @queue_layer ||= find_first_layer_of_type("Queue")
+      end
 
+      def job_layer
+        @job_layer ||= find_first_layer_of_type("Job")
+      end
+
+      def create_metrics
         metric_hash = Hash.new
         allocation_metric_hash = Hash.new
 
-        walker.on do |layer|
-          next if skip_layer?(layer)
+        # Keep a list of subscopes, but only ever use the front one.  The rest
+        # get pushed/popped in cases when we have many levels of subscopable
+        # layers.  This lets us push/pop without otherwise keeping track very closely.
+        subscope_layers = []
+
+        walker.before do |layer|
+          if layer.subscopable?
+            subscope_layers.push(layer)
+          end
+        end
+
+        walker.after do |layer|
+          if layer.subscopable?
+            subscope_layers.pop
+          end
+        end
+
+        walker.walk do |layer|
+          # Sometimes we start capturing a layer without knowing if we really
+          # want to make an entry for it.  See ActiveRecord instrumentation for
+          # an example. We start capturing before we know if a query is cached
+          # or not, and want to skip any cached queries.
+          next if layer.annotations[:ignorable]
 
           # The queue_layer is useful to capture for other reasons, but doesn't
           # create a MetricMeta/Stat of its own
           next if layer == queue_layer
 
-          store_specific_metric(layer, metric_hash, allocation_metric_hash)
-          store_aggregate_metric(layer, metric_hash, allocation_metric_hash)
-        end
-
-        # And now run through the walk we just defined
-        walker.walk
+          meta_options = if subscope_layers.first && layer != subscope_layers.first # Don't scope under ourself.
+                           subscope_name = subscope_layers.first.legacy_metric_name
+                           {:scope => subscope_name}
+                         elsif layer == job_layer # We don't scope the controller under itself
+                           {}
+                         else
+                           {:scope => job_layer.legacy_metric_name}
+                         end
+
+          # Specific Metric
+          meta_options.merge!(:desc => layer.desc.to_s) if layer.desc
+          meta = MetricMeta.new(layer.legacy_metric_name, meta_options)
+          meta.extra.merge!(layer.annotations)
+
+          if layer.backtrace
+            bt = ScoutApm::Utils::BacktraceParser.new(layer.backtrace).call
+            if bt.any? # we could walk thru the call stack and not find in-app code
+              meta.backtrace = bt
+              # Why not just call meta.backtrace and call it done? The walker could access a later later that generates the same MetricMeta but doesn't have a backtrace. This could be
+              # lost in the metric_hash if it is replaced by the new key.
+              @backtraces << meta
+            else
+              ScoutApm::Agent.instance.logger.debug { "Unable to capture an app-specific backtrace for #{meta.inspect}\n#{layer.backtrace}" }
+            end
+          end
+
+          metric_hash[meta] ||= MetricStats.new( meta_options.has_key?(:scope) )
+          allocation_metric_hash[meta] ||= MetricStats.new( meta_options.has_key?(:scope) )
+          stat = metric_hash[meta]
+          stat.update!(layer.total_call_time, layer.total_exclusive_time)
+          stat = allocation_metric_hash[meta]
+          stat.update!(layer.total_allocations, layer.total_exclusive_allocations)
+
+          # Merged Metric (no specifics, just sum up by type)
+          meta = MetricMeta.new("#{layer.type}/all")
+          metric_hash[meta] ||= MetricStats.new(false)
+          allocation_metric_hash[meta] ||= MetricStats.new(false)
+          stat = metric_hash[meta]
+          stat.update!(layer.total_call_time, layer.total_exclusive_time)
+          stat = allocation_metric_hash[meta]
+          stat.update!(layer.total_allocations, layer.total_exclusive_allocations)
+
+          stat.add_traces(layer.traces.as_json)
+
+          # Debug logging for scoutprof traces
+          if ScoutApm::Agent.instance.config.value('profile')
+            if layer.type =~ %r{^(Controller|Queue|Job)$}.freeze
+              ScoutApm::Agent.instance.logger.debug do
+                traces_inspect = layer.traces.inspect
+                "****** Slow Request #{layer.type} Traces (#{layer.name}, tet: #{layer.total_exclusive_time}, tct: #{layer.total_call_time}), total raw traces: #{layer.traces.cube.total_count}, total clean traces: #{layer.traces.total_count}, skipped gc: #{layer.traces.skipped_in_gc}, skipped handler: #{layer.traces.skipped_in_handler}, skipped registered #{layer.traces.skipped_in_job_registered}, skipped not_running #{layer.traces.skipped_in_not_running}:\n#{traces_inspect}"
+              end
+            end
+          else
+            if layer.type =~ %r{^(Controller|Queue|Job)$}.freeze
+              ScoutApm::Agent.instance.logger.debug "****** Slow Request #{layer.type} Traces: Scoutprof is not enabled"
+            end
+          end
+        end # walker.walk
 
         metric_hash = attach_backtraces(metric_hash)
         allocation_metric_hash = attach_backtraces(allocation_metric_hash)
 
-        [metric_hash, allocation_metric_hash]
+        [metric_hash,allocation_metric_hash]
       end
 
-      def skip_layer?(layer); super(layer) || layer == queue_layer; end
-      def queue_layer; layer_finder.queue; end
-      def job_layer; layer_finder.job; end
-
-      def span_trace
-        ScoutApm::LayerConverters::TraceConverter.
-          new(@context, @request, @layer_finder, @store).
-          call
+      def attach_backtraces(metric_hash)
+        @backtraces.each do |meta_with_backtrace|
+          metric_hash.keys.find { |k| k == meta_with_backtrace }.backtrace = meta_with_backtrace.backtrace
+        end
+        metric_hash
       end
     end
   end