scout_apm 2.2.0.pre3 → 2.3.0.pre

data/lib/scout_apm/layer_converters/slow_request_converter.rb

@@ -1,31 +1,29 @@
 module ScoutApm
   module LayerConverters
     class SlowRequestConverter < ConverterBase
-      def initialize(*)
-        @backtraces = [] # An Array of MetricMetas that have a backtrace
-        super
-
-        # After call to super, so @request is populated
-        @points = if request.web?
-                    ScoutApm::Agent.instance.slow_request_policy.score(request)
-                  else
-                    -1
-                  end
+      ###################
+      #  Converter API  #
+      ###################
+      def record!
+        return nil unless request.web?
+        @points = ScoutApm::Agent.instance.slow_request_policy.score(request)
+
+        # Let the store know we're here, and if it wants our data, it will call
+        # back into #call
+        @store.track_slow_transaction!(self)
       end
 
-      def name
-        request.unique_name
-      end
-
-      def score
-        @points
-      end
+      #####################
+      #  ScoreItemSet API #
+      #####################
+      def name; request.unique_name; end
+      def score; @points; end
 
       # Unconditionally attempts to convert this into a SlowTransaction object.
       # Can return nil if the request didn't have any scope_layer.
       def call
-        scope = scope_layer
-        return nil unless scope
+        return nil unless request.web?
+        return nil unless scope_layer
 
         ScoutApm::Agent.instance.slow_request_policy.stored!(request)
 
@@ -35,12 +33,13 @@ module ScoutApm
         uri = request.annotations[:uri] || ""
 
         timing_metrics, allocation_metrics = create_metrics
+
         unless ScoutApm::Instruments::Allocations::ENABLED
           allocation_metrics = {}
         end
 
         SlowTransaction.new(uri,
-                            scope.legacy_metric_name,
+                            scope_layer.legacy_metric_name,
                             root_layer.total_call_time,
                             timing_metrics,
                             allocation_metrics,
@@ -49,119 +48,36 @@ module ScoutApm
                             [], # stackprof, now unused.
                             mem_delta,
                             root_layer.total_allocations,
-                            @points)
-      end
-
-      # Iterates over the TrackedRequest's MetricMetas that have backtraces and attaches each to correct MetricMeta in the Metric Hash.
-      def attach_backtraces(metric_hash)
-        @backtraces.each do |meta_with_backtrace|
-          metric_hash.keys.find { |k| k == meta_with_backtrace }.backtrace = meta_with_backtrace.backtrace
-        end
-        metric_hash
+                            @points,
+                            limited?)
       end
 
       # Full metrics from this request. These get stored permanently in a SlowTransaction.
       # Some merging of metrics will happen here, so if a request calls the same
       # ActiveRecord or View repeatedly, it'll get merged.
-      # 
+      #
       # This returns a 2-element of Metric Hashes (the first element is timing metrics, the second element is allocation metrics)
       def create_metrics
+        # Create a new walker, and wire up the subscope stuff
+        walker = LayerConverters::DepthFirstWalker.new(self.root_layer)
+        register_hooks(walker)
+
         metric_hash = Hash.new
         allocation_metric_hash = Hash.new
 
-        # Keep a list of subscopes, but only ever use the front one.  The rest
-        # get pushed/popped in cases when we have many levels of subscopable
-        # layers.  This lets us push/pop without otherwise keeping track very closely.
-        subscope_layers = []
-
-        walker.before do |layer|
-          if layer.subscopable?
-            subscope_layers.push(layer)
-          end
-        end
-
-        walker.after do |layer|
-          if layer.subscopable?
-            subscope_layers.pop
-          end
+        walker.on do |layer|
+          next if skip_layer?(layer)
+          store_specific_metric(layer, metric_hash, allocation_metric_hash)
+          store_aggregate_metric(layer, metric_hash, allocation_metric_hash)
         end
 
-        walker.walk do |layer|
-          # Sometimes we start capturing a layer without knowing if we really
-          # want to make an entry for it.  See ActiveRecord instrumentation for
-          # an example. We start capturing before we know if a query is cached
-          # or not, and want to skip any cached queries.
-          if layer.annotations[:ignorable]
-            next
-          end
-
-          meta_options = if subscope_layers.first && layer != subscope_layers.first # Don't scope under ourself.
-                           subscope_name = subscope_layers.first.legacy_metric_name
-                           {:scope => subscope_name}
-                         elsif layer == scope_layer # We don't scope the controller under itself
-                           {}
-                         else
-                           {:scope => scope_layer.legacy_metric_name}
-                         end
-
-          # Specific Metric
-          meta_options.merge!(:desc => layer.desc.to_s) if layer.desc
-          meta = MetricMeta.new(layer.legacy_metric_name, meta_options)
-          meta.extra.merge!(layer.annotations)
-          if layer.backtrace
-            bt = ScoutApm::Utils::BacktraceParser.new(layer.backtrace).call
-            if bt.any? # we could walk thru the call stack and not find in-app code
-              meta.backtrace = bt
-              # Why not just call meta.backtrace and call it done? The walker
-              # could access a later later that generates the same MetricMeta
-              # but doesn't have a backtrace. This could be lost in the
-              # metric_hash if it is replaced by the new key.
-              @backtraces << meta
-            else
-              ScoutApm::Agent.instance.logger.debug { "Unable to capture an app-specific backtrace for #{meta.inspect}\n#{layer.backtrace}" }
-            end
-          end
-          metric_hash[meta] ||= MetricStats.new( meta_options.has_key?(:scope) )
-          allocation_metric_hash[meta] ||= MetricStats.new( meta_options.has_key?(:scope) )
-          # timing
-          stat = metric_hash[meta]
-          stat.update!(layer.total_call_time, layer.total_exclusive_time)
-          stat.add_traces(layer.traces.as_json)
-
-          # Debug logging for scoutprof traces
-          if ScoutApm::Agent.instance.config.value('profile')
-            if layer.type =~ %r{^(Controller|Queue|Job)$}.freeze
-              ScoutApm::Agent.instance.logger.debug do
-                traces_inspect = layer.traces.inspect
-                "****** Slow Request #{layer.type} Traces (#{layer.name}, tet: #{layer.total_exclusive_time}, tct: #{layer.total_call_time}), total raw traces: #{layer.traces.cube.total_count}, total clean traces: #{layer.traces.total_count}, skipped gc: #{layer.traces.skipped_in_gc}, skipped handler: #{layer.traces.skipped_in_handler}, skipped registered #{layer.traces.skipped_in_job_registered}, skipped not_running #{layer.traces.skipped_in_not_running}:\n#{traces_inspect}"
-              end
-            end
-          else
-            if layer.type =~ %r{^(Controller|Queue|Job)$}.freeze
-              ScoutApm::Agent.instance.logger.debug "****** Slow Request #{layer.type} Traces: Scoutprof is not enabled"
-            end
-          end
-
-          # allocations
-          stat = allocation_metric_hash[meta]
-          stat.update!(layer.total_allocations, layer.total_exclusive_allocations)
-
-          # Merged Metric (no specifics, just sum up by type)
-          meta = MetricMeta.new("#{layer.type}/all")
-          metric_hash[meta] ||= MetricStats.new(false)
-          allocation_metric_hash[meta] ||= MetricStats.new(false)
-          # timing
-          stat = metric_hash[meta]
-          stat.update!(layer.total_call_time, layer.total_exclusive_time)
-          # allocations
-          stat = allocation_metric_hash[meta]
-          stat.update!(layer.total_allocations, layer.total_exclusive_allocations)
-        end
+        # And now run through the walk we just defined
+        walker.walk
 
         metric_hash = attach_backtraces(metric_hash)
         allocation_metric_hash = attach_backtraces(allocation_metric_hash)
 
-        [metric_hash,allocation_metric_hash]
+        [metric_hash, allocation_metric_hash]
       end
     end
   end

data/lib/scout_apm/limited_layer.rb

@@ -0,0 +1,126 @@
+module ScoutApm
+  # A LimitedLayer is a lossy-compression approach to fall back on once we max out
+  # the number of detailed layer objects we store.  See LayerChildrenSet for the
+  # logic on when that change over happens
+  #
+  # QUESTION: What do we do if we attempt to merge an item that has children?
+  class LimitedLayer
+    attr_reader :type
+
+    def initialize(type)
+      @type = type
+
+      @total_call_time = 0
+      @total_exclusive_time = 0
+      @total_allocations = 0
+      @total_exclusive_allocations = 0
+      @total_layers = 0
+    end
+
+    def absorb(layer)
+      @total_layers += 1
+
+      @total_call_time += layer.total_call_time
+      @total_exclusive_time += layer.total_exclusive_time
+
+      @total_allocations += layer.total_allocations
+      @total_exclusive_allocations += layer.total_exclusive_allocations
+    end
+
+    def total_call_time
+      @total_call_time
+    end
+
+    def total_exclusive_time
+      @total_exclusive_time
+    end
+
+    def total_allocations
+      @total_allocations
+    end
+
+    def total_exclusive_allocations
+      @total_exclusive_allocations
+    end
+
+    def count
+      @total_layers
+    end
+
+    # This is the old style name. This function is used for now, but should be
+    # removed, and the new type & name split should be enforced through the
+    # app.
+    def legacy_metric_name
+      "#{type}/Limited"
+    end
+
+    def children
+      Set.new
+    end
+
+    def annotations
+      nil
+    end
+
+    def to_s
+      "<LimitedLayer type=#{type} count=#{count}>"
+    end
+
+    def limited?
+      true
+    end
+
+    ######################################################
+    #  Stub out some methods with static default values  #
+    ######################################################
+    def subscopable?
+      false
+    end
+
+    def desc
+      nil
+    end
+
+    def backtrace
+      nil
+    end
+
+
+    #######################################################################
+    #  Many methods don't make any sense on a limited layer. Raise errors  #
+    #  aggressively for now to detect mistaken calls                      #
+    #######################################################################
+
+    def add_child
+      raise "Should never call add_child on a limited_layer"
+    end
+
+    def record_stop_time!(*)
+      raise "Should never call record_stop_time! on a limited_layer"
+    end
+
+    def record_allocations!
+      raise "Should never call record_allocations! on a limited_layer"
+    end
+
+    def desc=(*)
+      raise "Should never call desc on a limited_layer"
+    end
+
+    def annotate_layer(*)
+      raise "Should never call annotate_layer on a limited_layer"
+    end
+
+    def subscopable!
+      raise "Should never call subscopable! on a limited_layer"
+    end
+
+    def capture_backtrace!
+      raise "Should never call capture_backtrace on a limited_layer"
+    end
+
+    def caller_array
+      raise "Should never call caller_array on a limited_layer"
+    end
+  end
+end

data/lib/scout_apm/metric_meta.rb

@@ -33,11 +33,6 @@ class MetricMeta
     !!(metric_name =~ /\A(Controller|Job)\//)
   end
 
-  # To avoid conflicts with different JSON libaries
-  def to_json(*a)
-     %Q[{"metric_id":#{metric_id || 'null'},"metric_name":#{metric_name.to_json},"scope":#{scope.to_json || 'null'}}]
-  end
-
   def ==(o)
     self.eql?(o)
   end

data/lib/scout_apm/metric_set.rb

@@ -1,6 +1,8 @@
 module ScoutApm
   class MetricSet
-    # We can't aggregate CPU, Memory, Capacity, or Controller, so pass through these metrics directly
+    # We can't aggregate a handful of things like samplers (CPU, Memory), or
+    # Controller, and Percentiles so pass through these metrics directly
+    #
     # TODO: Figure out a way to not have this duplicate what's in Samplers, and also on server's ingest
     PASSTHROUGH_METRICS = ["CPU", "Memory", "Instance", "Controller", "SlowTransaction", "Percentile", "Job"]
 
@@ -49,5 +51,11 @@ module ScoutApm
       @combine_in_progress = false
       self
     end
+
+
+    def eql?(other)
+      metrics == other.metrics
+    end
+    alias :== :eql?
   end
 end

data/lib/scout_apm/metric_stats.rb

@@ -9,7 +9,6 @@ class MetricStats
   attr_accessor :sum_of_squares
   attr_accessor :queue
   attr_accessor :latency
-  attr_accessor :traces
 
   def initialize(scoped = false)
     @scoped = scoped
@@ -19,7 +18,6 @@ class MetricStats
     self.min_call_time = 0.0
     self.max_call_time = 0.0
     self.sum_of_squares = 0.0
-    self.traces = []
   end
 
   # Note, that you must include exclusive_time if you wish to set
@@ -41,10 +39,6 @@ class MetricStats
     self
   end
 
-  def add_traces(traces)
-    self.traces += Array(traces)
-  end
-
   # combines data from another MetricStats object
   def combine!(other)
     self.call_count += other.call_count
@@ -53,12 +47,17 @@ class MetricStats
     self.min_call_time = other.min_call_time if self.min_call_time.zero? or other.min_call_time < self.min_call_time
     self.max_call_time = other.max_call_time if other.max_call_time > self.max_call_time
     self.sum_of_squares += other.sum_of_squares
-    self.traces = Array(self.traces) + Array(other.traces)
     self
   end
 
   def as_json
-    json_attributes = [:call_count, :total_call_time, :total_exclusive_time, :min_call_time, :max_call_time, :traces]
+    json_attributes = [
+      :call_count,
+      :max_call_time,
+      :min_call_time,
+      :total_call_time,
+      :total_exclusive_time,
+    ]
     ScoutApm::AttributeArranger.call(self, json_attributes)
   end
 end

data/lib/scout_apm/rack.rb

@@ -0,0 +1,26 @@
+module ScoutApm
+  module Rack
+    def self.install!
+      ScoutApm::Agent.instance.start(:skip_app_server_check => true)
+      ScoutApm::Agent.instance.start_background_worker
+    end
+
+    def self.transaction(endpoint_name, env)
+      req = ScoutApm::RequestManager.lookup
+      req.annotate_request(:uri => env["REQUEST_PATH"]) rescue nil
+      req.context.add_user(:ip => env["REMOTE_ADDR"]) rescue nil
+
+      req.web!
+      req.start_layer(ScoutApm::Layer.new('Controller', endpoint_name))
+
+      begin
+        yield
+      rescue
+        req.error!
+        raise
+      ensure
+        req.stop_layer
+      end
+    end
+  end
+end

data/lib/scout_apm/remote/message.rb

@@ -0,0 +1,23 @@
+module ScoutApm
+  module Remote
+    class Message
+      attr_reader :type
+      attr_reader :command
+      attr_reader :args
+
+      def initialize(type, command, *args)
+        @type = type
+        @command = command
+        @args = args
+      end
+
+      def self.decode(msg)
+        Marshal.load(msg)
+      end
+
+      def encode
+        Marshal.dump(self)
+      end
+    end
+  end
+end