scout_apm 2.1.32 → 2.2.0.pre0

data/lib/scout_apm/layer_converters/job_converter.rb

@@ -57,7 +57,7 @@ module ScoutApm
         walker.walk do |layer|
           next if layer == job_layer
           next if layer == queue_layer
-          next if skip_layer?(layer)
+          next if layer.annotations[:ignorable]
 
           # we don't need to use the full metric name for scoped metrics as we
           # only display metrics aggregrated by type, just use "ActiveRecord"

data/lib/scout_apm/layer_converters/metric_converter.rb

@@ -21,7 +21,7 @@ module ScoutApm
         metric_hash = Hash.new
 
         walker.walk do |layer|
-          next if skip_layer?(layer)
+          next if layer.annotations[:ignorable]
 
           meta_options = if layer == scope_layer # We don't scope the controller under itself
                           {}

data/lib/scout_apm/layer_converters/slow_job_converter.rb

@@ -2,6 +2,7 @@ module ScoutApm
   module LayerConverters
     class SlowJobConverter < ConverterBase
       def initialize(*)
+        @backtraces = []
         super
 
         # After call to super, so @request is populated
@@ -10,8 +11,6 @@ module ScoutApm
                   else
                     -1
                   end
-
-        setup_subscopable_callbacks
       end
 
       def name
@@ -33,7 +32,6 @@ module ScoutApm
         mem_delta = ScoutApm::Instruments::Process::ProcessMemory.rss_to_mb(request.capture_mem_delta!)
 
         timing_metrics, allocation_metrics = create_metrics
-
         unless ScoutApm::Instruments::Allocations::ENABLED
           allocation_metrics = {}
         end
@@ -49,9 +47,7 @@ module ScoutApm
           allocation_metrics,
           mem_delta,
           job_layer.total_allocations,
-          score,
-          limited?
-        )
+          score)
       end
 
       def queue_layer
@@ -62,29 +58,108 @@ module ScoutApm
         @job_layer ||= find_first_layer_of_type("Job")
       end
 
-      def skip_layer?(layer)
-        super(layer) || layer == queue_layer
-      end
-
       def create_metrics
         metric_hash = Hash.new
         allocation_metric_hash = Hash.new
 
+        # Keep a list of subscopes, but only ever use the front one.  The rest
+        # get pushed/popped in cases when we have many levels of subscopable
+        # layers.  This lets us push/pop without otherwise keeping track very closely.
+        subscope_layers = []
+
+        walker.before do |layer|
+          if layer.subscopable?
+            subscope_layers.push(layer)
+          end
+        end
+
+        walker.after do |layer|
+          if layer.subscopable?
+            subscope_layers.pop
+          end
+        end
+
         walker.walk do |layer|
-          next if skip_layer?(layer)
+          # Sometimes we start capturing a layer without knowing if we really
+          # want to make an entry for it.  See ActiveRecord instrumentation for
+          # an example. We start capturing before we know if a query is cached
+          # or not, and want to skip any cached queries.
+          next if layer.annotations[:ignorable]
 
           # The queue_layer is useful to capture for other reasons, but doesn't
           # create a MetricMeta/Stat of its own
           next if layer == queue_layer
 
-          store_specific_metric(layer, metric_hash, allocation_metric_hash)
-          store_aggregate_metric(layer, metric_hash, allocation_metric_hash)
-        end
+          meta_options = if subscope_layers.first && layer != subscope_layers.first # Don't scope under ourself.
+                           subscope_name = subscope_layers.first.legacy_metric_name
+                           {:scope => subscope_name}
+                         elsif layer == job_layer # We don't scope the controller under itself
+                           {}
+                         else
+                           {:scope => job_layer.legacy_metric_name}
+                         end
+
+          # Specific Metric
+          meta_options.merge!(:desc => layer.desc.to_s) if layer.desc
+          meta = MetricMeta.new(layer.legacy_metric_name, meta_options)
+          meta.extra.merge!(layer.annotations)
+
+          if layer.backtrace
+            bt = ScoutApm::Utils::BacktraceParser.new(layer.backtrace).call
+            if bt.any? # we could walk thru the call stack and not find in-app code
+              meta.backtrace = bt
+              # Why not just call meta.backtrace and call it done? The walker could access a later later that generates the same MetricMeta but doesn't have a backtrace. This could be
+              # lost in the metric_hash if it is replaced by the new key.
+              @backtraces << meta
+            else
+              ScoutApm::Agent.instance.logger.debug { "Unable to capture an app-specific backtrace for #{meta.inspect}\n#{layer.backtrace}" }
+            end
+          end
+
+          metric_hash[meta] ||= MetricStats.new( meta_options.has_key?(:scope) )
+          allocation_metric_hash[meta] ||= MetricStats.new( meta_options.has_key?(:scope) )
+          stat = metric_hash[meta]
+          stat.update!(layer.total_call_time, layer.total_exclusive_time)
+          stat = allocation_metric_hash[meta]
+          stat.update!(layer.total_allocations, layer.total_exclusive_allocations)
+
+          # Merged Metric (no specifics, just sum up by type)
+          meta = MetricMeta.new("#{layer.type}/all")
+          metric_hash[meta] ||= MetricStats.new(false)
+          allocation_metric_hash[meta] ||= MetricStats.new(false)
+          stat = metric_hash[meta]
+          stat.update!(layer.total_call_time, layer.total_exclusive_time)
+          stat = allocation_metric_hash[meta]
+          stat.update!(layer.total_allocations, layer.total_exclusive_allocations)
+
+          stat.add_traces(layer.traces.as_json)
+
+          # Debug logging for scoutprof traces
+          if ScoutApm::Agent.instance.config.value('profile')
+            if layer.type =~ %r{^(Controller|Queue|Job)$}.freeze
+              ScoutApm::Agent.instance.logger.debug do
+                traces_inspect = layer.traces.inspect
+                "****** Slow Request #{layer.type} Traces (#{layer.name}, tet: #{layer.total_exclusive_time}, tct: #{layer.total_call_time}), total raw traces: #{layer.traces.cube.total_count}, total clean traces: #{layer.traces.total_count}:\n#{traces_inspect}"
+              end
+            end
+          else
+            if layer.type =~ %r{^(Controller|Queue|Job)$}.freeze
+              ScoutApm::Agent.instance.logger.debug "****** Slow Request #{layer.type} Traces: Scoutprof is not enabled"
+            end
+          end
+        end # walker.walk
 
         metric_hash = attach_backtraces(metric_hash)
         allocation_metric_hash = attach_backtraces(allocation_metric_hash)
 
-        [metric_hash, allocation_metric_hash]
+        [metric_hash,allocation_metric_hash]
+      end
+
+      def attach_backtraces(metric_hash)
+        @backtraces.each do |meta_with_backtrace|
+          metric_hash.keys.find { |k| k == meta_with_backtrace }.backtrace = meta_with_backtrace.backtrace
+        end
+        metric_hash
       end
     end
   end

data/lib/scout_apm/layer_converters/slow_request_converter.rb

@@ -2,6 +2,7 @@ module ScoutApm
   module LayerConverters
     class SlowRequestConverter < ConverterBase
       def initialize(*)
+        @backtraces = [] # An Array of MetricMetas that have a backtrace
         super
 
         # After call to super, so @request is populated
@@ -10,8 +11,6 @@ module ScoutApm
                   else
                     -1
                   end
-
-        setup_subscopable_callbacks
       end
 
       def name
@@ -25,8 +24,8 @@ module ScoutApm
       # Unconditionally attempts to convert this into a SlowTransaction object.
       # Can return nil if the request didn't have any scope_layer.
       def call
-        return nil unless request.web?
-        return nil unless scope_layer
+        scope = scope_layer
+        return nil unless scope
 
         ScoutApm::Agent.instance.slow_request_policy.stored!(request)
 
@@ -36,13 +35,12 @@ module ScoutApm
         uri = request.annotations[:uri] || ""
 
         timing_metrics, allocation_metrics = create_metrics
-
         unless ScoutApm::Instruments::Allocations::ENABLED
           allocation_metrics = {}
         end
 
         SlowTransaction.new(uri,
-                            scope_layer.legacy_metric_name,
+                            scope.legacy_metric_name,
                             root_layer.total_call_time,
                             timing_metrics,
                             allocation_metrics,
@@ -51,29 +49,119 @@ module ScoutApm
                             [], # stackprof, now unused.
                             mem_delta,
                             root_layer.total_allocations,
-                            @points,
-                            limited?)
+                            @points)
+      end
+
+      # Iterates over the TrackedRequest's MetricMetas that have backtraces and attaches each to correct MetricMeta in the Metric Hash.
+      def attach_backtraces(metric_hash)
+        @backtraces.each do |meta_with_backtrace|
+          metric_hash.keys.find { |k| k == meta_with_backtrace }.backtrace = meta_with_backtrace.backtrace
+        end
+        metric_hash
       end
 
       # Full metrics from this request. These get stored permanently in a SlowTransaction.
       # Some merging of metrics will happen here, so if a request calls the same
       # ActiveRecord or View repeatedly, it'll get merged.
-      #
+      # 
       # This returns a 2-element of Metric Hashes (the first element is timing metrics, the second element is allocation metrics)
       def create_metrics
         metric_hash = Hash.new
         allocation_metric_hash = Hash.new
 
+        # Keep a list of subscopes, but only ever use the front one.  The rest
+        # get pushed/popped in cases when we have many levels of subscopable
+        # layers.  This lets us push/pop without otherwise keeping track very closely.
+        subscope_layers = []
+
+        walker.before do |layer|
+          if layer.subscopable?
+            subscope_layers.push(layer)
+          end
+        end
+
+        walker.after do |layer|
+          if layer.subscopable?
+            subscope_layers.pop
+          end
+        end
+
         walker.walk do |layer|
-          next if skip_layer?(layer)
-          store_specific_metric(layer, metric_hash, allocation_metric_hash)
-          store_aggregate_metric(layer, metric_hash, allocation_metric_hash)
+          # Sometimes we start capturing a layer without knowing if we really
+          # want to make an entry for it.  See ActiveRecord instrumentation for
+          # an example. We start capturing before we know if a query is cached
+          # or not, and want to skip any cached queries.
+          if layer.annotations[:ignorable]
+            next
+          end
+
+          meta_options = if subscope_layers.first && layer != subscope_layers.first # Don't scope under ourself.
+                           subscope_name = subscope_layers.first.legacy_metric_name
+                           {:scope => subscope_name}
+                         elsif layer == scope_layer # We don't scope the controller under itself
+                           {}
+                         else
+                           {:scope => scope_layer.legacy_metric_name}
+                         end
+
+          # Specific Metric
+          meta_options.merge!(:desc => layer.desc.to_s) if layer.desc
+          meta = MetricMeta.new(layer.legacy_metric_name, meta_options)
+          meta.extra.merge!(layer.annotations)
+          if layer.backtrace
+            bt = ScoutApm::Utils::BacktraceParser.new(layer.backtrace).call
+            if bt.any? # we could walk thru the call stack and not find in-app code
+              meta.backtrace = bt
+              # Why not just call meta.backtrace and call it done? The walker
+              # could access a later later that generates the same MetricMeta
+              # but doesn't have a backtrace. This could be lost in the
+              # metric_hash if it is replaced by the new key.
+              @backtraces << meta
+            else
+              ScoutApm::Agent.instance.logger.debug { "Unable to capture an app-specific backtrace for #{meta.inspect}\n#{layer.backtrace}" }
+            end
+          end
+          metric_hash[meta] ||= MetricStats.new( meta_options.has_key?(:scope) )
+          allocation_metric_hash[meta] ||= MetricStats.new( meta_options.has_key?(:scope) )
+          # timing
+          stat = metric_hash[meta]
+          stat.update!(layer.total_call_time, layer.total_exclusive_time)
+          stat.add_traces(layer.traces.as_json)
+
+          # Debug logging for scoutprof traces
+          if ScoutApm::Agent.instance.config.value('profile')
+            if layer.type =~ %r{^(Controller|Queue|Job)$}.freeze
+              ScoutApm::Agent.instance.logger.debug do
+                traces_inspect = layer.traces.inspect
+                "****** Slow Request #{layer.type} Traces (#{layer.name}, tet: #{layer.total_exclusive_time}, tct: #{layer.total_call_time}), total raw traces: #{layer.traces.cube.total_count}, total clean traces: #{layer.traces.total_count}:\n#{traces_inspect}"
+              end
+            end
+          else
+            if layer.type =~ %r{^(Controller|Queue|Job)$}.freeze
+              ScoutApm::Agent.instance.logger.debug "****** Slow Request #{layer.type} Traces: Scoutprof is not enabled"
+            end
+          end
+
+          # allocations
+          stat = allocation_metric_hash[meta]
+          stat.update!(layer.total_allocations, layer.total_exclusive_allocations)
+
+          # Merged Metric (no specifics, just sum up by type)
+          meta = MetricMeta.new("#{layer.type}/all")
+          metric_hash[meta] ||= MetricStats.new(false)
+          allocation_metric_hash[meta] ||= MetricStats.new(false)
+          # timing
+          stat = metric_hash[meta]
+          stat.update!(layer.total_call_time, layer.total_exclusive_time)
+          # allocations
+          stat = allocation_metric_hash[meta]
+          stat.update!(layer.total_allocations, layer.total_exclusive_allocations)
         end
 
         metric_hash = attach_backtraces(metric_hash)
         allocation_metric_hash = attach_backtraces(allocation_metric_hash)
 
-        [metric_hash, allocation_metric_hash]
+        [metric_hash,allocation_metric_hash]
       end
     end
   end

data/lib/scout_apm/metric_set.rb

@@ -1,8 +1,6 @@
 module ScoutApm
   class MetricSet
-    # We can't aggregate a handful of things like samplers (CPU, Memory), or
-    # Controller, and Percentiles so pass through these metrics directly
-    #
+    # We can't aggregate CPU, Memory, Capacity, or Controller, so pass through these metrics directly
     # TODO: Figure out a way to not have this duplicate what's in Samplers, and also on server's ingest
     PASSTHROUGH_METRICS = ["CPU", "Memory", "Instance", "Controller", "SlowTransaction", "Percentile", "Job"]
 
@@ -51,11 +49,5 @@ module ScoutApm
       @combine_in_progress = false
       self
     end
-
-
-    def eql?(other)
-      metrics == other.metrics
-    end
-    alias :== :eql?
   end
 end

data/lib/scout_apm/metric_stats.rb

@@ -9,6 +9,7 @@ class MetricStats
   attr_accessor :sum_of_squares
   attr_accessor :queue
   attr_accessor :latency
+  attr_accessor :traces
 
   def initialize(scoped = false)
     @scoped = scoped
@@ -18,6 +19,7 @@ class MetricStats
     self.min_call_time = 0.0
     self.max_call_time = 0.0
     self.sum_of_squares = 0.0
+    self.traces = []
   end
 
   # Note, that you must include exclusive_time if you wish to set
@@ -39,6 +41,10 @@ class MetricStats
     self
   end
 
+  def add_traces(traces)
+    self.traces += Array(traces)
+  end
+
   # combines data from another MetricStats object
   def combine!(other)
     self.call_count += other.call_count
@@ -47,18 +53,12 @@ class MetricStats
     self.min_call_time = other.min_call_time if self.min_call_time.zero? or other.min_call_time < self.min_call_time
     self.max_call_time = other.max_call_time if other.max_call_time > self.max_call_time
     self.sum_of_squares += other.sum_of_squares
+    self.traces = Array(self.traces) + Array(other.traces)
     self
   end
 
-  # To avoid conflicts with different JSON libaries handle JSON ourselves.
-  # Time-based metrics are converted to milliseconds from seconds.
-  def to_json(*a)
-     %Q[{"total_exclusive_time":#{total_exclusive_time*1000},"min_call_time":#{min_call_time*1000},"call_count":#{call_count},"sum_of_squares":#{sum_of_squares*1000},"total_call_time":#{total_call_time*1000},"max_call_time":#{max_call_time*1000}}]
-  end
-
   def as_json
-    json_attributes = [:call_count, :total_call_time, :total_exclusive_time, :min_call_time, :max_call_time]
-    # uri, context
+    json_attributes = [:call_count, :total_call_time, :total_exclusive_time, :min_call_time, :max_call_time, :traces]
     ScoutApm::AttributeArranger.call(self, json_attributes)
   end
 end

data/lib/scout_apm/reporter.rb

@@ -17,36 +17,31 @@ module ScoutApm
       @instant_key = instant_key
     end
 
+    # TODO: Parse & return a real response object, not the HTTP Response object
     def report(payload, headers = {})
-      hosts = determine_hosts
+      # Some posts (typically ones under development) bypass the ingestion pipeline and go directly to the webserver. They use direct_host instead of host
+      hosts = [:deploy_hook, :instant_trace].include?(type) ? config.value('direct_host') : config.value('host')
 
-      if config.value('compress_payload')
-        original_payload_size = payload.length
-
-        payload, compression_headers = compress_payload(payload)
-        headers.merge!(compression_headers)
-
-        compress_payload_size = payload.length
-        ScoutApm::Agent.instance.logger.debug("Original Size: #{original_payload_size} Compressed Size: #{compress_payload_size}")
+      Array(hosts).each do |host|
+        full_uri = uri(host)
+        response = post(full_uri, payload, headers)
+        unless response && response.is_a?(Net::HTTPSuccess)
+          logger.warn "Error on checkin to #{full_uri.to_s}: #{response.inspect}"
+        end
       end
-
-      post_payload(hosts, payload, headers)
     end
 
     def uri(host)
-      encoded_app_name = CGI.escape(Environment.instance.application_name)
-      key = config.value('key')
-
       case type
       when :checkin
-        URI.parse("#{host}/apps/checkin.scout?key=#{key}&name=#{encoded_app_name}")
+        URI.parse("#{host}/apps/checkin.scout?key=#{config.value('key')}&name=#{CGI.escape(Environment.instance.application_name)}")
       when :app_server_load
-        URI.parse("#{host}/apps/app_server_load.scout?key=#{key}&name=#{encoded_app_name}")
+        URI.parse("#{host}/apps/app_server_load.scout?key=#{config.value('key')}&name=#{CGI.escape(Environment.instance.application_name)}")
       when :deploy_hook
-        URI.parse("#{host}/apps/deploy.scout?key=#{key}&name=#{encoded_app_name}")
+        URI.parse("#{host}/apps/deploy.scout?key=#{config.value('key')}&name=#{CGI.escape(config.value('name'))}")
       when :instant_trace
-        URI.parse("#{host}/apps/instant_trace.scout?key=#{key}&name=#{encoded_app_name}&instant_key=#{instant_key}")
-      end.tap { |u| logger.debug("Posting to #{u}") }
+        URI.parse("#{host}/apps/instant_trace.scout?key=#{config.value('key')}&name=#{CGI.escape(config.value('name'))}&instant_key=#{instant_key}")
+      end.tap{|u| logger.debug("Posting to #{u.to_s}")}
     end
 
     def can_report?
@@ -111,10 +106,7 @@ module ScoutApm
     # Net::HTTP::Proxy returns a regular Net::HTTP class if the first argument (host) is nil.
     def http(url)
       proxy_uri = URI.parse(config.value('proxy').to_s)
-      http = Net::HTTP::Proxy(proxy_uri.host,
-                              proxy_uri.port,
-                              proxy_uri.user,
-                              proxy_uri.password).new(url.host, url.port)
+      http = Net::HTTP::Proxy(proxy_uri.host,proxy_uri.port,proxy_uri.user,proxy_uri.password).new(url.host, url.port)
       if url.is_a?(URI::HTTPS)
         http.use_ssl = true
         http.ca_file = CA_FILE
@@ -122,33 +114,5 @@ module ScoutApm
       end
       http
     end
-
-    def compress_payload(payload)
-      [
-        ScoutApm::Utils::GzipHelper.new.deflate(payload),
-        { 'Content-Encoding' => 'gzip' }
-      ]
-    end
-
-    # Some posts (typically ones under development) bypass the ingestion
-    # pipeline and go directly to the webserver. They use direct_host instead
-    # of host
-    def determine_hosts
-      if [:deploy_hook, :instant_trace].include?(type)
-        config.value('direct_host')
-      else
-        config.value('host')
-      end
-    end
-
-    def post_payload(hosts, payload, headers)
-      Array(hosts).each do |host|
-        full_uri = uri(host)
-        response = post(full_uri, payload, headers)
-        unless response && response.is_a?(Net::HTTPSuccess)
-          logger.warn "Error on checkin to #{full_uri}: #{response.inspect}"
-        end
-      end
-    end
   end
 end