scout_apm 2.1.8 → 2.1.9

data/lib/scout_apm/layer_converters/slow_job_converter.rb

@@ -2,7 +2,6 @@ module ScoutApm
   module LayerConverters
     class SlowJobConverter < ConverterBase
       def initialize(*)
-        @backtraces = []
         super
 
         # After call to super, so @request is populated
@@ -11,6 +10,8 @@ module ScoutApm
                   else
                     -1
                   end
+
+        setup_subscopable_callbacks
       end
 
       def name
@@ -32,6 +33,7 @@ module ScoutApm
         mem_delta = ScoutApm::Instruments::Process::ProcessMemory.rss_to_mb(request.capture_mem_delta!)
 
         timing_metrics, allocation_metrics = create_metrics
+
         unless ScoutApm::Instruments::Allocations::ENABLED
           allocation_metrics = {}
         end
@@ -47,7 +49,9 @@ module ScoutApm
           allocation_metrics,
           mem_delta,
           job_layer.total_allocations,
-          score)
+          score,
+          limited?
+        )
       end
 
       def queue_layer
@@ -58,92 +62,29 @@ module ScoutApm
         @job_layer ||= find_first_layer_of_type("Job")
       end
 
+      def skip_layer?(layer)
+        super(layer) || layer == queue_layer
+      end
+
       def create_metrics
         metric_hash = Hash.new
         allocation_metric_hash = Hash.new
 
-        # Keep a list of subscopes, but only ever use the front one.  The rest
-        # get pushed/popped in cases when we have many levels of subscopable
-        # layers.  This lets us push/pop without otherwise keeping track very closely.
-        subscope_layers = []
-
-        walker.before do |layer|
-          if layer.subscopable?
-            subscope_layers.push(layer)
-          end
-        end
-
-        walker.after do |layer|
-          if layer.subscopable?
-            subscope_layers.pop
-          end
-        end
-
         walker.walk do |layer|
-          # Sometimes we start capturing a layer without knowing if we really
-          # want to make an entry for it.  See ActiveRecord instrumentation for
-          # an example. We start capturing before we know if a query is cached
-          # or not, and want to skip any cached queries.
-          next if layer.annotations[:ignorable]
+          next if skip_layer?(layer)
 
           # The queue_layer is useful to capture for other reasons, but doesn't
           # create a MetricMeta/Stat of its own
           next if layer == queue_layer
 
-          meta_options = if subscope_layers.first && layer != subscope_layers.first # Don't scope under ourself.
-                           subscope_name = subscope_layers.first.legacy_metric_name
-                           {:scope => subscope_name}
-                         elsif layer == job_layer # We don't scope the controller under itself
-                           {}
-                         else
-                           {:scope => job_layer.legacy_metric_name}
-                         end
-
-          # Specific Metric
-          meta_options.merge!(:desc => layer.desc.to_s) if layer.desc
-          meta = MetricMeta.new(layer.legacy_metric_name, meta_options)
-          meta.extra.merge!(layer.annotations)
-
-          if layer.backtrace
-            bt = ScoutApm::Utils::BacktraceParser.new(layer.backtrace).call
-            if bt.any? # we could walk thru the call stack and not find in-app code
-              meta.backtrace = bt
-              # Why not just call meta.backtrace and call it done? The walker could access a later later that generates the same MetricMeta but doesn't have a backtrace. This could be
-              # lost in the metric_hash if it is replaced by the new key.
-              @backtraces << meta
-            else
-              ScoutApm::Agent.instance.logger.debug { "Unable to capture an app-specific backtrace for #{meta.inspect}\n#{layer.backtrace}" }
-            end
-          end
-
-          metric_hash[meta] ||= MetricStats.new( meta_options.has_key?(:scope) )
-          allocation_metric_hash[meta] ||= MetricStats.new( meta_options.has_key?(:scope) )
-          stat = metric_hash[meta]
-          stat.update!(layer.total_call_time, layer.total_exclusive_time)
-          stat = allocation_metric_hash[meta]
-          stat.update!(layer.total_allocations, layer.total_exclusive_allocations)
-
-          # Merged Metric (no specifics, just sum up by type)
-          meta = MetricMeta.new("#{layer.type}/all")
-          metric_hash[meta] ||= MetricStats.new(false)
-          allocation_metric_hash[meta] ||= MetricStats.new(false)
-          stat = metric_hash[meta]
-          stat.update!(layer.total_call_time, layer.total_exclusive_time)
-          stat = allocation_metric_hash[meta]
-          stat.update!(layer.total_allocations, layer.total_exclusive_allocations)
+          store_specific_metric(layer, metric_hash, allocation_metric_hash)
+          store_aggregate_metric(layer, metric_hash, allocation_metric_hash)
         end
 
         metric_hash = attach_backtraces(metric_hash)
         allocation_metric_hash = attach_backtraces(allocation_metric_hash)
 
-        [metric_hash,allocation_metric_hash]
-      end
-
-      def attach_backtraces(metric_hash)
-        @backtraces.each do |meta_with_backtrace|
-          metric_hash.keys.find { |k| k == meta_with_backtrace }.backtrace = meta_with_backtrace.backtrace
-        end
-        metric_hash
+        [metric_hash, allocation_metric_hash]
       end
     end
   end

data/lib/scout_apm/layer_converters/slow_request_converter.rb

@@ -2,7 +2,6 @@ module ScoutApm
   module LayerConverters
     class SlowRequestConverter < ConverterBase
       def initialize(*)
-        @backtraces = [] # An Array of MetricMetas that have a backtrace
         super
 
         # After call to super, so @request is populated
@@ -11,6 +10,8 @@ module ScoutApm
                   else
                     -1
                   end
+
+        setup_subscopable_callbacks
       end
 
       def name
@@ -24,8 +25,8 @@ module ScoutApm
       # Unconditionally attempts to convert this into a SlowTransaction object.
       # Can return nil if the request didn't have any scope_layer.
       def call
-        scope = scope_layer
-        return nil unless scope
+        return nil unless request.web?
+        return nil unless scope_layer
 
         ScoutApm::Agent.instance.slow_request_policy.stored!(request)
 
@@ -35,12 +36,13 @@ module ScoutApm
         uri = request.annotations[:uri] || ""
 
         timing_metrics, allocation_metrics = create_metrics
+
         unless ScoutApm::Instruments::Allocations::ENABLED
           allocation_metrics = {}
         end
 
         SlowTransaction.new(uri,
-                            scope.legacy_metric_name,
+                            scope_layer.legacy_metric_name,
                             root_layer.total_call_time,
                             timing_metrics,
                             allocation_metrics,
@@ -49,103 +51,29 @@ module ScoutApm
                             [], # stackprof, now unused.
                             mem_delta,
                             root_layer.total_allocations,
-                            @points)
-      end
-
-      # Iterates over the TrackedRequest's MetricMetas that have backtraces and attaches each to correct MetricMeta in the Metric Hash.
-      def attach_backtraces(metric_hash)
-        @backtraces.each do |meta_with_backtrace|
-          metric_hash.keys.find { |k| k == meta_with_backtrace }.backtrace = meta_with_backtrace.backtrace
-        end
-        metric_hash
+                            @points,
+                            limited?)
       end
 
       # Full metrics from this request. These get stored permanently in a SlowTransaction.
       # Some merging of metrics will happen here, so if a request calls the same
       # ActiveRecord or View repeatedly, it'll get merged.
-      # 
+      #
       # This returns a 2-element of Metric Hashes (the first element is timing metrics, the second element is allocation metrics)
       def create_metrics
         metric_hash = Hash.new
         allocation_metric_hash = Hash.new
 
-        # Keep a list of subscopes, but only ever use the front one.  The rest
-        # get pushed/popped in cases when we have many levels of subscopable
-        # layers.  This lets us push/pop without otherwise keeping track very closely.
-        subscope_layers = []
-
-        walker.before do |layer|
-          if layer.subscopable?
-            subscope_layers.push(layer)
-          end
-        end
-
-        walker.after do |layer|
-          if layer.subscopable?
-            subscope_layers.pop
-          end
-        end
-
         walker.walk do |layer|
-          # Sometimes we start capturing a layer without knowing if we really
-          # want to make an entry for it.  See ActiveRecord instrumentation for
-          # an example. We start capturing before we know if a query is cached
-          # or not, and want to skip any cached queries.
-          if layer.annotations[:ignorable]
-            next
-          end
-
-          meta_options = if subscope_layers.first && layer != subscope_layers.first # Don't scope under ourself.
-                           subscope_name = subscope_layers.first.legacy_metric_name
-                           {:scope => subscope_name}
-                         elsif layer == scope_layer # We don't scope the controller under itself
-                           {}
-                         else
-                           {:scope => scope_layer.legacy_metric_name}
-                         end
-
-          # Specific Metric
-          meta_options.merge!(:desc => layer.desc.to_s) if layer.desc
-          meta = MetricMeta.new(layer.legacy_metric_name, meta_options)
-          meta.extra.merge!(layer.annotations)
-          if layer.backtrace
-            bt = ScoutApm::Utils::BacktraceParser.new(layer.backtrace).call
-            if bt.any? # we could walk thru the call stack and not find in-app code
-              meta.backtrace = bt
-              # Why not just call meta.backtrace and call it done? The walker
-              # could access a later later that generates the same MetricMeta
-              # but doesn't have a backtrace. This could be lost in the
-              # metric_hash if it is replaced by the new key.
-              @backtraces << meta
-            else
-              ScoutApm::Agent.instance.logger.debug { "Unable to capture an app-specific backtrace for #{meta.inspect}\n#{layer.backtrace}" }
-            end
-          end
-          metric_hash[meta] ||= MetricStats.new( meta_options.has_key?(:scope) )
-          allocation_metric_hash[meta] ||= MetricStats.new( meta_options.has_key?(:scope) )
-          # timing
-          stat = metric_hash[meta]
-          stat.update!(layer.total_call_time, layer.total_exclusive_time)
-          # allocations
-          stat = allocation_metric_hash[meta]
-          stat.update!(layer.total_allocations, layer.total_exclusive_allocations)
-
-          # Merged Metric (no specifics, just sum up by type)
-          meta = MetricMeta.new("#{layer.type}/all")
-          metric_hash[meta] ||= MetricStats.new(false)
-          allocation_metric_hash[meta] ||= MetricStats.new(false)
-          # timing
-          stat = metric_hash[meta]
-          stat.update!(layer.total_call_time, layer.total_exclusive_time)
-          # allocations
-          stat = allocation_metric_hash[meta]
-          stat.update!(layer.total_allocations, layer.total_exclusive_allocations)
+          next if skip_layer?(layer)
+          store_specific_metric(layer, metric_hash, allocation_metric_hash)
+          store_aggregate_metric(layer, metric_hash, allocation_metric_hash)
         end
 
         metric_hash = attach_backtraces(metric_hash)
         allocation_metric_hash = attach_backtraces(allocation_metric_hash)
 
-        [metric_hash,allocation_metric_hash]
+        [metric_hash, allocation_metric_hash]
       end
     end
   end

data/lib/scout_apm/metric_set.rb

@@ -49,5 +49,11 @@ module ScoutApm
       @combine_in_progress = false
       self
     end
+
+
+    def eql?(other)
+      metrics == other.metrics
+    end
+    alias :== :eql?
   end
 end

data/lib/scout_apm/reporter.rb

@@ -17,31 +17,38 @@ module ScoutApm
       @instant_key = instant_key
     end
 
-    # TODO: Parse & return a real response object, not the HTTP Response object
     def report(payload, headers = {})
-      # Some posts (typically ones under development) bypass the ingestion pipeline and go directly to the webserver. They use direct_host instead of host
-      hosts = [:deploy_hook, :instant_trace].include?(type) ? config.value('direct_host') : config.value('host')
+      hosts = determine_hosts
 
-      Array(hosts).each do |host|
-        full_uri = uri(host)
-        response = post(full_uri, payload, headers)
-        unless response && response.is_a?(Net::HTTPSuccess)
-          logger.warn "Error on checkin to #{full_uri.to_s}: #{response.inspect}"
-        end
+      if config.value('compress_payload')
+        original_payload_size = payload.length
+
+        payload, compression_headers = compress_payload(payload)
+        headers.merge!(compression_headers)
+
+        compress_payload_size = payload.length
+        ScoutApm::Agent.instance.logger.debug("Compressed Payload: #{payload.inspect}")
+        ScoutApm::Agent.instance.logger.debug("Original Size: #{original_payload_size} Compressed Size: #{compress_payload_size}")
       end
+
+      post_payload(hosts, payload, headers)
     end
 
     def uri(host)
+      encoded_app_name = CGI.escape(Environment.instance.application_name)
+      encoded_name = CGI.escape(config.value('name'))
+      key = config.value('key')
+
       case type
       when :checkin
-        URI.parse("#{host}/apps/checkin.scout?key=#{config.value('key')}&name=#{CGI.escape(Environment.instance.application_name)}")
+        URI.parse("#{host}/apps/checkin.scout?key=#{key}&name=#{encoded_app_name}")
       when :app_server_load
-        URI.parse("#{host}/apps/app_server_load.scout?key=#{config.value('key')}&name=#{CGI.escape(Environment.instance.application_name)}")
+        URI.parse("#{host}/apps/app_server_load.scout?key=#{key}&name=#{encoded_app_name}")
       when :deploy_hook
-        URI.parse("#{host}/apps/deploy.scout?key=#{config.value('key')}&name=#{CGI.escape(config.value('name'))}")
+        URI.parse("#{host}/apps/deploy.scout?key=#{key}&name=#{encoded_name}")
       when :instant_trace
-        URI.parse("#{host}/apps/instant_trace.scout?key=#{config.value('key')}&name=#{CGI.escape(config.value('name'))}&instant_key=#{instant_key}")
-      end.tap{|u| logger.debug("Posting to #{u.to_s}")}
+        URI.parse("#{host}/apps/instant_trace.scout?key=#{key}&name=#{encoded_name}&instant_key=#{instant_key}")
+      end.tap { |u| logger.debug("Posting to #{u}") }
     end
 
     def can_report?
@@ -106,7 +113,10 @@ module ScoutApm
     # Net::HTTP::Proxy returns a regular Net::HTTP class if the first argument (host) is nil.
     def http(url)
       proxy_uri = URI.parse(config.value('proxy').to_s)
-      http = Net::HTTP::Proxy(proxy_uri.host,proxy_uri.port,proxy_uri.user,proxy_uri.password).new(url.host, url.port)
+      http = Net::HTTP::Proxy(proxy_uri.host,
+                              proxy_uri.port,
+                              proxy_uri.user,
+                              proxy_uri.password).new(url.host, url.port)
       if url.is_a?(URI::HTTPS)
         http.use_ssl = true
         http.ca_file = CA_FILE
@@ -114,5 +124,33 @@ module ScoutApm
       end
       http
     end
+
+    def compress_payload(payload)
+      [
+        ScoutApm::Utils::GzipHelper.new.deflate(payload),
+        { 'Content-Encoding' => 'gzip' }
+      ]
+    end
+
+    # Some posts (typically ones under development) bypass the ingestion
+    # pipeline and go directly to the webserver. They use direct_host instead
+    # of host
+    def determine_hosts
+      if [:deploy_hook, :instant_trace].include?(type)
+        config.value('direct_host')
+      else
+        config.value('host')
+      end
+    end
+
+    def post_payload(hosts, payload, headers)
+      Array(hosts).each do |host|
+        full_uri = uri(host)
+        response = post(full_uri, payload, headers)
+        unless response && response.is_a?(Net::HTTPSuccess)
+          logger.warn "Error on checkin to #{full_uri}: #{response.inspect}"
+        end
+      end
+    end
   end
 end

data/lib/scout_apm/request_histograms.rb

@@ -39,6 +39,10 @@ module ScoutApm
       initialize_histograms_hash
     end
 
+    def raw(item)
+      @histograms[item]
+    end
+
     def initialize_histograms_hash
       @histograms = Hash.new { |h, k| h[k] = NumericHistogram.new(histogram_size) }
     end

data/lib/scout_apm/scored_item_set.rb

@@ -63,6 +63,12 @@ module ScoutApm
       end
     end
 
+    # Equal to another set only if exactly the same set of items is inside
+    def eql?(other)
+      items == other.items
+    end
+
+    alias :== :eql?
 
     private
 
@@ -75,5 +81,6 @@ module ScoutApm
         items[new_item.name] = [new_item.score, new_item.call]
       end
     end
+
   end
 end

data/lib/scout_apm/serializers/histograms_serializer_to_json.rb

@@ -0,0 +1,21 @@
+
+module ScoutApm
+  module Serializers
+    class HistogramsSerializerToJson
+      attr_reader :histograms
+
+      def initialize(histograms)
+        @histograms = histograms
+      end
+
+      def as_json
+        histograms.map do |histo|
+          {
+            "name" => histo.name,
+            "histogram" => histo.histogram.as_json,
+          }
+        end
+      end
+    end
+  end
+end