scout_apm 3.0.0.pre1 → 3.0.0.pre2

data/lib/scout_apm/layer_converters/slow_job_converter.rb

@@ -2,7 +2,6 @@ module ScoutApm
   module LayerConverters
     class SlowJobConverter < ConverterBase
       def initialize(*)
-        @backtraces = []
         super
 
         # After call to super, so @request is populated
@@ -11,6 +10,8 @@ module ScoutApm
                   else
                     -1
                   end
+
+        setup_subscopable_callbacks
       end
 
       def name
@@ -32,6 +33,7 @@ module ScoutApm
         mem_delta = ScoutApm::Instruments::Process::ProcessMemory.rss_to_mb(request.capture_mem_delta!)
 
         timing_metrics, allocation_metrics = create_metrics
+
         unless ScoutApm::Instruments::Allocations::ENABLED
           allocation_metrics = {}
         end
@@ -47,7 +49,9 @@ module ScoutApm
           allocation_metrics,
           mem_delta,
           job_layer.total_allocations,
-          score)
+          score,
+          limited?
+        )
       end
 
       def queue_layer
@@ -58,108 +62,30 @@ module ScoutApm
         @job_layer ||= find_first_layer_of_type("Job")
       end
 
+
+      # The queue_layer is useful to capture for other reasons, but doesn't
+      # create a MetricMeta/Stat of its own
+      def skip_layer?(layer)
+        super(layer) || layer == queue_layer
+      end
+
       def create_metrics
         metric_hash = Hash.new
         allocation_metric_hash = Hash.new
 
-        # Keep a list of subscopes, but only ever use the front one.  The rest
-        # get pushed/popped in cases when we have many levels of subscopable
-        # layers.  This lets us push/pop without otherwise keeping track very closely.
-        subscope_layers = []
+        walker.walk do |layer|
+          next if skip_layer?(layer)
 
-        walker.before do |layer|
-          if layer.subscopable?
-            subscope_layers.push(layer)
-          end
-        end
+          debug_scoutprof(layer)
 
-        walker.after do |layer|
-          if layer.subscopable?
-            subscope_layers.pop
-          end
+          store_specific_metric(layer, metric_hash, allocation_metric_hash)
+          store_aggregate_metric(layer, metric_hash, allocation_metric_hash)
         end
 
-        walker.walk do |layer|
-          # Sometimes we start capturing a layer without knowing if we really
-          # want to make an entry for it.  See ActiveRecord instrumentation for
-          # an example. We start capturing before we know if a query is cached
-          # or not, and want to skip any cached queries.
-          next if layer.annotations[:ignorable]
-
-          # The queue_layer is useful to capture for other reasons, but doesn't
-          # create a MetricMeta/Stat of its own
-          next if layer == queue_layer
-
-          meta_options = if subscope_layers.first && layer != subscope_layers.first # Don't scope under ourself.
-                           subscope_name = subscope_layers.first.legacy_metric_name
-                           {:scope => subscope_name}
-                         elsif layer == job_layer # We don't scope the controller under itself
-                           {}
-                         else
-                           {:scope => job_layer.legacy_metric_name}
-                         end
-
-          # Specific Metric
-          meta_options.merge!(:desc => layer.desc.to_s) if layer.desc
-          meta = MetricMeta.new(layer.legacy_metric_name, meta_options)
-          meta.extra.merge!(layer.annotations)
-
-          if layer.backtrace
-            bt = ScoutApm::Utils::BacktraceParser.new(layer.backtrace).call
-            if bt.any? # we could walk thru the call stack and not find in-app code
-              meta.backtrace = bt
-              # Why not just call meta.backtrace and call it done? The walker could access a later later that generates the same MetricMeta but doesn't have a backtrace. This could be
-              # lost in the metric_hash if it is replaced by the new key.
-              @backtraces << meta
-            else
-              ScoutApm::Agent.instance.logger.debug { "Unable to capture an app-specific backtrace for #{meta.inspect}\n#{layer.backtrace}" }
-            end
-          end
-
-          metric_hash[meta] ||= MetricStats.new( meta_options.has_key?(:scope) )
-          allocation_metric_hash[meta] ||= MetricStats.new( meta_options.has_key?(:scope) )
-          stat = metric_hash[meta]
-          stat.update!(layer.total_call_time, layer.total_exclusive_time)
-          stat = allocation_metric_hash[meta]
-          stat.update!(layer.total_allocations, layer.total_exclusive_allocations)
-
-          # Merged Metric (no specifics, just sum up by type)
-          meta = MetricMeta.new("#{layer.type}/all")
-          metric_hash[meta] ||= MetricStats.new(false)
-          allocation_metric_hash[meta] ||= MetricStats.new(false)
-          stat = metric_hash[meta]
-          stat.update!(layer.total_call_time, layer.total_exclusive_time)
-          stat = allocation_metric_hash[meta]
-          stat.update!(layer.total_allocations, layer.total_exclusive_allocations)
-
-          stat.add_traces(layer.traces.as_json)
-
-          # Debug logging for scoutprof traces
-          if ScoutApm::Agent.instance.config.value('profile')
-            if layer.type =~ %r{^(Controller|Queue|Job)$}.freeze
-              ScoutApm::Agent.instance.logger.debug do
-                traces_inspect = layer.traces.inspect
-                "****** Slow Request #{layer.type} Traces (#{layer.name}, tet: #{layer.total_exclusive_time}, tct: #{layer.total_call_time}), total raw traces: #{layer.traces.cube.total_count}, total clean traces: #{layer.traces.total_count}, skipped gc: #{layer.traces.skipped_in_gc}, skipped handler: #{layer.traces.skipped_in_handler}, skipped registered #{layer.traces.skipped_in_job_registered}, skipped not_running #{layer.traces.skipped_in_not_running}:\n#{traces_inspect}"
-              end
-            end
-          else
-            if layer.type =~ %r{^(Controller|Queue|Job)$}.freeze
-              ScoutApm::Agent.instance.logger.debug "****** Slow Request #{layer.type} Traces: Scoutprof is not enabled"
-            end
-          end
-        end # walker.walk
-
         metric_hash = attach_backtraces(metric_hash)
         allocation_metric_hash = attach_backtraces(allocation_metric_hash)
 
-        [metric_hash,allocation_metric_hash]
-      end
-
-      def attach_backtraces(metric_hash)
-        @backtraces.each do |meta_with_backtrace|
-          metric_hash.keys.find { |k| k == meta_with_backtrace }.backtrace = meta_with_backtrace.backtrace
-        end
-        metric_hash
+        [metric_hash, allocation_metric_hash]
       end
     end
   end

data/lib/scout_apm/layer_converters/slow_request_converter.rb

@@ -2,7 +2,6 @@ module ScoutApm
   module LayerConverters
     class SlowRequestConverter < ConverterBase
       def initialize(*)
-        @backtraces = [] # An Array of MetricMetas that have a backtrace
         super
 
         # After call to super, so @request is populated
@@ -11,6 +10,8 @@ module ScoutApm
                   else
                     -1
                   end
+
+        setup_subscopable_callbacks
       end
 
       def name
@@ -24,8 +25,8 @@ module ScoutApm
       # Unconditionally attempts to convert this into a SlowTransaction object.
       # Can return nil if the request didn't have any scope_layer.
       def call
-        scope = scope_layer
-        return nil unless scope
+        return nil unless request.web?
+        return nil unless scope_layer
 
         ScoutApm::Agent.instance.slow_request_policy.stored!(request)
 
@@ -35,12 +36,13 @@ module ScoutApm
         uri = request.annotations[:uri] || ""
 
         timing_metrics, allocation_metrics = create_metrics
+
         unless ScoutApm::Instruments::Allocations::ENABLED
           allocation_metrics = {}
         end
 
         SlowTransaction.new(uri,
-                            scope.legacy_metric_name,
+                            scope_layer.legacy_metric_name,
                             root_layer.total_call_time,
                             timing_metrics,
                             allocation_metrics,
@@ -49,119 +51,32 @@ module ScoutApm
                             [], # stackprof, now unused.
                             mem_delta,
                             root_layer.total_allocations,
-                            @points)
-      end
-
-      # Iterates over the TrackedRequest's MetricMetas that have backtraces and attaches each to correct MetricMeta in the Metric Hash.
-      def attach_backtraces(metric_hash)
-        @backtraces.each do |meta_with_backtrace|
-          metric_hash.keys.find { |k| k == meta_with_backtrace }.backtrace = meta_with_backtrace.backtrace
-        end
-        metric_hash
+                            @points,
+                            limited?)
       end
 
       # Full metrics from this request. These get stored permanently in a SlowTransaction.
       # Some merging of metrics will happen here, so if a request calls the same
       # ActiveRecord or View repeatedly, it'll get merged.
-      # 
+      #
       # This returns a 2-element of Metric Hashes (the first element is timing metrics, the second element is allocation metrics)
       def create_metrics
         metric_hash = Hash.new
         allocation_metric_hash = Hash.new
 
-        # Keep a list of subscopes, but only ever use the front one.  The rest
-        # get pushed/popped in cases when we have many levels of subscopable
-        # layers.  This lets us push/pop without otherwise keeping track very closely.
-        subscope_layers = []
+        walker.walk do |layer|
+          next if skip_layer?(layer)
 
-        walker.before do |layer|
-          if layer.subscopable?
-            subscope_layers.push(layer)
-          end
-        end
+          debug_scoutprof(layer)
 
-        walker.after do |layer|
-          if layer.subscopable?
-            subscope_layers.pop
-          end
-        end
-
-        walker.walk do |layer|
-          # Sometimes we start capturing a layer without knowing if we really
-          # want to make an entry for it.  See ActiveRecord instrumentation for
-          # an example. We start capturing before we know if a query is cached
-          # or not, and want to skip any cached queries.
-          if layer.annotations[:ignorable]
-            next
-          end
-
-          meta_options = if subscope_layers.first && layer != subscope_layers.first # Don't scope under ourself.
-                           subscope_name = subscope_layers.first.legacy_metric_name
-                           {:scope => subscope_name}
-                         elsif layer == scope_layer # We don't scope the controller under itself
-                           {}
-                         else
-                           {:scope => scope_layer.legacy_metric_name}
-                         end
-
-          # Specific Metric
-          meta_options.merge!(:desc => layer.desc.to_s) if layer.desc
-          meta = MetricMeta.new(layer.legacy_metric_name, meta_options)
-          meta.extra.merge!(layer.annotations)
-          if layer.backtrace
-            bt = ScoutApm::Utils::BacktraceParser.new(layer.backtrace).call
-            if bt.any? # we could walk thru the call stack and not find in-app code
-              meta.backtrace = bt
-              # Why not just call meta.backtrace and call it done? The walker
-              # could access a later later that generates the same MetricMeta
-              # but doesn't have a backtrace. This could be lost in the
-              # metric_hash if it is replaced by the new key.
-              @backtraces << meta
-            else
-              ScoutApm::Agent.instance.logger.debug { "Unable to capture an app-specific backtrace for #{meta.inspect}\n#{layer.backtrace}" }
-            end
-          end
-          metric_hash[meta] ||= MetricStats.new( meta_options.has_key?(:scope) )
-          allocation_metric_hash[meta] ||= MetricStats.new( meta_options.has_key?(:scope) )
-          # timing
-          stat = metric_hash[meta]
-          stat.update!(layer.total_call_time, layer.total_exclusive_time)
-          stat.add_traces(layer.traces.as_json)
-
-          # Debug logging for scoutprof traces
-          if ScoutApm::Agent.instance.config.value('profile')
-            if layer.type =~ %r{^(Controller|Queue|Job)$}.freeze
-              ScoutApm::Agent.instance.logger.debug do
-                traces_inspect = layer.traces.inspect
-                "****** Slow Request #{layer.type} Traces (#{layer.name}, tet: #{layer.total_exclusive_time}, tct: #{layer.total_call_time}), total raw traces: #{layer.traces.cube.total_count}, total clean traces: #{layer.traces.total_count}, skipped gc: #{layer.traces.skipped_in_gc}, skipped handler: #{layer.traces.skipped_in_handler}, skipped registered #{layer.traces.skipped_in_job_registered}, skipped not_running #{layer.traces.skipped_in_not_running}:\n#{traces_inspect}"
-              end
-            end
-          else
-            if layer.type =~ %r{^(Controller|Queue|Job)$}.freeze
-              ScoutApm::Agent.instance.logger.debug "****** Slow Request #{layer.type} Traces: Scoutprof is not enabled"
-            end
-          end
-
-          # allocations
-          stat = allocation_metric_hash[meta]
-          stat.update!(layer.total_allocations, layer.total_exclusive_allocations)
-
-          # Merged Metric (no specifics, just sum up by type)
-          meta = MetricMeta.new("#{layer.type}/all")
-          metric_hash[meta] ||= MetricStats.new(false)
-          allocation_metric_hash[meta] ||= MetricStats.new(false)
-          # timing
-          stat = metric_hash[meta]
-          stat.update!(layer.total_call_time, layer.total_exclusive_time)
-          # allocations
-          stat = allocation_metric_hash[meta]
-          stat.update!(layer.total_allocations, layer.total_exclusive_allocations)
+          store_specific_metric(layer, metric_hash, allocation_metric_hash)
+          store_aggregate_metric(layer, metric_hash, allocation_metric_hash)
         end
 
         metric_hash = attach_backtraces(metric_hash)
         allocation_metric_hash = attach_backtraces(allocation_metric_hash)
 
-        [metric_hash,allocation_metric_hash]
+        [metric_hash, allocation_metric_hash]
       end
     end
   end

data/lib/scout_apm/metric_set.rb

@@ -49,5 +49,11 @@ module ScoutApm
       @combine_in_progress = false
       self
     end
+
+
+    def eql?(other)
+      metrics == other.metrics
+    end
+    alias :== :eql?
   end
 end

data/lib/scout_apm/reporter.rb

@@ -17,31 +17,38 @@ module ScoutApm
       @instant_key = instant_key
     end
 
-    # TODO: Parse & return a real response object, not the HTTP Response object
     def report(payload, headers = {})
-      # Some posts (typically ones under development) bypass the ingestion pipeline and go directly to the webserver. They use direct_host instead of host
-      hosts = [:deploy_hook, :instant_trace].include?(type) ? config.value('direct_host') : config.value('host')
+      hosts = determine_hosts
 
-      Array(hosts).each do |host|
-        full_uri = uri(host)
-        response = post(full_uri, payload, headers)
-        unless response && response.is_a?(Net::HTTPSuccess)
-          logger.warn "Error on checkin to #{full_uri.to_s}: #{response.inspect}"
-        end
+      if config.value('compress_payload')
+        original_payload_size = payload.length
+
+        payload, compression_headers = compress_payload(payload)
+        headers.merge!(compression_headers)
+
+        compress_payload_size = payload.length
+        ScoutApm::Agent.instance.logger.debug("Compressed Payload: #{payload.inspect}")
+        ScoutApm::Agent.instance.logger.debug("Original Size: #{original_payload_size} Compressed Size: #{compress_payload_size}")
       end
+
+      post_payload(hosts, payload, headers)
     end
 
     def uri(host)
+      encoded_app_name = CGI.escape(Environment.instance.application_name)
+      encoded_name = CGI.escape(config.value('name'))
+      key = config.value('key')
+
       case type
       when :checkin
-        URI.parse("#{host}/apps/checkin.scout?key=#{config.value('key')}&name=#{CGI.escape(Environment.instance.application_name)}")
+        URI.parse("#{host}/apps/checkin.scout?key=#{key}&name=#{encoded_app_name}")
       when :app_server_load
-        URI.parse("#{host}/apps/app_server_load.scout?key=#{config.value('key')}&name=#{CGI.escape(Environment.instance.application_name)}")
+        URI.parse("#{host}/apps/app_server_load.scout?key=#{key}&name=#{encoded_app_name}")
       when :deploy_hook
-        URI.parse("#{host}/apps/deploy.scout?key=#{config.value('key')}&name=#{CGI.escape(config.value('name'))}")
+        URI.parse("#{host}/apps/deploy.scout?key=#{key}&name=#{encoded_name}")
       when :instant_trace
-        URI.parse("#{host}/apps/instant_trace.scout?key=#{config.value('key')}&name=#{CGI.escape(config.value('name'))}&instant_key=#{instant_key}")
-      end.tap{|u| logger.debug("Posting to #{u.to_s}")}
+        URI.parse("#{host}/apps/instant_trace.scout?key=#{key}&name=#{encoded_name}&instant_key=#{instant_key}")
+      end.tap { |u| logger.debug("Posting to #{u}") }
     end
 
     def can_report?
@@ -106,7 +113,10 @@ module ScoutApm
     # Net::HTTP::Proxy returns a regular Net::HTTP class if the first argument (host) is nil.
     def http(url)
       proxy_uri = URI.parse(config.value('proxy').to_s)
-      http = Net::HTTP::Proxy(proxy_uri.host,proxy_uri.port,proxy_uri.user,proxy_uri.password).new(url.host, url.port)
+      http = Net::HTTP::Proxy(proxy_uri.host,
+                              proxy_uri.port,
+                              proxy_uri.user,
+                              proxy_uri.password).new(url.host, url.port)
       if url.is_a?(URI::HTTPS)
         http.use_ssl = true
         http.ca_file = CA_FILE
@@ -114,5 +124,33 @@ module ScoutApm
       end
       http
     end
+
+    def compress_payload(payload)
+      [
+        ScoutApm::Utils::GzipHelper.new.deflate(payload),
+        { 'Content-Encoding' => 'gzip' }
+      ]
+    end
+
+    # Some posts (typically ones under development) bypass the ingestion
+    # pipeline and go directly to the webserver. They use direct_host instead
+    # of host
+    def determine_hosts
+      if [:deploy_hook, :instant_trace].include?(type)
+        config.value('direct_host')
+      else
+        config.value('host')
+      end
+    end
+
+    def post_payload(hosts, payload, headers)
+      Array(hosts).each do |host|
+        full_uri = uri(host)
+        response = post(full_uri, payload, headers)
+        unless response && response.is_a?(Net::HTTPSuccess)
+          logger.warn "Error on checkin to #{full_uri}: #{response.inspect}"
+        end
+      end
+    end
   end
 end

data/lib/scout_apm/request_histograms.rb

@@ -39,6 +39,10 @@ module ScoutApm
       initialize_histograms_hash
     end
 
+    def raw(item)
+      @histograms[item]
+    end
+
     def initialize_histograms_hash
       @histograms = Hash.new { |h, k| h[k] = NumericHistogram.new(histogram_size) }
     end