scout_apm 1.4.6 → 1.5.0.pre

data/lib/scout_apm/layer_converters/depth_first_walker.rb

@@ -0,0 +1,36 @@
+module ScoutApm
+  module LayerConverters
+    class DepthFirstWalker
+      attr_reader :root_layer
+
+      def initialize(root_layer)
+        @root_layer = root_layer
+      end
+
+      def before(&block)
+        @before_block = block
+      end
+
+      def after(&block)
+        @after_block = block
+      end
+
+      def walk(layer=root_layer, &block)
+        # Need to run this for the root layer the first time through.
+        if layer == root_layer
+          @before_block.call(layer) if @before_block
+          yield layer
+          @after_block.call(layer) if @after_block
+        end
+
+        layer.children.each do |child|
+          @before_block.call(child) if @before_block
+          yield child
+          walk(child, &block)
+          @after_block.call(child) if @after_block
+        end
+        nil
+      end
+    end
+  end
+end

data/lib/scout_apm/layer_converters/error_converter.rb

@@ -0,0 +1,20 @@
+module ScoutApm
+  module LayerConverters
+    class ErrorConverter < ConverterBase
+      def call
+        scope = scope_layer
+
+        # Should we mark a request as errored out if a middleware raises?
+        # How does that interact w/ a tool like Sentry or Honeybadger?
+        return {} unless scope
+        return {} unless request.error?
+
+        meta = MetricMeta.new("Errors/#{scope.legacy_metric_name}", {})
+        stat = MetricStats.new
+        stat.update!(1)
+
+        { meta => stat }
+      end
+    end
+  end
+end

data/lib/scout_apm/layer_converters/job_converter.rb

@@ -0,0 +1,84 @@
+# Queue/Critical (implicit count)
+#   Job/PasswordResetJob Scope=Queue/Critical (implicit count, & total time)
+#     JobMetric/Latency 10 Scope=Job/PasswordResetJob
+#     ActiveRecord/User/find Scope=Job/PasswordResetJob
+#     ActiveRecord/Message/find Scope=Job/PasswordResetJob
+#     HTTP/request Scope=Job/PasswordResetJob
+#     View/message/text Scope=Job/PasswordResetJob
+#       ActiveRecord/Config/find Scope=View/message/text
+
+module ScoutApm
+  module LayerConverters
+    class JobConverter < ConverterBase
+      def call
+        return unless request.job?
+
+        JobRecord.new(
+          queue_layer.name,
+          job_layer.name,
+          job_layer.total_call_time,
+          job_layer.total_exclusive_time,
+          errors,
+          create_metrics
+        )
+      end
+
+      def queue_layer
+        @queue_layer ||= find_first_layer_of_type("Queue")
+      end
+
+      def job_layer
+        @job_layer ||= find_first_layer_of_type("Job")
+      end
+
+      def errors
+        if request.error?
+          1
+        else
+          0
+        end
+      end
+
+      def find_first_layer_of_type(layer_type)
+        walker.walk do |layer|
+          return layer if layer.type == layer_type
+        end
+      end
+
+      # Full metrics from this request. These get aggregated in Store for the
+      # overview metrics, or stored permanently in a SlowTransaction
+      # Some merging of metrics will happen here, so if a request calls the same
+      # ActiveRecord or View repeatedly, it'll get merged.
+      def create_metrics
+        metric_hash = Hash.new
+
+        meta_options = {:scope => job_layer.legacy_metric_name}
+
+        walker.walk do |layer|
+          next if layer == job_layer
+          next if layer == queue_layer
+
+          # we don't need to use the full metric name for scoped metrics as we
+          # only display metrics aggregrated by type, just use "ActiveRecord"
+          # or similar.
+          metric_name = layer.type
+
+          meta = MetricMeta.new(metric_name, meta_options)
+          metric_hash[meta] ||= MetricStats.new( meta_options.has_key?(:scope) )
+
+          stat = metric_hash[meta]
+          stat.update!(layer.total_call_time, layer.total_exclusive_time)
+        end
+
+        # Add the latency metric, which wasn't stored as a distinct layer
+        latency = request.annotations[:queue_latency] || 0
+        meta = MetricMeta.new("Latency", meta_options)
+        stat = MetricStats.new
+        stat.update!(latency)
+        metric_hash[meta] = stat
+
+        metric_hash
+      end
+    end
+  end
+end

data/lib/scout_apm/layer_converters/metric_converter.rb

@@ -0,0 +1,45 @@
+# Take a TrackedRequest and turn it into a hash of:
+#   MetricMeta => MetricStats
+
+module ScoutApm
+  module LayerConverters
+    class MetricConverter < ConverterBase
+      def call
+        scope = scope_layer
+
+        # TODO: Track requests that never reach a Controller (for example, when
+        # Middleware decides to return rather than passing onward)
+        return {} unless scope
+
+        create_metrics
+      end
+
+      # Full metrics from this request. These get aggregated in Store for the
+      # overview metrics, or stored permanently in a SlowTransaction
+      # Some merging of metrics will happen here, so if a request calls the same
+      # ActiveRecord or View repeatedly, it'll get merged.
+      def create_metrics
+        metric_hash = Hash.new
+
+        walker.walk do |layer|
+          meta_options = if layer == scope_layer # We don't scope the controller under itself
+                          {}
+                        else
+                          {:scope => scope_layer.legacy_metric_name}
+                        end
+
+          # we don't need to use the full metric name for scoped metrics as we only display metrics aggregrated
+          # by type.
+          metric_name = meta_options.has_key?(:scope) ? layer.type : layer.legacy_metric_name
+
+          meta = MetricMeta.new(metric_name, meta_options)
+          metric_hash[meta] ||= MetricStats.new( meta_options.has_key?(:scope) )
+
+          stat = metric_hash[meta]
+          stat.update!(layer.total_call_time, layer.total_exclusive_time)
+        end
+        metric_hash
+      end
+    end
+  end
+end

data/lib/scout_apm/layer_converters/request_queue_time_converter.rb

@@ -0,0 +1,60 @@
+module ScoutApm
+  module LayerConverters
+    class RequestQueueTimeConverter < ConverterBase
+
+      HEADERS = %w(X-Queue-Start X-Request-Start X-QUEUE-START X-REQUEST-START x-queue-start x-request-start)
+
+      # Headers is a hash of request headers.  In Rails, request.headers would be appropriate
+      def initialize(request)
+        super(request)
+        @headers = request.headers
+      end
+
+      def call
+        return {} unless headers
+
+        raw_start = locate_timestamp
+        return {} unless raw_start
+
+        parsed_start = parse(raw_start)
+        return {} unless parsed_start
+
+        request_start = root_layer.start_time
+        queue_time = (request_start - parsed_start).to_f
+
+        # If we end up with a negative value, just bail out and don't report anything
+        return {} if queue_time < 0
+
+        meta = MetricMeta.new("QueueTime/Request", {:scope => scope_layer.legacy_metric_name})
+        stat = MetricStats.new(true)
+        stat.update!(queue_time)
+
+        { meta => stat }
+      end
+
+      private
+
+      attr_reader :headers
+
+      # Looks through the possible headers with this data, and extracts the raw
+      # value of the header
+      # Returns nil if not found
+      def locate_timestamp
+        return nil unless headers
+
+        header = HEADERS.find { |candidate| headers[candidate] }
+        if header
+          data = headers[header]
+          data.to_s.gsub(/(t=|\.)/, '')
+        else
+          nil
+        end
+      end
+
+      # Returns a timestamp in fractional seconds since epoch
+      def parse(time_string)
+        Time.at("#{time_string[0,10]}.#{time_string[10,13]}".to_f)
+      end
+    end
+  end
+end

data/lib/scout_apm/layer_converters/slow_job_converter.rb

@@ -0,0 +1,88 @@
+module ScoutApm
+  module LayerConverters
+    class SlowJobConverter < ConverterBase
+      def call
+        return unless request.job?
+
+        job_name = [queue_layer.name, job_layer.name]
+
+        slow_enough = ScoutApm::Agent.instance.slow_job_policy.slow?(job_name, root_layer.total_call_time)
+        return unless slow_enough
+
+        SlowJobRecord.new(
+          queue_layer.name,
+          job_layer.name,
+          root_layer.stop_time,
+          job_layer.total_call_time,
+          job_layer.total_exclusive_time,
+          request.context,
+          create_metrics)
+      end
+
+      def queue_layer
+        @queue_layer ||= find_first_layer_of_type("Queue")
+      end
+
+      def job_layer
+        @job_layer ||= find_first_layer_of_type("Job")
+      end
+
+      def find_first_layer_of_type(layer_type)
+        walker.walk do |layer|
+          return layer if layer.type == layer_type
+        end
+      end
+
+      def create_metrics
+        metric_hash = Hash.new
+
+        # Keep a list of subscopes, but only ever use the front one.  The rest
+        # get pushed/popped in cases when we have many levels of subscopable
+        # layers.  This lets us push/pop without otherwise keeping track very closely.
+        subscope_layers = []
+
+        walker.before do |layer|
+          if layer.subscopable?
+            subscope_layers.push(layer)
+          end
+        end
+
+        walker.after do |layer|
+          if layer.subscopable?
+            subscope_layers.pop
+          end
+        end
+
+        walker.walk do |layer|
+          next if layer == queue_layer
+
+          meta_options = if subscope_layers.first && layer != subscope_layers.first # Don't scope under ourself.
+                           subscope_name = subscope_layers.first.legacy_metric_name
+                           {:scope => subscope_name}
+                         elsif layer == job_layer # We don't scope the controller under itself
+                           {}
+                         else
+                           {:scope => job_layer.legacy_metric_name}
+                         end
+
+          # Specific Metric
+          meta_options.merge!(:desc => layer.desc.to_s) if layer.desc
+          meta = MetricMeta.new(layer.legacy_metric_name, meta_options)
+          # this has moved - commenting out for now. will copy over bits from SlowRequestConverter
+          # meta.extra.merge!(:backtrace => ScoutApm::SlowTransaction.backtrace_parser(layer.backtrace)) if layer.backtrace
+          metric_hash[meta] ||= MetricStats.new( meta_options.has_key?(:scope) )
+          stat = metric_hash[meta]
+          stat.update!(layer.total_call_time, layer.total_exclusive_time)
+
+          # Merged Metric (no specifics, just sum up by type)
+          meta = MetricMeta.new("#{layer.type}/all")
+          metric_hash[meta] ||= MetricStats.new(false)
+          stat = metric_hash[meta]
+          stat.update!(layer.total_call_time, layer.total_exclusive_time)
+        end
+
+        metric_hash
+      end
+    end
+  end
+end

data/lib/scout_apm/layer_converters/slow_request_converter.rb

@@ -0,0 +1,111 @@
+module ScoutApm
+  module LayerConverters
+    class SlowRequestConverter < ConverterBase
+      def call
+        scope = scope_layer
+        return [nil, {}] unless scope
+
+        policy = ScoutApm::Agent.instance.slow_request_policy.capture_type(root_layer.total_call_time)
+        if policy == ScoutApm::SlowRequestPolicy::CAPTURE_NONE
+          return [nil, {}]
+        end
+
+        # increment the slow transaction count if this is a slow transaction.
+        meta = MetricMeta.new("SlowTransaction/#{scope.legacy_metric_name}")
+        stat = MetricStats.new
+        stat.update!(1)
+
+        @backtraces = [] # An Array of MetricMetas that have a backtrace
+
+        uri = request.annotations[:uri] || ""
+
+        metrics = create_metrics
+        # Disable stackprof output for now
+        stackprof = [] # request.stackprof
+
+        [
+          SlowTransaction.new(uri,
+                              scope.legacy_metric_name,
+                              root_layer.total_call_time,
+                              metrics,
+                              request.context,
+                              root_layer.stop_time,
+                              stackprof),
+          { meta => stat }
+        ]
+      end
+
+      # Iterates over the TrackedRequest's MetricMetas that have backtraces and attaches each to correct MetricMeta in the Metric Hash.
+      def attach_backtraces(metric_hash)
+        @backtraces.each do |meta_with_backtrace|
+          metric_hash.keys.find { |k| k == meta_with_backtrace }.backtrace = meta_with_backtrace.backtrace
+        end
+        metric_hash
+      end
+
+      # Full metrics from this request. These get aggregated in Store for the
+      # overview metrics, or stored permanently in a SlowTransaction
+      # Some merging of metrics will happen here, so if a request calls the same
+      # ActiveRecord or View repeatedly, it'll get merged.
+      def create_metrics
+        metric_hash = Hash.new
+
+        # Keep a list of subscopes, but only ever use the front one.  The rest
+        # get pushed/popped in cases when we have many levels of subscopable
+        # layers.  This lets us push/pop without otherwise keeping track very closely.
+        subscope_layers = []
+
+        walker.before do |layer|
+          if layer.subscopable?
+            subscope_layers.push(layer)
+          end
+        end
+
+        walker.after do |layer|
+          if layer.subscopable?
+            subscope_layers.pop
+          end
+        end
+
+        walker.walk do |layer|
+          meta_options = if subscope_layers.first && layer != subscope_layers.first # Don't scope under ourself.
+                           subscope_name = subscope_layers.first.legacy_metric_name
+                           {:scope => subscope_name}
+                         elsif layer == scope_layer # We don't scope the controller under itself
+                           {}
+                         else
+                           {:scope => scope_layer.legacy_metric_name}
+                         end
+
+          # Specific Metric
+          meta_options.merge!(:desc => layer.desc.to_s) if layer.desc
+          meta = MetricMeta.new(layer.legacy_metric_name, meta_options)
+          if layer.backtrace
+            bt = ScoutApm::Utils::BacktraceParser.new(layer.backtrace).call
+            if bt.any? # we could walk thru the call stack and not find in-app code
+              meta.backtrace = bt
+              # Why not just call meta.backtrace and call it done? The walker could access a later later that generates the same MetricMeta but doesn't have a backtrace. This could be
+              # lost in the metric_hash if it is replaced by the new key.
+              @backtraces << meta
+            else
+              ScoutApm::Agent.instance.logger.debug { "Unable to capture an app-specific backtrace for #{meta.inspect}\n#{layer.backtrace}" }
+            end
+          end
+          metric_hash[meta] ||= MetricStats.new( meta_options.has_key?(:scope) )
+          stat = metric_hash[meta]
+          stat.update!(layer.total_call_time, layer.total_exclusive_time)
+
+          # Merged Metric (no specifics, just sum up by type)
+          meta = MetricMeta.new("#{layer.type}/all")
+          metric_hash[meta] ||= MetricStats.new(false)
+          stat = metric_hash[meta]
+          stat.update!(layer.total_call_time, layer.total_exclusive_time)
+        end
+
+        metric_hash = attach_backtraces(metric_hash)
+
+        metric_hash
+      end
+    end
+  end
+end

data/lib/scout_apm/metric_meta.rb

@@ -38,6 +38,15 @@ class MetricMeta
     self.eql?(o)
   end
 
+  # This should be abstracted to a true accessor ... earned it.
+  def backtrace=(bt)
+    extra[:backtrace] = bt
+  end
+
+  def backtrace
+    extra[:backtrace]
+  end
+
   def hash
     h = metric_name.downcase.hash
     h ^= scope.downcase.hash unless scope.nil?

data/lib/scout_apm/metric_set.rb

@@ -0,0 +1,44 @@
+module ScoutApm
+  class MetricSet
+    # We can't aggregate CPU, Memory, Capacity, or Controller, so pass through these metrics directly
+    # TODO: Figure out a way to not have this duplicate what's in Samplers, and also on server's ingest
+    PASSTHROUGH_METRICS = ["CPU", "Memory", "Instance", "Controller", "SlowTransaction"]
+
+    attr_reader :metrics
+
+    def initialize
+      @metrics = Hash.new
+    end
+
+    def absorb_all(metrics)
+      Array(metrics).each { |m| absorb(m) }
+    end
+
+    # Absorbs a single new metric into the aggregates
+    def absorb(metric)
+      meta, stat = metric
+
+      if PASSTHROUGH_METRICS.include?(meta.type) # Leave as-is, don't attempt to combine into an /all key
+        @metrics[meta] ||= MetricStats.new
+        @metrics[meta].combine!(stat)
+
+      elsif meta.type == "Errors" # Sadly special cased, we want both raw and aggregate values
+        @metrics[meta] ||= MetricStats.new
+        @metrics[meta].combine!(stat)
+        agg_meta = MetricMeta.new("Errors/Request", :scope => meta.scope)
+        @metrics[agg_meta] ||= MetricStats.new
+        @metrics[agg_meta].combine!(stat)
+
+      else # Combine down to a single /all key
+        agg_meta = MetricMeta.new("#{meta.type}/all", :scope => meta.scope)
+        @metrics[agg_meta] ||= MetricStats.new
+        @metrics[agg_meta].combine!(stat)
+      end
+    end
+
+    def combine!(other)
+      absorb_all(other.metrics)
+      self
+    end
+  end
+end

data/lib/scout_apm/reporter.rb

@@ -17,17 +17,24 @@ module ScoutApm
 
     # TODO: Parse & return a real response object, not the HTTP Response object
     def report(payload, headers = {})
-      post(uri, payload, headers)
+      Array(config.value('host')).each do |host|
+
+        full_uri = uri(host)
+        response = post(full_uri, payload, headers)
+        unless response && response.is_a?(Net::HTTPSuccess)
+          logger.warn "Error on checkin to #{full_uri.to_s}: #{response.inspect}"
+        end
+      end
     end
 
-    def uri
+    def uri(host)
       case type
       when :checkin
-        URI.parse("#{config.value('host')}/apps/checkin.scout?key=#{config.value('key')}&name=#{CGI.escape(Environment.instance.application_name)}")
+        URI.parse("#{host}/apps/checkin.scout?key=#{config.value('key')}&name=#{CGI.escape(Environment.instance.application_name)}")
       when :app_server_load
-        URI.parse("#{config.value('host')}/apps/app_server_load.scout?key=#{config.value('key')}&name=#{CGI.escape(Environment.instance.application_name)}")
+        URI.parse("#{host}/apps/app_server_load.scout?key=#{config.value('key')}&name=#{CGI.escape(Environment.instance.application_name)}")
       when :deploy_hook
-        URI.parse("#{config.value('host')}/apps/deploy.scout?key=#{config.value('key')}&name=#{CGI.escape(config.value('name'))}")
+        URI.parse("#{host}/apps/deploy.scout?key=#{config.value('key')}&name=#{CGI.escape(config.value('name'))}")
       end.tap{|u| logger.debug("Posting to #{u.to_s}")}
     end
 

data/lib/scout_apm/serializers/jobs_serializer_to_json.rb

@@ -0,0 +1,28 @@
+module ScoutApm
+  module Serializers
+    class JobsSerializerToJson
+      attr_reader :jobs
+
+      # Jobs is a pre-deduped/combined set of job records.
+      def initialize(jobs)
+        @jobs = jobs
+      end
+
+      # An array of job records
+      def as_json
+        jobs.map do |job|
+          {
+            "queue" => job.queue_name,
+            "name" => job.job_name,
+            "count" => job.run_count,
+            "errors" => job.errors,
+            "total_time" => job.total_time.as_json,
+            "exclusive_time" => job.exclusive_time.as_json,
+            "metrics" => MetricsToJsonSerializer.new(job.metrics).as_json, # New style of metrics
+          }
+        end
+      end
+    end
+  end
+end
+

data/lib/scout_apm/serializers/metrics_to_json_serializer.rb

@@ -0,0 +1,54 @@
+module ScoutApm
+  module Serializers
+    class MetricsToJsonSerializer
+      attr_reader :metrics
+
+      # A hash of meta => stat pairs
+      def initialize(metrics)
+        @metrics = metrics
+      end
+
+      def as_json
+        metrics.map{|meta, stat| metric_as_json(meta, stat) }
+      end
+
+      # Children metrics is a hash of meta=>stat pairs. Leave empty for no children.
+      # Supports only a single-level nesting, until we have redone metric
+      # classes, instead of Meta and Stats
+      def metric_as_json(meta, stat, child_metrics={})
+
+        { "bucket" => meta.type,
+          "name" => meta.name, # No scope values needed here, since it's implied by the nesting.
+
+          "count" => stat.call_count,
+          "total_call_time" => stat.total_call_time,
+          "total_exclusive_time" => stat.total_exclusive_time,
+          "min_call_time" => stat.min_call_time,
+          "max_call_time" => stat.max_call_time,
+
+          # Pretty unsure how to synthesize histograms out of what we store now
+          "total_histogram" => [
+            [stat.total_exclusive_time / stat.call_count, stat.call_count],
+          ],
+          "exclusive_histogram" => [
+            [stat.total_exclusive_time / stat.call_count, stat.call_count]
+          ],
+
+          "metrics" => transform_child_metrics(child_metrics),
+
+          # Will later hold the exact SQL, or URL or whatever other detail
+          # about this query is necessary
+          "detail" => { :desc => meta.desc }.merge(meta.extra || {}),
+        }
+      end
+
+      def transform_child_metrics(metrics)
+        metrics.map do |meta, stat|
+          metric_as_json(meta, stat)
+        end
+      end
+    end
+  end
+end
+
+

data/lib/scout_apm/serializers/payload_serializer.rb

@@ -2,9 +2,9 @@
 module ScoutApm
   module Serializers
     class PayloadSerializer
-      def self.serialize(metadata, metrics, slow_transactions)
+      def self.serialize(metadata, metrics, slow_transactions, jobs, slow_jobs)
         if ScoutApm::Agent.instance.config.value("report_format") == 'json'
-          ScoutApm::Serializers::PayloadSerializerToJson.serialize(metadata, metrics, slow_transactions)
+          ScoutApm::Serializers::PayloadSerializerToJson.serialize(metadata, metrics, slow_transactions, jobs, slow_jobs)
         else
           metadata = metadata.dup
           metadata.default = nil
@@ -13,7 +13,9 @@ module ScoutApm
           metrics.default = nil
           Marshal.dump(:metadata          => metadata,
                        :metrics           => metrics,
-                       :slow_transactions => slow_transactions)
+                       :slow_transactions => slow_transactions,
+                       :jobs              => jobs,
+                       :slow_jobs         => slow_jobs)
         end
       end
 

data/lib/scout_apm/serializers/payload_serializer_to_json.rb

@@ -2,13 +2,18 @@ module ScoutApm
   module Serializers
     module PayloadSerializerToJson
       class << self
-        def serialize(metadata, metrics, slow_transactions)
-          rearranged_metrics = rearrange_the_metrics(metrics)
-          rearranged_slow_transactions = rearrange_the_slow_transactions(slow_transactions)
+        def serialize(metadata, metrics, slow_transactions, jobs, slow_jobs)
           metadata.merge!({:payload_version => 2})
-          jsonify_hash({:metadata => metadata, :metrics => rearranged_metrics, :slow_transactions => rearranged_slow_transactions})
+
+          jsonify_hash({:metadata => metadata,
+                        :metrics => rearrange_the_metrics(metrics),
+                        :slow_transactions => rearrange_the_slow_transactions(slow_transactions),
+                        :jobs => JobsSerializerToJson.new(jobs).as_json,
+                        :slow_jobs => SlowJobsSerializerToJson.new(slow_jobs).as_json,
+                      })
         end
 
+        # Old style of metric serializing.
         def rearrange_the_metrics(metrics)
           metrics.to_a.map do |meta, stats|
             stats.as_json.merge(:key => meta.as_json)