scout_apm 3.0.0.pre11 → 3.0.0.pre12

data/lib/scout_apm/layer_converters/request_queue_time_converter.rb

@@ -4,38 +4,36 @@ module ScoutApm
 
       HEADERS = %w(X-Queue-Start X-Request-Start X-QUEUE-START X-REQUEST-START x-queue-start x-request-start)
 
-      # Headers is a hash of request headers.  In Rails, request.headers would be appropriate
-      def initialize(request)
-        super(request)
-        @headers = request.headers
+      def headers
+        request.headers
       end
 
-      def call
-        return {} unless headers
+      def record!
+        return unless request.web?
+
+        return unless headers
 
         raw_start = locate_timestamp
-        return {} unless raw_start
+        return unless raw_start
 
         parsed_start = parse(raw_start)
-        return {} unless parsed_start
+        return unless parsed_start
 
         request_start = root_layer.start_time
         queue_time = (request_start - parsed_start).to_f
 
         # If we end up with a negative value, just bail out and don't report anything
-        return {} if queue_time < 0
+        return if queue_time < 0
 
         meta = MetricMeta.new("QueueTime/Request", {:scope => scope_layer.legacy_metric_name})
         stat = MetricStats.new(true)
         stat.update!(queue_time)
 
-        { meta => stat }
+        @store.track!({ meta => stat })
       end
 
       private
 
-      attr_reader :headers
-
       # Looks through the possible headers with this data, and extracts the raw
       # value of the header
       # Returns nil if not found

data/lib/scout_apm/layer_converters/slow_job_converter.rb

@@ -1,31 +1,33 @@
+# Uses a different workflow than normal metrics. We ignore the shared walk of
+# the layer tree, and instead wait until we're sure we even want to do any
+# work. Only then do we go realize all the SlowJobRecord & metrics associated.
+#
 module ScoutApm
   module LayerConverters
     class SlowJobConverter < ConverterBase
-      def initialize(*)
-        super
-
-        # After call to super, so @request is populated
-        @points = if request.job?
-                    ScoutApm::Agent.instance.slow_job_policy.score(request)
-                  else
-                    -1
-                  end
-
-        setup_subscopable_callbacks
-      end
+      ###################
+      #  Converter API  #
+      ###################
+      def record!
+        return nil unless request.job?
+        @points = ScoutApm::Agent.instance.slow_job_policy.score(request)
 
-      def name
-        request.unique_name
+        # Let the store know we're here, and if it wants our data, it will call
+        # back into #call
+        @store.track_slow_job!(self)
       end
 
-      def score
-        @points
-      end
+      #####################
+      #  ScoreItemSet API #
+      #####################
+      def name; request.unique_name; end
+      def score; @points; end
 
+      # Called by the set to force this to actually be created.
       def call
         return nil unless request.job?
-        return nil unless queue_layer
-        return nil unless job_layer
+        return nil unless layer_finder.queue
+        return nil unless layer_finder.job
 
         ScoutApm::Agent.instance.slow_job_policy.stored!(request)
 
@@ -54,26 +56,15 @@ module ScoutApm
         )
       end
 
-      def queue_layer
-        @queue_layer ||= find_first_layer_of_type("Queue")
-      end
-
-      def job_layer
-        @job_layer ||= find_first_layer_of_type("Job")
-      end
-
-
-      # The queue_layer is useful to capture for other reasons, but doesn't
-      # create a MetricMeta/Stat of its own
-      def skip_layer?(layer)
-        super(layer) || layer == queue_layer
-      end
-
       def create_metrics
+        # Create a new walker, and wire up the subscope stuff
+        walker = LayerConverters::DepthFirstWalker.new(self.root_layer)
+        register_hooks(walker)
+
         metric_hash = Hash.new
         allocation_metric_hash = Hash.new
 
-        walker.walk do |layer|
+        walker.on do |layer|
           next if skip_layer?(layer)
 
           debug_scoutprof(layer)
@@ -82,11 +73,18 @@ module ScoutApm
           store_aggregate_metric(layer, metric_hash, allocation_metric_hash)
         end
 
+        # And now run through the walk we just defined
+        walker.walk
+
         metric_hash = attach_backtraces(metric_hash)
         allocation_metric_hash = attach_backtraces(allocation_metric_hash)
 
         [metric_hash, allocation_metric_hash]
       end
+
+      def skip_layer?(layer); super(layer) || layer == queue_layer; end
+      def queue_layer; layer_finder.queue; end
+      def job_layer; layer_finder.job; end
     end
   end
 end

data/lib/scout_apm/layer_converters/slow_request_converter.rb

@@ -1,26 +1,23 @@
 module ScoutApm
   module LayerConverters
     class SlowRequestConverter < ConverterBase
-      def initialize(*)
-        super
-
-        # After call to super, so @request is populated
-        @points = if request.web?
-                    ScoutApm::Agent.instance.slow_request_policy.score(request)
-                  else
-                    -1
-                  end
+      ###################
+      #  Converter API  #
+      ###################
+      def record!
+        return nil unless request.web?
+        @points = ScoutApm::Agent.instance.slow_request_policy.score(request)
 
-        setup_subscopable_callbacks
+        # Let the store know we're here, and if it wants our data, it will call
+        # back into #call
+        @store.track_slow_transaction!(self)
       end
 
-      def name
-        request.unique_name
-      end
-
-      def score
-        @points
-      end
+      #####################
+      #  ScoreItemSet API #
+      #####################
+      def name; request.unique_name; end
+      def score; @points; end
 
       # Unconditionally attempts to convert this into a SlowTransaction object.
       # Can return nil if the request didn't have any scope_layer.
@@ -61,10 +58,14 @@ module ScoutApm
       #
       # This returns a 2-element of Metric Hashes (the first element is timing metrics, the second element is allocation metrics)
       def create_metrics
+        # Create a new walker, and wire up the subscope stuff
+        walker = LayerConverters::DepthFirstWalker.new(self.root_layer)
+        register_hooks(walker)
+
         metric_hash = Hash.new
         allocation_metric_hash = Hash.new
 
-        walker.walk do |layer|
+        walker.on do |layer|
           next if skip_layer?(layer)
 
           debug_scoutprof(layer)
@@ -73,6 +74,9 @@ module ScoutApm
           store_aggregate_metric(layer, metric_hash, allocation_metric_hash)
         end
 
+        # And now run through the walk we just defined
+        walker.walk
+
         metric_hash = attach_backtraces(metric_hash)
         allocation_metric_hash = attach_backtraces(allocation_metric_hash)
 

data/lib/scout_apm/limited_layer.rb

@@ -66,6 +66,10 @@ module ScoutApm
       "<LimitedLayer type=#{type} count=#{count}>"
     end
 
+    def limited?
+      true
+    end
+
     ######################################################
     #  Stub out some methods with static default values  #
     ######################################################

data/lib/scout_apm/metric_meta.rb

@@ -33,11 +33,6 @@ class MetricMeta
     !!(metric_name =~ /\A(Controller|Job)\//)
   end
 
-  # To avoid conflicts with different JSON libaries
-  def to_json(*a)
-     %Q[{"metric_id":#{metric_id || 'null'},"metric_name":#{metric_name.to_json},"scope":#{scope.to_json || 'null'}}]
-  end
-
   def ==(o)
     self.eql?(o)
   end

data/lib/scout_apm/metric_stats.rb

@@ -58,7 +58,14 @@ class MetricStats
   end
 
   def as_json
-    json_attributes = [:call_count, :total_call_time, :total_exclusive_time, :min_call_time, :max_call_time, :traces]
+    json_attributes = [
+      :call_count,
+      :max_call_time,
+      :min_call_time,
+      :total_call_time,
+      :total_exclusive_time,
+      :traces,
+    ]
     ScoutApm::AttributeArranger.call(self, json_attributes)
   end
 end

data/lib/scout_apm/serializers/db_query_serializer_to_json.rb

@@ -0,0 +1,15 @@
+module ScoutApm
+  module Serializers
+    class DbQuerySerializerToJson
+      attr_reader :db_query_metrics
+
+      def initialize(db_query_metrics)
+        @db_query_metrics = db_query_metrics
+      end
+
+      def as_json
+        db_query_metrics.map{|metric| metric.as_json }
+      end
+    end
+  end
+end

data/lib/scout_apm/serializers/payload_serializer.rb

@@ -2,9 +2,9 @@
 module ScoutApm
   module Serializers
     class PayloadSerializer
-      def self.serialize(metadata, metrics, slow_transactions, jobs, slow_jobs, histograms)
+      def self.serialize(metadata, metrics, slow_transactions, jobs, slow_jobs, histograms, db_query_metrics)
         if ScoutApm::Agent.instance.config.value("report_format") == 'json'
-          ScoutApm::Serializers::PayloadSerializerToJson.serialize(metadata, metrics, slow_transactions, jobs, slow_jobs, histograms)
+          ScoutApm::Serializers::PayloadSerializerToJson.serialize(metadata, metrics, slow_transactions, jobs, slow_jobs, histograms, db_query_metrics)
         else
           metadata = metadata.dup
           metadata.default = nil
@@ -21,7 +21,8 @@ module ScoutApm
                        # array, use this to maintain compatibility with json
                        # payloads. At this point, the marshal code branch is
                        # very rarely used anyway.
-                       :histograms        => HistogramsSerializerToJson.new(histograms).as_json)
+                       :histograms        => HistogramsSerializerToJson.new(histograms).as_json,
+                       :db_query_metrics  => db_query_metrics)
         end
       end
 

data/lib/scout_apm/serializers/payload_serializer_to_json.rb

@@ -2,7 +2,7 @@ module ScoutApm
   module Serializers
     module PayloadSerializerToJson
       class << self
-        def serialize(metadata, metrics, slow_transactions, jobs, slow_jobs, histograms)
+        def serialize(metadata, metrics, slow_transactions, jobs, slow_jobs, histograms, db_query_metrics)
           metadata.merge!({:payload_version => 2})
 
           jsonify_hash({:metadata => metadata,
@@ -11,7 +11,10 @@ module ScoutApm
                         :jobs => JobsSerializerToJson.new(jobs).as_json,
                         :slow_jobs => SlowJobsSerializerToJson.new(slow_jobs).as_json,
                         :histograms => HistogramsSerializerToJson.new(histograms).as_json,
-                      })
+                        :db_metrics => {
+                          :query => DbQuerySerializerToJson.new(db_query_metrics).as_json,
+                        },
+          })
         end
 
         # For the old style of metric serializing.

data/lib/scout_apm/slow_job_policy.rb

@@ -31,7 +31,7 @@ module ScoutApm
     end
 
     def stored!(request)
-      last_seen[unique_name_for(request)] = Time.now
+      last_seen[request.unique_name] = Time.now
     end
 
     # Determine if this job trace should be fully analyzed by scoring it
@@ -61,15 +61,6 @@ module ScoutApm
 
     private
 
-    def unique_name_for(request)
-      scope_layer = LayerConverters::ConverterBase.new(request).scope_layer
-      if scope_layer
-        scope_layer.legacy_metric_name
-      else
-        :unknown
-      end
-    end
-
     # Time in seconds
     # Logarithm keeps huge times from swamping the other metrics.
     # 1+ is necessary to keep the log function in positive territory.

data/lib/scout_apm/slow_request_policy.rb

@@ -31,7 +31,7 @@ module ScoutApm
     end
 
     def stored!(request)
-      last_seen[unique_name_for(request)] = Time.now
+      last_seen[request.unique_name] = Time.now
     end
 
     # Determine if this request trace should be fully analyzed by scoring it
@@ -61,15 +61,6 @@ module ScoutApm
 
     private
 
-    def unique_name_for(request)
-      scope_layer = LayerConverters::ConverterBase.new(request).scope_layer
-      if scope_layer
-        scope_layer.legacy_metric_name
-      else
-        :unknown
-      end
-    end
-
     # Time in seconds
     # Logarithm keeps huge times from swamping the other metrics.
     # 1+ is necessary to keep the log function in positive territory.

data/lib/scout_apm/store.rb

@@ -3,12 +3,6 @@
 # the layaway file for cross-process aggregation.
 module ScoutApm
   class Store
-    # A hash of reporting periods. { StoreReportingPeriodTimestamp => StoreReportingPeriod }
-    attr_reader :reporting_periods
-
-    # Used to pull metrics into each reporting period, as that reporting period is finished.
-    attr_reader :samplers
-
     def initialize
       @mutex = Mutex.new
       @reporting_periods = Hash.new { |h,k| h[k] = StoreReportingPeriod.new(k) }
@@ -20,32 +14,41 @@ module ScoutApm
     end
 
     def current_period
-      reporting_periods[current_timestamp]
+      @reporting_periods[current_timestamp]
     end
+    private :current_period
+
+    def find_period(timestamp = nil)
+      if timestamp
+        @reporting_periods[timestamp]
+      else
+        current_period
+      end
+    end
+    private :find_period
 
     # Save newly collected metrics
     def track!(metrics, options={})
       @mutex.synchronize {
-        period = if options[:timestamp]
-                   @reporting_periods[options[:timestamp]]
-                 else
-                   current_period
-                 end
+        period = find_period(options[:timestamp])
         period.absorb_metrics!(metrics)
       }
     end
 
     def track_histograms!(histograms, options={})
       @mutex.synchronize {
-        period = if options[:timestamp]
-                   @reporting_periods[options[:timestamp]]
-                 else
-                   current_period
-                 end
+        period = find_period(options[:timestamp])
         period.merge_histograms!(histograms)
       }
     end
 
+    def track_db_query_metrics!(db_query_metric_set, options={})
+      @mutex.synchronize {
+        period = find_period(options[:timestamp])
+        period.merge_db_query_metrics!(db_query_metric_set)
+      }
+    end
+
     def track_one!(type, name, value, options={})
       meta = MetricMeta.new("#{type}/#{name}")
       stat = MetricStats.new(false)
@@ -83,7 +86,7 @@ module ScoutApm
     def write_to_layaway(layaway, force=false)
       ScoutApm::Agent.instance.logger.debug("Writing to layaway#{" (Forced)" if force}")
 
-        reporting_periods.select { |time, rp| force || (time.timestamp < current_timestamp.timestamp) }.
+        @reporting_periods.select { |time, rp| force || (time.timestamp < current_timestamp.timestamp) }.
                           each   { |time, rp| collect_samplers(rp) }.
                           each   { |time, rp| write_reporting_period(layaway, time, rp) }
     end
@@ -95,10 +98,11 @@ module ScoutApm
     rescue => e
       ScoutApm::Agent.instance.logger.warn("Failed writing data to layaway file: #{e.message} / #{e.backtrace}")
     ensure
-      ScoutApm::Agent.instance.logger.debug("Before delete, reporting periods length: #{reporting_periods.size}")
-      deleted_items = reporting_periods.delete(time)
-      ScoutApm::Agent.instance.logger.debug("After delete, reporting periods length: #{reporting_periods.size}. Did delete #{deleted_items}")
+      ScoutApm::Agent.instance.logger.debug("Before delete, reporting periods length: #{@reporting_periods.size}")
+      deleted_items = @reporting_periods.delete(time)
+      ScoutApm::Agent.instance.logger.debug("After delete, reporting periods length: #{@reporting_periods.size}. Did delete #{deleted_items}")
     end
+    private :write_reporting_period
 
     ######################################
     # Sampler support
@@ -116,6 +120,7 @@ module ScoutApm
         end
       end
     end
+    private :collect_samplers
   end
 
   # A timestamp, normalized to the beginning of a minute. Used as a hash key to
@@ -184,6 +189,8 @@ module ScoutApm
 
     attr_reader :metric_set
 
+    attr_reader :db_query_metric_set
+
     def initialize(timestamp)
       @timestamp = timestamp
 
@@ -193,6 +200,8 @@ module ScoutApm
       @histograms = []
 
       @metric_set = MetricSet.new
+      @db_query_metric_set = DbQueryMetricSet.new
+
       @jobs = Hash.new
     end
 
@@ -203,7 +212,8 @@ module ScoutApm
         merge_slow_transactions!(other.slow_transactions_payload).
         merge_jobs!(other.jobs).
         merge_slow_jobs!(other.slow_jobs_payload).
-        merge_histograms!(other.histograms)
+        merge_histograms!(other.histograms).
+        merge_db_query_metrics!(other.db_query_metric_set)
       self
     end
 
@@ -224,6 +234,11 @@ module ScoutApm
       self
     end
 
+    def merge_db_query_metrics!(other_metric_set)
+      db_query_metric_set.combine!(other_metric_set)
+      self
+    end
+
     def merge_slow_transactions!(new_transactions)
       Array(new_transactions).each do |one_transaction|
         request_traces << one_transaction
@@ -282,6 +297,10 @@ module ScoutApm
       job_traces.to_a
     end
 
+    def db_query_metrics_payload
+      db_query_metric_set.metrics_to_report
+    end
+
     #################################
     # Debug Helpers
     #################################