scout_apm 2.2.0.pre3 → 2.3.0.pre

data/lib/scout_apm/slow_job_policy.rb

@@ -31,7 +31,7 @@ module ScoutApm
     end
 
     def stored!(request)
-      last_seen[unique_name_for(request)] = Time.now
+      last_seen[request.unique_name] = Time.now
     end
 
     # Determine if this job trace should be fully analyzed by scoring it
@@ -61,15 +61,6 @@ module ScoutApm
 
     private
 
-    def unique_name_for(request)
-      scope_layer = LayerConverters::ConverterBase.new(request).scope_layer
-      if scope_layer
-        scope_layer.legacy_metric_name
-      else
-        :unknown
-      end
-    end
-
     # Time in seconds
     # Logarithm keeps huge times from swamping the other metrics.
     # 1+ is necessary to keep the log function in positive territory.

data/lib/scout_apm/slow_job_record.rb

@@ -20,8 +20,10 @@ module ScoutApm
     attr_reader :hostname
     attr_reader :seconds_since_startup
     attr_reader :score
+    attr_reader :git_sha
+    attr_reader :truncated_metrics
 
-    def initialize(queue_name, job_name, time, total_time, exclusive_time, context, metrics, allocation_metrics, mem_delta, allocations, score)
+    def initialize(queue_name, job_name, time, total_time, exclusive_time, context, metrics, allocation_metrics, mem_delta, allocations, score, truncated_metrics)
       @queue_name = queue_name
       @job_name = job_name
       @time = time
@@ -34,7 +36,10 @@ module ScoutApm
       @allocations = allocations
       @seconds_since_startup = (Time.now - ScoutApm::Agent.instance.process_start_time)
       @hostname = ScoutApm::Environment.instance.hostname
+      @git_sha = ScoutApm::Environment.instance.git_revision.sha
       @score = score
+      @truncated_metrics = truncated_metrics
+
       ScoutApm::Agent.instance.logger.debug { "Slow Job [#{metric_name}] - Call Time: #{total_call_time} Mem Delta: #{mem_delta}"}
     end
 

data/lib/scout_apm/slow_request_policy.rb

@@ -31,7 +31,7 @@ module ScoutApm
     end
 
     def stored!(request)
-      last_seen[unique_name_for(request)] = Time.now
+      last_seen[request.unique_name] = Time.now
     end
 
     # Determine if this request trace should be fully analyzed by scoring it
@@ -61,15 +61,6 @@ module ScoutApm
 
     private
 
-    def unique_name_for(request)
-      scope_layer = LayerConverters::ConverterBase.new(request).scope_layer
-      if scope_layer
-        scope_layer.legacy_metric_name
-      else
-        :unknown
-      end
-    end
-
     # Time in seconds
     # Logarithm keeps huge times from swamping the other metrics.
     # 1+ is necessary to keep the log function in positive territory.

data/lib/scout_apm/slow_transaction.rb

@@ -15,8 +15,11 @@ module ScoutApm
     attr_reader :allocations
     attr_accessor :hostname # hack - we need to reset these server side.
     attr_accessor :seconds_since_startup # hack - we need to reset these server side.
+    attr_accessor :git_sha # hack - we need to reset these server side.
 
-    def initialize(uri, metric_name, total_call_time, metrics, allocation_metrics, context, time, raw_stackprof, mem_delta, allocations, score)
+    attr_reader :truncated_metrics # True/False that says if we had to truncate the metrics of this trace
+
+    def initialize(uri, metric_name, total_call_time, metrics, allocation_metrics, context, time, raw_stackprof, mem_delta, allocations, score, truncated_metrics)
       @uri = uri
       @metric_name = metric_name
       @total_call_time = total_call_time
@@ -30,6 +33,9 @@ module ScoutApm
       @seconds_since_startup = (Time.now - ScoutApm::Agent.instance.process_start_time)
       @hostname = ScoutApm::Environment.instance.hostname
       @score = score
+      @git_sha = ScoutApm::Environment.instance.git_revision.sha
+      @truncated_metrics = truncated_metrics
+
       ScoutApm::Agent.instance.logger.debug { "Slow Request [#{uri}] - Call Time: #{total_call_time} Mem Delta: #{mem_delta} Score: #{score}"}
     end
 
@@ -44,7 +50,19 @@ module ScoutApm
     end
 
     def as_json
-      json_attributes = [:key, :time, :total_call_time, :uri, [:context, :context_hash], :score, :prof, :mem_delta, :allocations, :seconds_since_startup, :hostname]
+      json_attributes = [:key,
+                         :time,
+                         :total_call_time,
+                         :uri,
+                         [:context, :context_hash],
+                         :score,
+                         :prof,
+                         :mem_delta,
+                         :allocations,
+                         :seconds_since_startup,
+                         :hostname,
+                         :git_sha,
+                         :truncated_metrics]
       ScoutApm::AttributeArranger.call(self, json_attributes)
     end
 

data/lib/scout_apm/store.rb

@@ -23,10 +23,33 @@ module ScoutApm
       reporting_periods[current_timestamp]
     end
 
+    def find_period(timestamp = nil)
+      if timestamp
+        @reporting_periods[timestamp]
+      else
+        current_period
+      end
+    end
+
     # Save newly collected metrics
     def track!(metrics, options={})
       @mutex.synchronize {
-        current_period.absorb_metrics!(metrics)
+        period = find_period(options[:timestamp])
+        period.absorb_metrics!(metrics)
+      }
+    end
+
+    def track_histograms!(histograms, options={})
+      @mutex.synchronize {
+        period = find_period(options[:timestamp])
+        period.merge_histograms!(histograms)
+      }
+    end
+
+    def track_db_query_metrics!(db_query_metric_set, options={})
+      @mutex.synchronize {
+        period = find_period(options[:timestamp])
+        period.merge_db_query_metrics!(db_query_metric_set)
       }
     end
 
@@ -67,19 +90,21 @@ module ScoutApm
     def write_to_layaway(layaway, force=false)
       ScoutApm::Agent.instance.logger.debug("Writing to layaway#{" (Forced)" if force}")
 
-      @mutex.synchronize {
-        reporting_periods.select { |time, rp| force || time.timestamp < current_timestamp.timestamp}.
+        reporting_periods.select { |time, rp| force || (time.timestamp < current_timestamp.timestamp) }.
+                          each   { |time, rp| collect_samplers(rp) }.
                           each   { |time, rp| write_reporting_period(layaway, time, rp) }
-      }
     end
 
     def write_reporting_period(layaway, time, rp)
-      collect_samplers(rp)
-      layaway.write_reporting_period(rp)
+      @mutex.synchronize {
+        layaway.write_reporting_period(rp)
+      }
     rescue => e
       ScoutApm::Agent.instance.logger.warn("Failed writing data to layaway file: #{e.message} / #{e.backtrace}")
     ensure
-      reporting_periods.delete(time)
+      ScoutApm::Agent.instance.logger.debug("Before delete, reporting periods length: #{reporting_periods.size}")
+      deleted_items = reporting_periods.delete(time)
+      ScoutApm::Agent.instance.logger.debug("After delete, reporting periods length: #{reporting_periods.size}. Did delete #{deleted_items}")
     end
 
     ######################################
@@ -91,12 +116,10 @@ module ScoutApm
     def collect_samplers(rp)
       @samplers.each do |sampler|
         begin
-          metrics = sampler.metrics(rp.timestamp)
-          rp.absorb_metrics!(metrics)
+          sampler.metrics(rp.timestamp, self)
         rescue => e
           ScoutApm::Agent.instance.logger.info "Error reading #{sampler.human_name} for period: #{rp}"
-          ScoutApm::Agent.instance.logger.debug e.message
-          ScoutApm::Agent.instance.logger.debug e.backtrace.join("\n")
+          ScoutApm::Agent.instance.logger.debug "#{e.message}\n\t#{e.backtrace.join("\n\t")}"
         end
       end
     end
@@ -159,19 +182,28 @@ module ScoutApm
     # A ScoredItemSet holding the "best" traces for the period
     attr_reader :job_traces
 
+    # An Array of HistogramsReport
+    attr_reader :histograms
+
     # A StoreReportingPeriodTimestamp representing the time that this
     # collection of metrics is for
     attr_reader :timestamp
 
     attr_reader :metric_set
 
+    attr_reader :db_query_metric_set
+
     def initialize(timestamp)
       @timestamp = timestamp
 
       @request_traces = ScoredItemSet.new
       @job_traces = ScoredItemSet.new
 
+      @histograms = []
+
       @metric_set = MetricSet.new
+      @db_query_metric_set = DbQueryMetricSet.new
+
       @jobs = Hash.new
     end
 
@@ -181,7 +213,9 @@ module ScoutApm
         merge_metrics!(other.metric_set).
         merge_slow_transactions!(other.slow_transactions_payload).
         merge_jobs!(other.jobs).
-        merge_slow_jobs!(other.slow_jobs_payload)
+        merge_slow_jobs!(other.slow_jobs_payload).
+        merge_histograms!(other.histograms).
+        merge_db_query_metrics!(other.db_query_metric_set)
       self
     end
 
@@ -202,6 +236,11 @@ module ScoutApm
       self
     end
 
+    def merge_db_query_metrics!(other_metric_set)
+      db_query_metric_set.combine!(other_metric_set)
+      self
+    end
+
     def merge_slow_transactions!(new_transactions)
       Array(new_transactions).each do |one_transaction|
         request_traces << one_transaction
@@ -230,6 +269,17 @@ module ScoutApm
       self
     end
 
+    def merge_histograms!(new_histograms)
+      new_histograms = Array(new_histograms)
+      @histograms = (histograms + new_histograms).
+        group_by { |histo| histo.name }.
+        map { |(_, histos)|
+          histos.inject { |merged, histo| merged.combine!(histo) }
+        }
+
+      self
+    end
+
     #################################
     # Retrieve Metrics for reporting
     #################################
@@ -249,6 +299,10 @@ module ScoutApm
       job_traces.to_a
     end
 
+    def db_query_metrics_payload
+      db_query_metric_set.metrics_to_report
+    end
+
     #################################
     # Debug Helpers
     #################################

data/lib/scout_apm/synchronous_recorder.rb

@@ -0,0 +1,26 @@
+# Provide a synchronous approach to recording TrackedRequests
+# Doesn't attempt to background the work, or do it elsewhere. It happens
+# inline, in the caller thread right when record! is called
+
+module ScoutApm
+  class SynchronousRecorder
+    attr_reader :logger
+
+    def initialize(logger)
+      @logger = logger
+    end
+
+    def start
+      # nothing to do
+      self
+    end
+
+    def stop
+      # nothing to do
+    end
+
+    def record!(request)
+      request.record!
+    end
+  end
+end

data/lib/scout_apm/tracked_request.rb

@@ -42,6 +42,9 @@ module ScoutApm
     # Whereas the instant_key gets set per-request in reponse to a URL param, dev_trace is set in the config file
     attr_accessor :dev_trace
 
+    # An object that responds to `record!(TrackedRequest)` to store this tracked request
+    attr_reader :recorder
+
     def initialize(store)
       @store = store #this is passed in so we can use a real store (normal operation) or fake store (instant mode only)
       @layers = []
@@ -51,26 +54,35 @@ module ScoutApm
       @context = Context.new
       @root_layer = nil
       @error = false
+      @stopping = false
       @instant_key = nil
       @mem_start = mem_usage
-      @dev_trace =  ScoutApm::Agent.instance.config.value('dev_trace') && Rails.env.development?
+      @dev_trace =  ScoutApm::Agent.instance.config.value('dev_trace') && ScoutApm::Agent.instance.environment.env == "development"
+      @recorder = ScoutApm::Agent.instance.recorder
+
+      ignore_request! if @recorder.nil?
     end
 
     def start_layer(layer)
-      if ignoring_children?
-        return
-      end
+      # If we're already stopping, don't do additional layers
+      return if stopping?
 
-      layer.start_sampling
+      return if ignoring_children?
+
+      return ignoring_start_layer if ignoring_request?
 
       start_request(layer) unless @root_layer
-      @layers[-1].add_child(layer) if @layers.any?
       @layers.push(layer)
     end
 
     def stop_layer
+      # If we're already stopping, don't do additional layers
+      return if stopping?
+
       return if ignoring_children?
 
+      return ignoring_stop_layer if ignoring_request?
+
       layer = @layers.pop
 
       # Safeguard against a mismatch in the layer tracking in an instrument.
@@ -78,15 +90,16 @@ module ScoutApm
       # lined up correctly. If stop_layer gets called twice, when it should
       # only have been called once you'll end up with this error.
       if layer.nil?
-        ScoutApm::Agent.instance.logger.warn("Error stopping layer, was nil. Root Layer: #{@root_layer.inspect}")
+        logger.warn("Error stopping layer, was nil. Root Layer: #{@root_layer.inspect}")
         stop_request
         return
       end
 
-      layer.record_traces!
       layer.record_stop_time!
       layer.record_allocations!
 
+      @layers[-1].add_child(layer) if @layers.any?
+
       # This must be called before checking if a backtrace should be collected as the call count influences our capture logic.
       # We call `#update_call_counts in stop layer to ensure the layer has a final desc. Layer#desc is updated during the AR instrumentation flow.
       update_call_counts!(layer)
@@ -96,8 +109,6 @@ module ScoutApm
 
       if finalized?
         stop_request
-      else
-        continue_sampling_for_layers if ScoutApm::Agent.instance.config.value('profile')
       end
     end
 
@@ -115,6 +126,8 @@ module ScoutApm
 
     BACKTRACE_BLACKLIST = ["Controller", "Job"]
     def capture_backtrace?(layer)
+      return if ignoring_request?
+
       # Never capture backtraces for this kind of layer. The backtrace will
       # always be 100% framework code.
       return false if BACKTRACE_BLACKLIST.include?(layer.type)
@@ -158,13 +171,6 @@ module ScoutApm
       @layers.none?
     end
 
-    def continue_sampling_for_layers
-      if last_traced_layer = @layers.select{|layer| layer.traced?}.last
-        ScoutApm::Instruments::Stacks.update_indexes(@layers.last.frame_index, @layers.last.trace_index)
-        ScoutApm::Instruments::Stacks.start_sampling
-      end
-    end
-
     # Run at the beginning of the whole request
     #
     # * Capture the first layer as the root_layer
@@ -176,21 +182,15 @@ module ScoutApm
     #
     # * Send the request off to be stored
     def stop_request
-      if ScoutApm::Agent.instance.config.value('profile')
-        ScoutApm::Instruments::Stacks.stop_sampling(true)
-        ScoutApm::Instruments::Stacks.update_indexes(0, 0)
-      end
-      record!
-    end
+      @stopping = true
 
-    # Enable ScoutProf for this thread
-    def enable_profiled_thread!
-        ScoutApm::Instruments::Stacks.add_profiled_thread
+      if recorder
+        recorder.record!(self)
+      end
     end
 
-    # Disable ScoutProf for this thread
-    def disable_profiled_thread!
-        ScoutApm::Instruments::Stacks.remove_profiled_thread
+    def stopping?
+      @stopping
     end
 
     ###################################
@@ -236,6 +236,8 @@ module ScoutApm
     end
 
     def instant?
+      return false if ignoring_request?
+
       instant_key
     end
 
@@ -243,62 +245,61 @@ module ScoutApm
     # Persist the Request
     ###################################
 
+    def recorded!
+      @recorded = true
+    end
+
     # Convert this request to the appropriate structure, then report it into
     # the peristent Store object
     def record!
-      @recorded = true
+      recorded!
+
+      return if ignoring_request?
+
+      # If we didn't have store, but we're trying to record anyway, go
+      # figure that out. (this happens in Remote Agent scenarios)
+      restore_store if @store.nil?
 
       # Bail out early if the user asked us to ignore this uri
       return if ScoutApm::Agent.instance.ignored_uris.ignore?(annotations[:uri])
 
-      # Update immediate and long-term histograms for both job and web requests
-      if unique_name != :unknown
-        ScoutApm::Agent.instance.request_histograms.add(unique_name, root_layer.total_call_time)
-        ScoutApm::Agent.instance.request_histograms_by_time[@store.current_timestamp].
-          add(unique_name, root_layer.total_call_time)
+      converters = [
+        LayerConverters::Histograms,
+        LayerConverters::MetricConverter,
+        LayerConverters::ErrorConverter,
+        LayerConverters::AllocationMetricConverter,
+        LayerConverters::RequestQueueTimeConverter,
+        LayerConverters::JobConverter,
+        LayerConverters::DatabaseConverter,
+
+        LayerConverters::SlowJobConverter,
+        LayerConverters::SlowRequestConverter,
+      ]
+
+      layer_finder = LayerConverters::FindLayerByType.new(self)
+      walker = LayerConverters::DepthFirstWalker.new(self.root_layer)
+      converters = converters.map do |klass|
+        instance = klass.new(self, layer_finder, @store)
+        instance.register_hooks(walker)
+        instance
       end
-
-      metrics = LayerConverters::MetricConverter.new(self).call
-      @store.track!(metrics)
-
-      error_metrics = LayerConverters::ErrorConverter.new(self).call
-      @store.track!(error_metrics)
-
-      allocation_metrics = LayerConverters::AllocationMetricConverter.new(self).call
-      @store.track!(allocation_metrics)
-
-      if web?
-        # Don't #call this - that's the job of the ScoredItemSet later.
-        slow_converter = LayerConverters::SlowRequestConverter.new(self)
-        @store.track_slow_transaction!(slow_converter)
-
-        queue_time_metrics = LayerConverters::RequestQueueTimeConverter.new(self).call
-        @store.track!(queue_time_metrics)
-
-        # If there's an instant_key, it means we need to report this right away
-        if instant?
-          trace = slow_converter.call
-          ScoutApm::InstantReporting.new(trace, instant_key).call()
-        end
+      walker.walk
+      converters.each {|i| i.record! }
+
+      # If there's an instant_key, it means we need to report this right away
+      if web? && instant?
+        converter = converters.find{|c| c.class == LayerConverters::SlowRequestConverter}
+        trace = converter.call
+        ScoutApm::InstantReporting.new(trace, instant_key).call
       end
-
-      if job?
-        job_metrics = LayerConverters::JobConverter.new(self).call
-        @store.track_job!(job_metrics)
-
-        job_converter = LayerConverters::SlowJobConverter.new(self)
-        @store.track_slow_job!(job_converter)
-      end
-
-      allocation_metrics = LayerConverters::AllocationMetricConverter.new(self).call
-      @store.track!(allocation_metrics)
-
     end
 
     # Only call this after the request is complete
     def unique_name
+      return nil if ignoring_request?
+
       @unique_name ||= begin
-                         scope_layer = LayerConverters::ConverterBase.new(self).scope_layer
+                         scope_layer = LayerConverters::FindLayerByType.new(self).scope
                          if scope_layer
                            scope_layer.legacy_metric_name
                          else
@@ -311,6 +312,8 @@ module ScoutApm
     # Used to know when we should just create a new one (don't attempt to add
     # data to an already-recorded request). See RequestManager
     def recorded?
+      return ignoring_recorded? if ignoring_request?
+
       @recorded
     end
 
@@ -359,5 +362,67 @@ module ScoutApm
     def backtrace_threshold
       dev_trace ? 0.05 : 0.5 # the minimum threshold in seconds to record the backtrace for a metric.
     end
+
+    ################################################################################
+    # Ignoring the rest of a request
+    ################################################################################
+
+    # At any point in the request, calling code or instrumentation can call
+    # `ignore_request!` to immediately stop recording any information about new
+    # layers, and delete any existing layer info.  This class will still exist,
+    # and respond to methods as normal, but `record!` won't be called, and no
+    # data will be recorded.
+
+    def ignore_request!
+      return if @ignoring_request
+
+      # Set instance variable
+      @ignoring_request = true
+
+      # Store data we'll need
+      @ignoring_depth = @layers.length
+
+      # Clear data
+      @layers = []
+      @root_layer = nil
+      @call_set = nil
+      @annotations = {}
+      @instant_key = nil
+    end
+
+    def ignoring_request?
+      @ignoring_request
+    end
+
+    def ignoring_start_layer
+      @ignoring_depth += 1
+    end
+
+    def ignoring_stop_layer
+      @ignoring_depth -= 1
+    end
+
+    def ignoring_recorded?
+      @ignoring_depth <= 0
+    end
+
+    def logger
+      ScoutApm::Agent.instance.logger
+    end
+
+    # Actually go fetch & make-real any lazily created data.
+    # Clean up any cleverness in objects.
+    # Makes this object ready to be Marshal Dumped (or otherwise serialized)
+    def prepare_to_dump!
+      @call_set = nil
+      @store = nil
+      @recorder = nil
+    end
+
+    # Go re-fetch the store based on what the Agent's official one is. Used
+    # after hydrating a dumped TrackedRequest
+    def restore_store
+      @store = ScoutApm::Agent.instance.store
+    end
   end
 end