scout_apm 2.1.32 → 2.2.0.pre0

data/lib/scout_apm/request_histograms.rb

@@ -39,10 +39,6 @@ module ScoutApm
       initialize_histograms_hash
     end
 
-    def raw(item)
-      @histograms[item]
-    end
-
     def initialize_histograms_hash
       @histograms = Hash.new { |h, k| h[k] = NumericHistogram.new(histogram_size) }
     end

data/lib/scout_apm/request_manager.rb

@@ -11,7 +11,7 @@ module ScoutApm
     def self.find
       req = Thread.current[:scout_request]
 
-      if req && (req.stopping? || req.recorded?)
+      if req && req.recorded?
         nil
       else
         req
@@ -25,7 +25,6 @@ module ScoutApm
               else
                 ScoutApm::FakeStore.new
               end
-
       Thread.current[:scout_request] = TrackedRequest.new(store)
     end
   end

data/lib/scout_apm/scored_item_set.rb

@@ -63,12 +63,6 @@ module ScoutApm
       end
     end
 
-    # Equal to another set only if exactly the same set of items is inside
-    def eql?(other)
-      items == other.items
-    end
-
-    alias :== :eql?
 
     private
 
@@ -81,6 +75,5 @@ module ScoutApm
         items[new_item.name] = [new_item.score, new_item.call]
       end
     end
-
   end
 end

data/lib/scout_apm/serializers/deploy_serializer.rb

@@ -0,0 +1,16 @@
+# Serialize & deserialize deploy data up to the APM server
+module ScoutApm
+  module Serializers
+    class DeploySerializer
+      HTTP_HEADERS = {'Content-Type' => 'application/x-www-form-urlencoded'}
+
+      def self.serialize(data)
+        URI.encode_www_form(data)
+      end
+
+      def self.deserialize(data)
+        Marshal.load(data)
+      end
+    end
+  end
+end

data/lib/scout_apm/serializers/payload_serializer.rb

@@ -2,9 +2,9 @@
 module ScoutApm
   module Serializers
     class PayloadSerializer
-      def self.serialize(metadata, metrics, slow_transactions, jobs, slow_jobs, histograms)
+      def self.serialize(metadata, metrics, slow_transactions, jobs, slow_jobs)
         if ScoutApm::Agent.instance.config.value("report_format") == 'json'
-          ScoutApm::Serializers::PayloadSerializerToJson.serialize(metadata, metrics, slow_transactions, jobs, slow_jobs, histograms)
+          ScoutApm::Serializers::PayloadSerializerToJson.serialize(metadata, metrics, slow_transactions, jobs, slow_jobs)
         else
           metadata = metadata.dup
           metadata.default = nil
@@ -15,13 +15,7 @@ module ScoutApm
                        :metrics           => metrics,
                        :slow_transactions => slow_transactions,
                        :jobs              => jobs,
-                       :slow_jobs         => slow_jobs,
-
-                       # as_json returns a ruby object. Since it's not a simple
-                       # array, use this to maintain compatibility with json
-                       # payloads. At this point, the marshal code branch is
-                       # very rarely used anyway.
-                       :histograms        => HistogramsSerializerToJson.new(histograms).as_json)
+                       :slow_jobs         => slow_jobs)
         end
       end
 

data/lib/scout_apm/serializers/payload_serializer_to_json.rb

@@ -2,7 +2,7 @@ module ScoutApm
   module Serializers
     module PayloadSerializerToJson
       class << self
-        def serialize(metadata, metrics, slow_transactions, jobs, slow_jobs, histograms)
+        def serialize(metadata, metrics, slow_transactions, jobs, slow_jobs)
           metadata.merge!({:payload_version => 2})
 
           jsonify_hash({:metadata => metadata,
@@ -10,7 +10,6 @@ module ScoutApm
                         :slow_transactions => rearrange_the_slow_transactions(slow_transactions),
                         :jobs => JobsSerializerToJson.new(jobs).as_json,
                         :slow_jobs => SlowJobsSerializerToJson.new(slow_jobs).as_json,
-                        :histograms => HistogramsSerializerToJson.new(histograms).as_json,
                       })
         end
 
@@ -68,6 +67,10 @@ module ScoutApm
             %Q["#{formatee.iso8601}"]
           when nil
             "null"
+          when TrueClass
+            "true"
+          when FalseClass
+            "false"
           else # strings and everything
             %Q["#{escape(formatee)}"]
           end

data/lib/scout_apm/serializers/slow_jobs_serializer_to_json.rb

@@ -21,11 +21,9 @@ module ScoutApm
             "allocations" => job.allocations,
             "seconds_since_startup" => job.seconds_since_startup,
             "hostname" => job.hostname,
-            "git_sha" => job.git_sha,
             "metrics" => MetricsToJsonSerializer.new(job.metrics).as_json, # New style of metrics
             "allocation_metrics" => MetricsToJsonSerializer.new(job.allocation_metrics).as_json, # New style of metrics
             "context" => job.context.to_hash,
-            "truncated_metrics" => job.truncated_metrics,
 
             "score" => job.score,
           }
@@ -34,3 +32,4 @@ module ScoutApm
     end
   end
 end
+

data/lib/scout_apm/server_integrations/puma.rb

@@ -24,13 +24,10 @@ module ScoutApm
       end
 
       def install
-        old = ::Puma.cli_config.options[:before_worker_boot] || []
-        new = Array(old) + [Proc.new do
+        ::Puma.cli_config.options[:before_worker_boot] << Proc.new do
           logger.info "Installing Puma worker loop."
           ScoutApm::Agent.instance.start_background_worker
-        end]
-
-        ::Puma.cli_config.options[:before_worker_boot] = new
+        end
       rescue
         logger.warn "Unable to install Puma worker loop: #{$!.message}"
       end

data/lib/scout_apm/slow_item_set.rb

@@ -0,0 +1,80 @@
+# In order to keep load down, only record a sample of Slow Items (Transactions
+# or Jobs).  In order to make that sampling as fair as possible, follow a basic
+# algorithm:
+#
+# When adding a new Slow Item:
+#  * Just add it if there is an open spot
+#  * If there isn't an open spot, attempt to remove an over-represented
+#    item instead ("attempt_to_evict"). Overrepresented is simply "has more
+#    than @fair number of Matching Items in the set". The fastest of the
+#    overrepresented items is removed.
+#  * If there isn't an open spot, and no Item is valid to evict, drop the
+#    incoming Item without adding.
+#
+# There is no way to remove Items from this set, create a new object
+# for each reporting period.
+#
+# Item must respond to:
+#   #metric_name - string - grouping key to see if one kind of thing is overrepresented
+#   #total_call_time - float - duration of the item
+
+module ScoutApm
+  class SlowItemSet
+    include Enumerable
+
+    DEFAULT_TOTAL = 10
+    DEFAULT_FAIR = 1
+
+    attr_reader :total
+    attr_reader :fair
+
+    def initialize(total=DEFAULT_TOTAL, fair=DEFAULT_FAIR)
+      @total = total
+      @fair = fair
+      @items = []
+    end
+
+    def each
+      @items.each { |s| yield s }
+    end
+
+    def <<(item)
+      return if attempt_append(item)
+      attempt_to_evict
+      attempt_append(item)
+    end
+
+    def empty_slot?
+      @items.length < total
+    end
+
+    def attempt_append(item)
+      if empty_slot?
+        @items.push(item)
+        true
+      else
+        false
+      end
+    end
+
+    def attempt_to_evict
+      return if @items.length == 0
+
+      overrepresented = @items.
+        group_by { |item| unique_name_for(item) }.
+        to_a.
+        sort_by { |(_, items)| items.length }.
+        last
+
+      if overrepresented[1].length > fair
+        fastest = overrepresented[1].sort_by { |item| item.total_call_time }.first
+        @items.delete(fastest)
+      end
+    end
+
+    # Determine this items' "hash key"
+    def unique_name_for(item)
+      item.metric_name
+    end
+  end
+end

data/lib/scout_apm/slow_job_record.rb

@@ -20,10 +20,8 @@ module ScoutApm
     attr_reader :hostname
     attr_reader :seconds_since_startup
     attr_reader :score
-    attr_reader :git_sha
-    attr_reader :truncated_metrics
 
-    def initialize(queue_name, job_name, time, total_time, exclusive_time, context, metrics, allocation_metrics, mem_delta, allocations, score, truncated_metrics)
+    def initialize(queue_name, job_name, time, total_time, exclusive_time, context, metrics, allocation_metrics, mem_delta, allocations, score)
       @queue_name = queue_name
       @job_name = job_name
       @time = time
@@ -36,10 +34,7 @@ module ScoutApm
       @allocations = allocations
       @seconds_since_startup = (Time.now - ScoutApm::Agent.instance.process_start_time)
       @hostname = ScoutApm::Environment.instance.hostname
-      @git_sha = ScoutApm::Environment.instance.git_revision.sha
       @score = score
-      @truncated_metrics = truncated_metrics
-
       ScoutApm::Agent.instance.logger.debug { "Slow Job [#{metric_name}] - Call Time: #{total_call_time} Mem Delta: #{mem_delta}"}
     end
 

data/lib/scout_apm/slow_transaction.rb

@@ -15,11 +15,8 @@ module ScoutApm
     attr_reader :allocations
     attr_accessor :hostname # hack - we need to reset these server side.
     attr_accessor :seconds_since_startup # hack - we need to reset these server side.
-    attr_accessor :git_sha # hack - we need to reset these server side.
 
-    attr_reader :truncated_metrics # True/False that says if we had to truncate the metrics of this trace
-
-    def initialize(uri, metric_name, total_call_time, metrics, allocation_metrics, context, time, raw_stackprof, mem_delta, allocations, score, truncated_metrics)
+    def initialize(uri, metric_name, total_call_time, metrics, allocation_metrics, context, time, raw_stackprof, mem_delta, allocations, score)
       @uri = uri
       @metric_name = metric_name
       @total_call_time = total_call_time
@@ -33,9 +30,6 @@ module ScoutApm
       @seconds_since_startup = (Time.now - ScoutApm::Agent.instance.process_start_time)
       @hostname = ScoutApm::Environment.instance.hostname
       @score = score
-      @git_sha = ScoutApm::Environment.instance.git_revision.sha
-      @truncated_metrics = truncated_metrics
-
       ScoutApm::Agent.instance.logger.debug { "Slow Request [#{uri}] - Call Time: #{total_call_time} Mem Delta: #{mem_delta} Score: #{score}"}
     end
 
@@ -50,19 +44,7 @@ module ScoutApm
     end
 
     def as_json
-      json_attributes = [:key,
-                         :time,
-                         :total_call_time,
-                         :uri,
-                         [:context, :context_hash],
-                         :score,
-                         :prof,
-                         :mem_delta,
-                         :allocations,
-                         :seconds_since_startup,
-                         :hostname,
-                         :git_sha,
-                         :truncated_metrics]
+      json_attributes = [:key, :time, :total_call_time, :uri, [:context, :context_hash], :score, :prof, :mem_delta, :allocations, :seconds_since_startup, :hostname]
       ScoutApm::AttributeArranger.call(self, json_attributes)
     end
 

data/lib/scout_apm/store.rb

@@ -26,23 +26,7 @@ module ScoutApm
     # Save newly collected metrics
     def track!(metrics, options={})
       @mutex.synchronize {
-        period = if options[:timestamp]
-                   @reporting_periods[options[:timestamp]]
-                 else
-                   current_period
-                 end
-        period.absorb_metrics!(metrics)
-      }
-    end
-
-    def track_histograms!(histograms, options={})
-      @mutex.synchronize {
-        period = if options[:timestamp]
-                   @reporting_periods[options[:timestamp]]
-                 else
-                   current_period
-                 end
-        period.merge_histograms!(histograms)
+        current_period.absorb_metrics!(metrics)
       }
     end
 
@@ -83,21 +67,14 @@ module ScoutApm
     def write_to_layaway(layaway, force=false)
       ScoutApm::Agent.instance.logger.debug("Writing to layaway#{" (Forced)" if force}")
 
-        reporting_periods.select { |time, rp| force || (time.timestamp < current_timestamp.timestamp) }.
-                          each   { |time, rp| collect_samplers(rp) }.
-                          each   { |time, rp| write_reporting_period(layaway, time, rp) }
-    end
-
-    def write_reporting_period(layaway, time, rp)
       @mutex.synchronize {
-        layaway.write_reporting_period(rp)
+        reporting_periods.select { |time, rp| force || time.timestamp < current_timestamp.timestamp}.
+                          each   { |time, rp|
+                                   collect_samplers(rp)
+                                   layaway.write_reporting_period(rp)
+                                   reporting_periods.delete(time)
+                                 }
       }
-    rescue => e
-      ScoutApm::Agent.instance.logger.warn("Failed writing data to layaway file: #{e.message} / #{e.backtrace}")
-    ensure
-      ScoutApm::Agent.instance.logger.debug("Before delete, reporting periods length: #{reporting_periods.size}")
-      deleted_items = reporting_periods.delete(time)
-      ScoutApm::Agent.instance.logger.debug("After delete, reporting periods length: #{reporting_periods.size}. Did delete #{deleted_items}")
     end
 
     ######################################
@@ -109,10 +86,12 @@ module ScoutApm
     def collect_samplers(rp)
       @samplers.each do |sampler|
         begin
-          sampler.metrics(rp.timestamp, self)
+          metrics = sampler.metrics(rp.timestamp)
+          rp.absorb_metrics!(metrics)
         rescue => e
           ScoutApm::Agent.instance.logger.info "Error reading #{sampler.human_name} for period: #{rp}"
-          ScoutApm::Agent.instance.logger.debug "#{e.message}\n\t#{e.backtrace.join("\n\t")}"
+          ScoutApm::Agent.instance.logger.debug e.message
+          ScoutApm::Agent.instance.logger.debug e.backtrace.join("\n")
         end
       end
     end
@@ -175,9 +154,6 @@ module ScoutApm
     # A ScoredItemSet holding the "best" traces for the period
     attr_reader :job_traces
 
-    # An Array of HistogramsReport
-    attr_reader :histograms
-
     # A StoreReportingPeriodTimestamp representing the time that this
     # collection of metrics is for
     attr_reader :timestamp
@@ -190,8 +166,6 @@ module ScoutApm
       @request_traces = ScoredItemSet.new
       @job_traces = ScoredItemSet.new
 
-      @histograms = []
-
       @metric_set = MetricSet.new
       @jobs = Hash.new
     end
@@ -202,8 +176,7 @@ module ScoutApm
         merge_metrics!(other.metric_set).
         merge_slow_transactions!(other.slow_transactions_payload).
         merge_jobs!(other.jobs).
-        merge_slow_jobs!(other.slow_jobs_payload).
-        merge_histograms!(other.histograms)
+        merge_slow_jobs!(other.slow_jobs_payload)
       self
     end
 
@@ -252,17 +225,6 @@ module ScoutApm
       self
     end
 
-    def merge_histograms!(new_histograms)
-      new_histograms = Array(new_histograms)
-      @histograms = (histograms + new_histograms).
-        group_by { |histo| histo.name }.
-        map { |(_, histos)|
-          histos.inject { |merged, histo| merged.combine!(histo) }
-        }
-
-      self
-    end
-
     #################################
     # Retrieve Metrics for reporting
     #################################

data/lib/scout_apm/trace_compactor.rb

@@ -0,0 +1,311 @@
+# Takes in a ton of traces. Structure is a several nested arrays:
+# [                             # Traces
+#    [                          # Trace
+#      [file,line,method,klass] # TraceLine (raw)
+#    ]
+# ]
+#
+# Cleans them
+# Merges them by gem/app
+#
+module ScoutApm
+class TraceSet
+  # A TraceCube object which is a glorified hash of { Trace -> Count }. Used to
+  # collect up the count of each unique trace we've seen
+  attr_reader :cube
+
+  # Allow layer to push values in
+  attr_accessor :raw_traces
+  attr_accessor :skipped_in_gc
+  attr_accessor :skipped_in_handler
+  attr_accessor :skipped_in_job_registered
+
+  def initialize
+    @raw_traces = []
+    @cube = TraceCube.new
+  end
+
+  # We need to know what the "Start" of this trace is.  An untrimmed trace generally is:
+  #
+  # Gem
+  # Gem
+  # App
+  # App
+  # App <---- set root_class of this.
+  # Rails
+  # Rails
+  def set_root_class(klass_name)
+    @root_klass = klass_name.to_s
+  end
+
+  def to_a
+    res = []
+    create_cube!
+    @cube.each do |(trace, count)|
+      res << [trace.to_a, count]
+    end
+
+    res
+  end
+
+  def as_json
+    res = []
+    create_cube!
+    @cube.each do |(trace, count)|
+      res << [trace.as_json, count]
+    end
+
+    res
+  end
+
+  def create_cube!
+    while raw_trace = @raw_traces.shift
+      clean_trace = ScoutApm::CleanTrace.new(raw_trace, @root_klass)
+      @cube << clean_trace
+    end
+    @raw_traces = []
+  end
+
+  def total_count
+    create_cube!
+    cube.inject(0) do |sum, (_, count)|
+      sum + count
+    end
+  end
+
+  def inspect
+    create_cube!
+    cube.map do |(trace, count)|
+      "\t#{count} -- #{trace.first.klass}##{trace.first.method}\n\t\t#{trace.to_a[1].try(:klass)}##{trace.to_a[1].try(:method)}"
+    end.join("\n")
+  end
+end
+
+# A trace is a list of individual lines, where one called another, forming a backtrace.
+# Each line is made up of File, Line #, Klass, Method
+#
+# For the purpouses of this class:
+#   "Top" of the trace means the currently-running method.
+#   "Bottom" means the root of the call tree, from program start into rails and so on.
+#
+# This class trims off top and bottom to get a the meat of the trace
+class CleanTrace
+  include Enumerable
+
+  attr_reader :lines
+
+  def initialize(raw_trace, root_klass=nil)
+    @lines = Array(raw_trace).map {|frame, lineno| TraceLine.new(frame, lineno)}
+    @root_klass = root_klass
+
+    # A trace has interesting data in the middle of it, since normally it'll go
+    # RailsCode -> App Code -> Gem Code.
+    #
+    # So we drop the code that leads up to your app, since a deep trace that
+    # always says that you went through middleware and the rails router doesn't
+    # help diagnose issues.
+    drop_below_app
+
+    # Then we drop most of the Gem Code, since you didn't write it, and in the
+    # vast majority of the cases, the time spent there is because your app code
+    # asked, not because of inherent issues with the gem. For instance, if you
+    # fire off a slow query to a database gem, you probably want to be
+    # optimizing the query, not trying to make the database gem faster.
+    drop_above_app
+  end
+
+  # Iterate starting at END of array until a controller line is found. Pop off at that index - 1.
+  def drop_below_app
+    pops = 0
+    index = lines.size - 1 # last index, not size.
+
+    while index >= 0 && !lines[index].controller?(@root_klass)
+      index -= 1
+      pops += 1
+    end
+
+    lines.pop(pops)
+  end
+
+  # Find the closest mention of the application code from the currently-running method.
+  # Then adjust by 1 if possible to capture the "first" line 
+  def drop_above_app
+    ai = @lines.find_index(&:app?)
+    if ai
+      ai -= 1 if ai > 0
+      @lines = @lines[ai .. -1]
+    else
+      @lines = [] # No app line in backtrace, nothing to show?
+    end
+  end
+
+  def each
+    @lines.each { |line| yield line }
+  end
+
+  def empty?
+    @lines.empty?
+  end
+
+  def as_json
+    @lines.map { |line| line.as_json }
+  end
+
+  ###############################
+  # Hash Key interface
+  def hash
+    @lines.hash
+  end
+
+  def eql?(other)
+    @lines.eql?(other.lines)
+  end
+  ###############################
+end
+
+class TraceLine
+  # An opaque C object, only call Stacks#frame_* methods on this.
+  attr_reader :frame
+
+  # The line number. This doesn't appear to be obtainable from the frame itself
+  attr_reader :lineno
+
+  def initialize(frame, lineno)
+    @frame = frame
+    @lineno = lineno
+  end
+
+  # Returns the name of the last gem in the line
+  def gem_name
+    @gem_name ||= begin
+                    r = %r{\/gems/(.*?)/}.freeze
+                    results = file.scan(r)
+                    results[-1][0] # Scan will return a nested array, so extract out that nesting
+                  rescue
+                    nil
+                  end
+  end
+
+  def stdlib_name
+    @stdlib_name ||= begin
+                    r = %r{#{Regexp.escape(RbConfig::TOPDIR)}/(.*?)}.freeze
+                    results = file.scan(r)
+                    results[-1][0] # Scan will return a nested array, so extract out that nesting
+                  rescue
+                    nil
+                  end
+  end
+
+  def file
+    ScoutApm::Instruments::Stacks.frame_file(frame)
+  end
+
+
+  # If we ever want to get the "first line of the method" - ScoutApm::Instruments::Stacks.frame_lineno(frame)
+  def line
+    lineno
+  end
+
+  def klass
+    ScoutApm::Instruments::Stacks.frame_klass(frame)
+  end
+
+  def method
+    ScoutApm::Instruments::Stacks.frame_method(frame)
+  end
+
+  def gem?
+    !!gem_name
+  end
+
+  def stdlib?
+    !!stdlib_name
+  end
+
+  def app?
+    r = %r|^#{Regexp.escape(ScoutApm::Environment.instance.root.to_s)}/|.freeze
+    result = !gem_name && !stdlib_name && file =~ r
+    !!result # coerce to a bool
+  end
+
+  def trim_file(file_path)
+    return if file_path.nil?
+    if gem?
+      r = %r{.*gems/.*?/}.freeze
+      file_path.sub(r, "/")
+    elsif stdlib?
+      file_path.sub(RbConfig::TOPDIR, '')
+    elsif app?
+      file_path.sub(ScoutApm::Environment.instance.root.to_s, '')
+    end
+  end
+
+  # If root_klass is provided, just see if this is exactly that class. If not,
+  # fall back on "is this in the app"
+  def controller?(root_klass)
+    return false if klass.nil? # main function doesn't have a file associated
+
+    if root_klass
+      klass == root_klass
+    else
+      app?
+    end
+  end
+
+  def formatted_to_s
+    "#{stdlib_name} #{klass}##{method} -- #{file}:#{line}"
+  end
+
+  def as_json
+    [ trim_file(file), line, klass, method, app?, gem_name, stdlib_name ]
+  end
+
+  ###############################
+  # Hash Key interface
+
+  def hash
+    # Note that this does not include line number. It caused a few situations
+    # where we had a bunch of time spent in one method, but across a few lines,
+    # we decided that it made sense to group them together.
+    file.hash ^ method.hash ^ klass.hash
+  end
+
+  def eql?(other)
+    file == other.file &&
+      method == other.method &&
+      klass == other.klass
+  end
+
+  ###############################
+end
+
+# Collects clean traces and counts how many of each we have.
+class TraceCube
+  include Enumerable
+
+  attr_reader :traces
+  attr_reader :total_count
+
+  def initialize
+    @traces = Hash.new{ |h,k| h[k] = 0 }
+    @total_count = 0
+  end
+
+  def <<(clean_trace)
+    @total_count += 1
+    @traces[clean_trace] += 1
+  end
+
+  # Yields two element array, the trace and the count of that trace
+  # In descending order of count.
+  def each
+    @traces
+      .to_a
+      .each { |(trace, count)|
+        next if trace.empty?
+        yield [trace, count]
+      }
+  end
+end
+end
+