scout_apm 3.0.0.pre25 → 4.0.1

data/lib/scout_apm/layer_converters/converter_base.rb

@@ -174,14 +174,13 @@ module ScoutApm
         metric_hash[meta] ||= MetricStats.new(meta_options.has_key?(:scope))
         allocation_metric_hash[meta] ||= MetricStats.new(meta_options.has_key?(:scope))
 
-        # Timing
-        timing_stat = metric_hash[meta]
-        timing_stat.update!(layer.total_call_time, layer.total_exclusive_time)
-        timing_stat.add_traces(layer.traces.as_json)
+        # timing
+        stat = metric_hash[meta]
+        stat.update!(layer.total_call_time, layer.total_exclusive_time)
 
-        # Allocations
-        allocation_stat = allocation_metric_hash[meta]
-        allocation_stat.update!(layer.total_allocations, layer.total_exclusive_allocations)
+        # allocations
+        stat = allocation_metric_hash[meta]
+        stat.update!(layer.total_allocations, layer.total_exclusive_allocations)
 
         if LimitedLayer === layer
           metric_hash[meta].call_count = layer.count
@@ -191,18 +190,18 @@ module ScoutApm
 
       # Merged Metric - no specifics, just sum up by type (ActiveRecord, View, HTTP, etc)
       def store_aggregate_metric(layer, metric_hash, allocation_metric_hash)
-        meta = MetricMeta.new("#{layer.type}/all")
+          meta = MetricMeta.new("#{layer.type}/all")
 
-        metric_hash[meta] ||= MetricStats.new(false)
-        allocation_metric_hash[meta] ||= MetricStats.new(false)
+          metric_hash[meta] ||= MetricStats.new(false)
+          allocation_metric_hash[meta] ||= MetricStats.new(false)
 
-        # timing
-        timing_stat = metric_hash[meta]
-        timing_stat.update!(layer.total_call_time, layer.total_exclusive_time)
+          # timing
+          stat = metric_hash[meta]
+          stat.update!(layer.total_call_time, layer.total_exclusive_time)
 
-        # allocations
-        allocation_stat = allocation_metric_hash[meta]
-        allocation_stat.update!(layer.total_allocations, layer.total_exclusive_allocations)
+          # allocations
+          stat = allocation_metric_hash[meta]
+          stat.update!(layer.total_allocations, layer.total_exclusive_allocations)
       end
 
       ################################################################################
@@ -217,20 +216,6 @@ module ScoutApm
         return false if layer.annotations.nil?
         return true  if layer.annotations[:ignorable]
       end
-
-      # Debug logging for scoutprof traces
-      def debug_scoutprof(layer)
-        if layer.type =~ %r{^(Controller|Queue|Job)$}.freeze
-          if context.config.value('profile')
-            context.logger.debug do
-              traces_inspect = layer.traces.inspect
-              "****** Slow Request #{layer.type} Traces (#{layer.name}, tet: #{layer.total_exclusive_time}, tct: #{layer.total_call_time}), total raw traces: #{layer.traces.cube.total_count}, total clean traces: #{layer.traces.total_count}, skipped gc: #{layer.traces.skipped_in_gc}, skipped handler: #{layer.traces.skipped_in_handler}, skipped registered #{layer.traces.skipped_in_job_registered}, skipped not_running #{layer.traces.skipped_in_not_running}:\n#{traces_inspect}"
-            end
-          else
-            context.logger.debug "****** Slow Request #{layer.type} Traces: Scoutprof is not enabled"
-          end
-        end
-      end
     end
   end
 end

data/lib/scout_apm/layer_converters/database_converter.rb

@@ -13,7 +13,6 @@ module ScoutApm
 
         walker.on do |layer|
           next if skip_layer?(layer)
-
           stat = DbQueryMetricStats.new(
             model_name(layer),
             operation_name(layer),
@@ -52,23 +51,11 @@ module ScoutApm
       DEFAULT_OPERATION = "other"
 
       def model_name(layer)
-        if layer.name.respond_to?(:model)
-          layer.name.model || DEFAULT_MODEL
-        else
-          DEFAULT_MODEL
-        end
-      rescue
-        DEFAULT_MODEL
+        layer.name.to_s.split("/").first || DEFAULT_MODEL
       end
 
       def operation_name(layer)
-        if layer.name.respond_to?(:normalized_operation)
-          layer.name.normalized_operation || DEFAULT_OPERATION
-        else
-          DEFAULT_OPERATION
-        end
-      rescue
-        DEFAULT_OPERATION
+        layer.name.to_s.split("/")[1] || DEFAULT_OPERATION
       end
 
       def records_returned(layer)

data/lib/scout_apm/layer_converters/slow_job_converter.rb

@@ -15,6 +15,7 @@ module ScoutApm
         # Let the store know we're here, and if it wants our data, it will call
         # back into #call
         @store.track_slow_job!(self)
+
         nil # not returning anything in the layer results ... not used
       end
 
@@ -54,7 +55,8 @@ module ScoutApm
           mem_delta,
           job_layer.total_allocations,
           score,
-          limited?
+          limited?,
+          span_trace
         )
       end
 
@@ -69,7 +71,9 @@ module ScoutApm
         walker.on do |layer|
           next if skip_layer?(layer)
 
-          debug_scoutprof(layer)
+          # The queue_layer is useful to capture for other reasons, but doesn't
+          # create a MetricMeta/Stat of its own
+          next if layer == queue_layer
 
           store_specific_metric(layer, metric_hash, allocation_metric_hash)
           store_aggregate_metric(layer, metric_hash, allocation_metric_hash)
@@ -87,6 +91,12 @@ module ScoutApm
       def skip_layer?(layer); super(layer) || layer == queue_layer; end
       def queue_layer; layer_finder.queue; end
       def job_layer; layer_finder.job; end
+
+      def span_trace
+        ScoutApm::LayerConverters::TraceConverter.
+          new(@context, @request, @layer_finder, @store).
+          call
+      end
     end
   end
 end

data/lib/scout_apm/layer_converters/slow_request_converter.rb

@@ -11,6 +11,7 @@ module ScoutApm
         # Let the store know we're here, and if it wants our data, it will call
         # back into #call
         @store.track_slow_transaction!(self)
+
         nil # not returning anything in the layer results ... not used
       end
 
@@ -51,7 +52,8 @@ module ScoutApm
                             mem_delta,
                             root_layer.total_allocations,
                             @points,
-                            limited?)
+                            limited?,
+                            span_trace)
       end
 
       # Full metrics from this request. These get stored permanently in a SlowTransaction.
@@ -69,9 +71,6 @@ module ScoutApm
 
         walker.on do |layer|
           next if skip_layer?(layer)
-
-          debug_scoutprof(layer)
-
           store_specific_metric(layer, metric_hash, allocation_metric_hash)
           store_aggregate_metric(layer, metric_hash, allocation_metric_hash)
         end
@@ -84,6 +83,17 @@ module ScoutApm
 
         [metric_hash, allocation_metric_hash]
       end
+
+      ###########################################################
+      #  Also create a new style trace. This is not a good      #
+      #  spot for this long term, but fixes an issue for now.   #
+      ###########################################################
+      
+      def span_trace
+        ScoutApm::LayerConverters::TraceConverter.
+          new(@context, @request, @layer_finder, @store).
+          call
+      end
     end
   end
 end

data/lib/scout_apm/layer_converters/trace_converter.rb

@@ -0,0 +1,184 @@
+module ScoutApm
+  module LayerConverters
+    class TraceConverter < ConverterBase
+      ###################
+      #  Converter API  #
+      ###################
+
+
+      def record!
+        @points = context.slow_request_policy.score(request)
+
+        # Let the store know we're here, and if it wants our data, it will call
+        # back into #call
+        @store.track_trace!(self)
+
+        nil # not returning anything in the layer results ... not used
+      end
+
+      #####################
+      #  ScoreItemSet API #
+      #####################
+      def name; request.unique_name; end
+      def score; @points; end
+
+      # Unconditionally attempts to convert this into a DetailedTrace object.
+      # Can return nil if the request didn't have any scope_layer or if `timeline_traces` aren't enabled.
+      def call
+        return nil unless scope_layer
+        return nil unless context.config.value('timeline_traces')
+
+        # Since this request is being stored, update the needed counters
+        context.slow_request_policy.stored!(request)
+
+        # record the change in memory usage
+        mem_delta = ScoutApm::Instruments::Process::ProcessMemory.new(context).rss_to_mb(@request.capture_mem_delta!)
+
+        transaction_id = request.transaction_id
+        revision = context.environment.git_revision.sha
+        start_instant = request.root_layer.start_time
+        stop_instant = request.root_layer.stop_time
+        type = if request.web?
+                 "Web"
+               elsif request.job?
+                 "Job"
+               else
+                 "Unknown"
+               end
+
+        # Create request tags
+        #
+        tags = {
+          :allocations => request.root_layer.total_allocations,
+          :mem_delta => mem_delta,
+        }.merge(request.context.to_flat_hash)
+
+        host = context.environment.hostname
+        path = request.annotations[:uri] || ""
+        code = "" # User#index for instance
+
+        spans = create_spans(request.root_layer)
+        if limited?
+          tags[:"scout.reached_span_cap"] = true
+        end
+
+        DetailedTrace.new(
+          transaction_id,
+          revision,
+          host,
+          start_instant,
+          stop_instant,
+          type,
+
+          path,
+          code,
+
+          spans,
+          tags
+
+          # total_score = 0,
+          # percentile_score = 0,
+          # age_score = 0,
+          # memory_delta_score = 0,
+          # memory_allocations_score = 0
+        )
+      end
+
+      # Returns an array of span objects. Uses recursion to get all children
+      # wired up w/ correct parent_ids
+      def create_spans(layer, parent_id = nil)
+        span_id = ScoutApm::Utils::SpanId.new.to_s
+
+        start_instant = layer.start_time
+        stop_instant = layer.stop_time
+        operation = layer.legacy_metric_name
+        tags = {
+          :start_allocations => layer.allocations_start,
+          :stop_allocations => layer.allocations_stop,
+        }
+        if layer.desc
+          tags[:desc] = layer.desc.to_s
+        end
+        if layer.annotations && layer.annotations[:record_count]
+          tags["db.record_count"] = layer.annotations[:record_count]
+        end
+        if layer.annotations && layer.annotations[:class_name]
+          tags["db.class_name"] = layer.annotations[:class_name]
+        end
+        if layer.backtrace
+          tags[:backtrace] = backtrace_parser(layer.backtrace) rescue nil
+        end
+
+        # Collect up self, and all children into result array
+        result = []
+        result << DetailedTraceSpan.new(
+          span_id.to_s,
+          parent_id.to_s,
+          start_instant,
+          stop_instant,
+          operation,
+          tags)
+
+        layer.children.each do |child|
+          # Don't create spans from limited layers. These don't have start/stop times and our processing can't
+          # handle these yet.
+          unless over_span_limit?(result) || child.is_a?(LimitedLayer)
+            result += create_spans(child, span_id)
+          end
+        end
+
+        return result
+      end
+
+      # Take an array of ruby backtrace lines and split it into an array of hashes like:
+      # ["/Users/cschneid/.rvm/rubies/ruby-2.2.7/lib/ruby/2.2.0/irb/workspace.rb:86:in `eval'", ...]
+      #    turns into:
+      # [ {
+      #     "file": "app/controllers/users_controller.rb",
+      #     "line": 10,
+      #     "function": "index"
+      # },
+      # ]
+      def backtrace_parser(lines)
+        bt = ScoutApm::Utils::BacktraceParser.new(lines).call
+
+        bt.map do |line|
+          match = line.match(/(.*):(\d+):in `(.*)'/)
+          {
+            "file" => match[1],
+            "line" => match[2],
+            "function" => match[3],
+          }
+        end
+
+      end
+
+      ################################################################################
+      # Limit Handling
+      ################################################################################
+
+      # To prevent huge traces from being generated, we stop collecting
+      # spans as we go beyond some reasonably large count.
+      MAX_SPANS = 1500
+
+      def over_span_limit?(spans)
+        if spans.size > MAX_SPANS
+          log_over_span_limit
+          @limited = true
+        else
+          false
+        end
+      end
+
+      def log_over_span_limit
+        unless limited?
+          context.logger.debug "Not recording additional spans for #{name}. Over the span limit."
+        end
+      end
+
+      def limited?
+        !! @limited
+      end
+    end
+  end
+end

data/lib/scout_apm/limited_layer.rb

@@ -85,13 +85,6 @@ module ScoutApm
       nil
     end
 
-    def add_traces(*)
-      # noop
-    end
-
-    def traces
-      ScoutApm::TraceSet.new
-    end
 
     #######################################################################
     #  Many methods don't make any sense on a limited layer. Raise errors  #

data/lib/scout_apm/metric_stats.rb

@@ -9,7 +9,6 @@ class MetricStats
   attr_accessor :sum_of_squares
   attr_accessor :queue
   attr_accessor :latency
-  attr_accessor :traces
 
   def initialize(scoped = false)
     @scoped = scoped
@@ -19,7 +18,6 @@ class MetricStats
     self.min_call_time = 0.0
     self.max_call_time = 0.0
     self.sum_of_squares = 0.0
-    self.traces = []
   end
 
   # Note, that you must include exclusive_time if you wish to set
@@ -41,10 +39,6 @@ class MetricStats
     self
   end
 
-  def add_traces(traces)
-    self.traces += Array(traces)
-  end
-
   # combines data from another MetricStats object
   def combine!(other)
     self.call_count += other.call_count
@@ -53,7 +47,6 @@ class MetricStats
     self.min_call_time = other.min_call_time if self.min_call_time.zero? or other.min_call_time < self.min_call_time
     self.max_call_time = other.max_call_time if other.max_call_time > self.max_call_time
     self.sum_of_squares += other.sum_of_squares
-    self.traces = Array(self.traces) + Array(other.traces)
     self
   end
 
@@ -64,7 +57,6 @@ class MetricStats
       :min_call_time,
       :total_call_time,
       :total_exclusive_time,
-      :traces,
     ]
     ScoutApm::AttributeArranger.call(self, json_attributes)
   end

data/lib/scout_apm/middleware.rb

@@ -26,7 +26,7 @@ module ScoutApm
       ScoutApm::Agent.instance.start
       @started = ScoutApm::Agent.instance.context.started? && ScoutApm::Agent.instance.background_worker_running?
     rescue => e
-      ScoutApm::Agent.instance.context.logger("Failed to start via Middleware: #{e.message}\n\t#{e.backtrace.join("\n\t")}")
+      ScoutApm::Agent.instance.context.logger.info("Failed to start via Middleware: #{e.message}\n\t#{e.backtrace.join("\n\t")}")
     end
   end
 end

data/lib/scout_apm/periodic_work.rb

@@ -12,10 +12,29 @@ module ScoutApm
       ScoutApm::Debug.instance.call_periodic_hooks
       @reporting.process_metrics
       clean_old_percentiles
+
+      if context.config.value('auto_instruments')
+        log_autoinstrument_significant_counts rescue nil
+      end
     end
 
     private
 
+    def log_autoinstrument_significant_counts
+      # Ex key/value -
+      # "/Users/dlite/projects/scout/apm/app/controllers/application_controller.rb"=>[[0.0, 689], [1.0, 16]]
+      hists = context.auto_instruments_layer_histograms.as_json
+      hists_summary = hists.map { |file, buckets|
+        total = buckets.map(&:last).inject(0) { |sum, count| sum + count }
+        significant = (buckets.last.last / total.to_f).round(2)
+        [
+          file,
+          {:total => total, :significant => significant}
+        ]
+      }.to_h
+      context.logger.debug("AutoInstrument Significant Layer Histograms: #{hists_summary.pretty_inspect}")
+    end
+
     # XXX: Move logic into a RequestHistogramsByTime class that can keep the timeout logic in it
     def clean_old_percentiles
       context.