scout_apm 3.0.0.pre28 → 4.0.0

data/lib/scout_apm/job_record.rb

@@ -32,8 +32,10 @@ module ScoutApm
 
     # Modifies self and returns self, after merging in `other`.
     def combine!(other)
-      same_job = queue_name == other.queue_name && job_name == other.job_name
-      raise "Mismatched Merge of Background Job" unless same_job
+      if !self.eql?(other)
+        ScoutApm::Agent.instance.logger.debug("Mismatched Merge of Background Job: (Queue #{queue_name} == #{other.queue_name}) (Name #{job_name} == #{other.job_name}) (Hash #{hash} == #{other.hash})")
+        return self
+      end
 
       @errors += other.errors
       @metric_set = metric_set.combine!(other.metric_set)

data/lib/scout_apm/layaway_file.rb

@@ -30,6 +30,10 @@ module ScoutApm
 
     def serialize(data)
       Marshal.dump(data)
+    rescue
+      ScoutApm::Agent.instance.logger.info("Failed Marshalling LayawayFile")
+      ScoutApm::Agent.instance.logger.info(ScoutApm::Utils::MarshalLogging.new(data).dive) rescue nil
+      raise
     end
 
     def deserialize(data)

data/lib/scout_apm/layer.rb

@@ -36,7 +36,10 @@ module ScoutApm
 
     # If this layer took longer than a fixed amount of time, store the
     # backtrace of where it occurred.
-    attr_reader :backtrace
+    attr_accessor :backtrace
+
+    # The file name associated with the layer. Only used for autoinstruments overhead logging.
+    attr_accessor :file_name
 
     # As we go through a part of a request, instrumentation can store additional data
     # Known Keys:
@@ -46,19 +49,7 @@ module ScoutApm
     # If no annotations are ever set, this will return nil
     attr_reader :annotations
 
-    # ScoutProf - trace_index is an index into the Stack structure in the C
-    # code, used to store captured traces.
-    attr_reader :trace_index
-
-    # ScoutProf - frame_index is an optimization to not capture a few frames
-    # during scoutprof instrumentation
-    attr_reader :frame_index
-
-    # Captured backtraces from ScoutProf. This is distinct from the backtrace
-    # attribute, which gets the ruby backtrace of any given layer. StackProf
-    # focuses on Controller layers, and requires a native extension and a
-    # reasonably recent Ruby.
-    attr_reader :traces
+    attr_reader :allocations_start, :allocations_stop
 
     BACKTRACE_CALLER_LIMIT = 50 # maximum number of lines to send thru for backtrace analysis
 
@@ -73,11 +64,6 @@ module ScoutApm
       @children = nil
       @annotations = nil
       @desc = nil
-
-      @traces = ScoutApm::TraceSet.new
-      @raw_frames = []
-      @frame_index = ScoutApm::Instruments::Stacks.current_frame_index # For efficiency sake, try to skip the bottom X frames when collecting traces
-      @trace_index = ScoutApm::Instruments::Stacks.current_trace_index
     end
 
     def limited?
@@ -116,14 +102,6 @@ module ScoutApm
       @subscopable
     end
 
-    def traced!
-      @traced = true
-    end
-
-    def traced?
-      @traced
-    end
-
     # This is the old style name. This function is used for now, but should be
     # removed, and the new type & name split should be enforced through the
     # app.
@@ -145,35 +123,6 @@ module ScoutApm
       end
     end
 
-    # Set the name of the file that this action is coming from.
-    # TraceSet uses this to more accurately filter backtraces
-    def set_root_class(klass_name)
-      @traces.set_root_class(klass_name)
-    end
-
-    def start_sampling
-      if ScoutApm::Agent.instance.context.config.value('profile') && traced?
-        ScoutApm::Instruments::Stacks.update_indexes(frame_index, trace_index)
-        ScoutApm::Instruments::Stacks.start_sampling
-      else
-        ScoutApm::Instruments::Stacks.stop_sampling(false)
-      end
-    end
-
-    def record_traces!
-      if ScoutApm::Agent.instance.context.config.value('profile')
-        ScoutApm::Instruments::Stacks.stop_sampling(false)
-        if traced?
-          traces.raw_traces = ScoutApm::Instruments::Stacks.profile_frames
-          traces.skipped_in_gc = ScoutApm::Instruments::Stacks.skipped_in_gc
-          traces.skipped_in_handler = ScoutApm::Instruments::Stacks.skipped_in_handler
-          traces.skipped_in_job_registered = ScoutApm::Instruments::Stacks.skipped_in_job_registered
-          traces.skipped_in_not_running = ScoutApm::Instruments::Stacks.skipped_in_not_running
-        end
-      end
-    end
-
-
     ######################################
     # Debugging Helpers
     ######################################

data/lib/scout_apm/layer_children_set.rb

@@ -46,9 +46,15 @@ module ScoutApm
       set = child_set(metric_type)
 
       if set.size >= unique_cutoff
-        # find limited_layer
-        @limited_layers || init_limited_layers
-        @limited_layers[metric_type].absorb(child)
+        # find or create limited_layer
+        @limited_layers ||= Hash.new 
+        layer = if @limited_layers.has_key?(metric_type)
+                  @limited_layers[metric_type]
+                else
+                  @limited_layers[metric_type] = LimitedLayer.new(metric_type)
+                end
+
+        layer.absorb(child)
       else
         # we have space just add it
         set << child
@@ -76,10 +82,5 @@ module ScoutApm
     def size
       @children.size
     end
-
-    # hold off initializing this until we know we need it
-    def init_limited_layers
-      @limited_layers ||= Hash.new { |hash, key| hash[key] = LimitedLayer.new(key) }
-    end
   end
 end

data/lib/scout_apm/layer_converters/converter_base.rb

@@ -174,14 +174,13 @@ module ScoutApm
         metric_hash[meta] ||= MetricStats.new(meta_options.has_key?(:scope))
         allocation_metric_hash[meta] ||= MetricStats.new(meta_options.has_key?(:scope))
 
-        # Timing
-        timing_stat = metric_hash[meta]
-        timing_stat.update!(layer.total_call_time, layer.total_exclusive_time)
-        timing_stat.add_traces(layer.traces.as_json)
+        # timing
+        stat = metric_hash[meta]
+        stat.update!(layer.total_call_time, layer.total_exclusive_time)
 
-        # Allocations
-        allocation_stat = allocation_metric_hash[meta]
-        allocation_stat.update!(layer.total_allocations, layer.total_exclusive_allocations)
+        # allocations
+        stat = allocation_metric_hash[meta]
+        stat.update!(layer.total_allocations, layer.total_exclusive_allocations)
 
         if LimitedLayer === layer
           metric_hash[meta].call_count = layer.count
@@ -191,18 +190,18 @@ module ScoutApm
 
       # Merged Metric - no specifics, just sum up by type (ActiveRecord, View, HTTP, etc)
       def store_aggregate_metric(layer, metric_hash, allocation_metric_hash)
-        meta = MetricMeta.new("#{layer.type}/all")
+          meta = MetricMeta.new("#{layer.type}/all")
 
-        metric_hash[meta] ||= MetricStats.new(false)
-        allocation_metric_hash[meta] ||= MetricStats.new(false)
+          metric_hash[meta] ||= MetricStats.new(false)
+          allocation_metric_hash[meta] ||= MetricStats.new(false)
 
-        # timing
-        timing_stat = metric_hash[meta]
-        timing_stat.update!(layer.total_call_time, layer.total_exclusive_time)
+          # timing
+          stat = metric_hash[meta]
+          stat.update!(layer.total_call_time, layer.total_exclusive_time)
 
-        # allocations
-        allocation_stat = allocation_metric_hash[meta]
-        allocation_stat.update!(layer.total_allocations, layer.total_exclusive_allocations)
+          # allocations
+          stat = allocation_metric_hash[meta]
+          stat.update!(layer.total_allocations, layer.total_exclusive_allocations)
       end
 
       ################################################################################
@@ -217,20 +216,6 @@ module ScoutApm
         return false if layer.annotations.nil?
         return true  if layer.annotations[:ignorable]
       end
-
-      # Debug logging for scoutprof traces
-      def debug_scoutprof(layer)
-        if layer.type =~ %r{^(Controller|Queue|Job)$}.freeze
-          if context.config.value('profile')
-            context.logger.debug do
-              traces_inspect = layer.traces.inspect
-              "****** Slow Request #{layer.type} Traces (#{layer.name}, tet: #{layer.total_exclusive_time}, tct: #{layer.total_call_time}), total raw traces: #{layer.traces.cube.total_count}, total clean traces: #{layer.traces.total_count}, skipped gc: #{layer.traces.skipped_in_gc}, skipped handler: #{layer.traces.skipped_in_handler}, skipped registered #{layer.traces.skipped_in_job_registered}, skipped not_running #{layer.traces.skipped_in_not_running}:\n#{traces_inspect}"
-            end
-          else
-            context.logger.debug "****** Slow Request #{layer.type} Traces: Scoutprof is not enabled"
-          end
-        end
-      end
     end
   end
 end

data/lib/scout_apm/layer_converters/slow_job_converter.rb

@@ -15,6 +15,7 @@ module ScoutApm
         # Let the store know we're here, and if it wants our data, it will call
         # back into #call
         @store.track_slow_job!(self)
+
         nil # not returning anything in the layer results ... not used
       end
 
@@ -54,7 +55,8 @@ module ScoutApm
           mem_delta,
           job_layer.total_allocations,
           score,
-          limited?
+          limited?,
+          span_trace
         )
       end
 
@@ -69,7 +71,9 @@ module ScoutApm
         walker.on do |layer|
           next if skip_layer?(layer)
 
-          debug_scoutprof(layer)
+          # The queue_layer is useful to capture for other reasons, but doesn't
+          # create a MetricMeta/Stat of its own
+          next if layer == queue_layer
 
           store_specific_metric(layer, metric_hash, allocation_metric_hash)
           store_aggregate_metric(layer, metric_hash, allocation_metric_hash)
@@ -87,6 +91,12 @@ module ScoutApm
       def skip_layer?(layer); super(layer) || layer == queue_layer; end
       def queue_layer; layer_finder.queue; end
       def job_layer; layer_finder.job; end
+
+      def span_trace
+        ScoutApm::LayerConverters::TraceConverter.
+          new(@context, @request, @layer_finder, @store).
+          call
+      end
     end
   end
 end

data/lib/scout_apm/layer_converters/slow_request_converter.rb

@@ -11,6 +11,7 @@ module ScoutApm
         # Let the store know we're here, and if it wants our data, it will call
         # back into #call
         @store.track_slow_transaction!(self)
+
         nil # not returning anything in the layer results ... not used
       end
 
@@ -51,7 +52,8 @@ module ScoutApm
                             mem_delta,
                             root_layer.total_allocations,
                             @points,
-                            limited?)
+                            limited?,
+                            span_trace)
       end
 
       # Full metrics from this request. These get stored permanently in a SlowTransaction.
@@ -69,9 +71,6 @@ module ScoutApm
 
         walker.on do |layer|
           next if skip_layer?(layer)
-
-          debug_scoutprof(layer)
-
           store_specific_metric(layer, metric_hash, allocation_metric_hash)
           store_aggregate_metric(layer, metric_hash, allocation_metric_hash)
         end
@@ -84,6 +83,17 @@ module ScoutApm
 
         [metric_hash, allocation_metric_hash]
       end
+
+      ###########################################################
+      #  Also create a new style trace. This is not a good      #
+      #  spot for this long term, but fixes an issue for now.   #
+      ###########################################################
+      
+      def span_trace
+        ScoutApm::LayerConverters::TraceConverter.
+          new(@context, @request, @layer_finder, @store).
+          call
+      end
     end
   end
 end

data/lib/scout_apm/layer_converters/trace_converter.rb

@@ -0,0 +1,184 @@
+module ScoutApm
+  module LayerConverters
+    class TraceConverter < ConverterBase
+      ###################
+      #  Converter API  #
+      ###################
+
+
+      def record!
+        @points = context.slow_request_policy.score(request)
+
+        # Let the store know we're here, and if it wants our data, it will call
+        # back into #call
+        @store.track_trace!(self)
+
+        nil # not returning anything in the layer results ... not used
+      end
+
+      #####################
+      #  ScoreItemSet API #
+      #####################
+      def name; request.unique_name; end
+      def score; @points; end
+
+      # Unconditionally attempts to convert this into a DetailedTrace object.
+      # Can return nil if the request didn't have any scope_layer or if `timeline_traces` aren't enabled.
+      def call
+        return nil unless scope_layer
+        return nil unless context.config.value('timeline_traces')
+
+        # Since this request is being stored, update the needed counters
+        context.slow_request_policy.stored!(request)
+
+        # record the change in memory usage
+        mem_delta = ScoutApm::Instruments::Process::ProcessMemory.new(context).rss_to_mb(@request.capture_mem_delta!)
+
+        transaction_id = request.transaction_id
+        revision = context.environment.git_revision.sha
+        start_instant = request.root_layer.start_time
+        stop_instant = request.root_layer.stop_time
+        type = if request.web?
+                 "Web"
+               elsif request.job?
+                 "Job"
+               else
+                 "Unknown"
+               end
+
+        # Create request tags
+        #
+        tags = {
+          :allocations => request.root_layer.total_allocations,
+          :mem_delta => mem_delta,
+        }.merge(request.context.to_flat_hash)
+
+        host = context.environment.hostname
+        path = request.annotations[:uri] || ""
+        code = "" # User#index for instance
+
+        spans = create_spans(request.root_layer)
+        if limited?
+          tags[:"scout.reached_span_cap"] = true
+        end
+
+        DetailedTrace.new(
+          transaction_id,
+          revision,
+          host,
+          start_instant,
+          stop_instant,
+          type,
+
+          path,
+          code,
+
+          spans,
+          tags
+
+          # total_score = 0,
+          # percentile_score = 0,
+          # age_score = 0,
+          # memory_delta_score = 0,
+          # memory_allocations_score = 0
+        )
+      end
+
+      # Returns an array of span objects. Uses recursion to get all children
+      # wired up w/ correct parent_ids
+      def create_spans(layer, parent_id = nil)
+        span_id = ScoutApm::Utils::SpanId.new.to_s
+
+        start_instant = layer.start_time
+        stop_instant = layer.stop_time
+        operation = layer.legacy_metric_name
+        tags = {
+          :start_allocations => layer.allocations_start,
+          :stop_allocations => layer.allocations_stop,
+        }
+        if layer.desc
+          tags[:desc] = layer.desc.to_s
+        end
+        if layer.annotations && layer.annotations[:record_count]
+          tags["db.record_count"] = layer.annotations[:record_count]
+        end
+        if layer.annotations && layer.annotations[:class_name]
+          tags["db.class_name"] = layer.annotations[:class_name]
+        end
+        if layer.backtrace
+          tags[:backtrace] = backtrace_parser(layer.backtrace) rescue nil
+        end
+
+        # Collect up self, and all children into result array
+        result = []
+        result << DetailedTraceSpan.new(
+          span_id.to_s,
+          parent_id.to_s,
+          start_instant,
+          stop_instant,
+          operation,
+          tags)
+
+        layer.children.each do |child|
+          # Don't create spans from limited layers. These don't have start/stop times and our processing can't
+          # handle these yet.
+          unless over_span_limit?(result) || child.is_a?(LimitedLayer)
+            result += create_spans(child, span_id)
+          end
+        end
+
+        return result
+      end
+
+      # Take an array of ruby backtrace lines and split it into an array of hashes like:
+      # ["/Users/cschneid/.rvm/rubies/ruby-2.2.7/lib/ruby/2.2.0/irb/workspace.rb:86:in `eval'", ...]
+      #    turns into:
+      # [ {
+      #     "file": "app/controllers/users_controller.rb",
+      #     "line": 10,
+      #     "function": "index"
+      # },
+      # ]
+      def backtrace_parser(lines)
+        bt = ScoutApm::Utils::BacktraceParser.new(lines).call
+
+        bt.map do |line|
+          match = line.match(/(.*):(\d+):in `(.*)'/)
+          {
+            "file" => match[1],
+            "line" => match[2],
+            "function" => match[3],
+          }
+        end
+
+      end
+
+      ################################################################################
+      # Limit Handling
+      ################################################################################
+
+      # To prevent huge traces from being generated, we stop collecting
+      # spans as we go beyond some reasonably large count.
+      MAX_SPANS = 1500
+
+      def over_span_limit?(spans)
+        if spans.size > MAX_SPANS
+          log_over_span_limit
+          @limited = true
+        else
+          false
+        end
+      end
+
+      def log_over_span_limit
+        unless limited?
+          context.logger.debug "Not recording additional spans for #{name}. Over the span limit."
+        end
+      end
+
+      def limited?
+        !! @limited
+      end
+    end
+  end
+end