scout_apm 3.0.0.pre23 → 4.0.0

data/lib/scout_apm/layer_converters/slow_request_converter.rb

@@ -11,6 +11,7 @@ module ScoutApm
         # Let the store know we're here, and if it wants our data, it will call
         # back into #call
         @store.track_slow_transaction!(self)
+
         nil # not returning anything in the layer results ... not used
       end
 
@@ -51,7 +52,8 @@ module ScoutApm
                             mem_delta,
                             root_layer.total_allocations,
                             @points,
-                            limited?)
+                            limited?,
+                            span_trace)
       end
 
       # Full metrics from this request. These get stored permanently in a SlowTransaction.
@@ -69,9 +71,6 @@ module ScoutApm
 
         walker.on do |layer|
           next if skip_layer?(layer)
-
-          debug_scoutprof(layer)
-
           store_specific_metric(layer, metric_hash, allocation_metric_hash)
           store_aggregate_metric(layer, metric_hash, allocation_metric_hash)
         end
@@ -84,6 +83,17 @@ module ScoutApm
 
         [metric_hash, allocation_metric_hash]
       end
+
+      ###########################################################
+      #  Also create a new style trace. This is not a good      #
+      #  spot for this long term, but fixes an issue for now.   #
+      ###########################################################
+      
+      def span_trace
+        ScoutApm::LayerConverters::TraceConverter.
+          new(@context, @request, @layer_finder, @store).
+          call
+      end
     end
   end
 end

data/lib/scout_apm/layer_converters/trace_converter.rb

@@ -0,0 +1,184 @@
+module ScoutApm
+  module LayerConverters
+    class TraceConverter < ConverterBase
+      ###################
+      #  Converter API  #
+      ###################
+
+
+      def record!
+        @points = context.slow_request_policy.score(request)
+
+        # Let the store know we're here, and if it wants our data, it will call
+        # back into #call
+        @store.track_trace!(self)
+
+        nil # not returning anything in the layer results ... not used
+      end
+
+      #####################
+      #  ScoreItemSet API #
+      #####################
+      def name; request.unique_name; end
+      def score; @points; end
+
+      # Unconditionally attempts to convert this into a DetailedTrace object.
+      # Can return nil if the request didn't have any scope_layer or if `timeline_traces` aren't enabled.
+      def call
+        return nil unless scope_layer
+        return nil unless context.config.value('timeline_traces')
+
+        # Since this request is being stored, update the needed counters
+        context.slow_request_policy.stored!(request)
+
+        # record the change in memory usage
+        mem_delta = ScoutApm::Instruments::Process::ProcessMemory.new(context).rss_to_mb(@request.capture_mem_delta!)
+
+        transaction_id = request.transaction_id
+        revision = context.environment.git_revision.sha
+        start_instant = request.root_layer.start_time
+        stop_instant = request.root_layer.stop_time
+        type = if request.web?
+                 "Web"
+               elsif request.job?
+                 "Job"
+               else
+                 "Unknown"
+               end
+
+        # Create request tags
+        #
+        tags = {
+          :allocations => request.root_layer.total_allocations,
+          :mem_delta => mem_delta,
+        }.merge(request.context.to_flat_hash)
+
+        host = context.environment.hostname
+        path = request.annotations[:uri] || ""
+        code = "" # User#index for instance
+
+        spans = create_spans(request.root_layer)
+        if limited?
+          tags[:"scout.reached_span_cap"] = true
+        end
+
+        DetailedTrace.new(
+          transaction_id,
+          revision,
+          host,
+          start_instant,
+          stop_instant,
+          type,
+
+          path,
+          code,
+
+          spans,
+          tags
+
+          # total_score = 0,
+          # percentile_score = 0,
+          # age_score = 0,
+          # memory_delta_score = 0,
+          # memory_allocations_score = 0
+        )
+      end
+
+      # Returns an array of span objects. Uses recursion to get all children
+      # wired up w/ correct parent_ids
+      def create_spans(layer, parent_id = nil)
+        span_id = ScoutApm::Utils::SpanId.new.to_s
+
+        start_instant = layer.start_time
+        stop_instant = layer.stop_time
+        operation = layer.legacy_metric_name
+        tags = {
+          :start_allocations => layer.allocations_start,
+          :stop_allocations => layer.allocations_stop,
+        }
+        if layer.desc
+          tags[:desc] = layer.desc.to_s
+        end
+        if layer.annotations && layer.annotations[:record_count]
+          tags["db.record_count"] = layer.annotations[:record_count]
+        end
+        if layer.annotations && layer.annotations[:class_name]
+          tags["db.class_name"] = layer.annotations[:class_name]
+        end
+        if layer.backtrace
+          tags[:backtrace] = backtrace_parser(layer.backtrace) rescue nil
+        end
+
+        # Collect up self, and all children into result array
+        result = []
+        result << DetailedTraceSpan.new(
+          span_id.to_s,
+          parent_id.to_s,
+          start_instant,
+          stop_instant,
+          operation,
+          tags)
+
+        layer.children.each do |child|
+          # Don't create spans from limited layers. These don't have start/stop times and our processing can't
+          # handle these yet.
+          unless over_span_limit?(result) || child.is_a?(LimitedLayer)
+            result += create_spans(child, span_id)
+          end
+        end
+
+        return result
+      end
+
+      # Take an array of ruby backtrace lines and split it into an array of hashes like:
+      # ["/Users/cschneid/.rvm/rubies/ruby-2.2.7/lib/ruby/2.2.0/irb/workspace.rb:86:in `eval'", ...]
+      #    turns into:
+      # [ {
+      #     "file": "app/controllers/users_controller.rb",
+      #     "line": 10,
+      #     "function": "index"
+      # },
+      # ]
+      def backtrace_parser(lines)
+        bt = ScoutApm::Utils::BacktraceParser.new(lines).call
+
+        bt.map do |line|
+          match = line.match(/(.*):(\d+):in `(.*)'/)
+          {
+            "file" => match[1],
+            "line" => match[2],
+            "function" => match[3],
+          }
+        end
+
+      end
+
+      ################################################################################
+      # Limit Handling
+      ################################################################################
+
+      # To prevent huge traces from being generated, we stop collecting
+      # spans as we go beyond some reasonably large count.
+      MAX_SPANS = 1500
+
+      def over_span_limit?(spans)
+        if spans.size > MAX_SPANS
+          log_over_span_limit
+          @limited = true
+        else
+          false
+        end
+      end
+
+      def log_over_span_limit
+        unless limited?
+          context.logger.debug "Not recording additional spans for #{name}. Over the span limit."
+        end
+      end
+
+      def limited?
+        !! @limited
+      end
+    end
+  end
+end

data/lib/scout_apm/limited_layer.rb

@@ -85,13 +85,6 @@ module ScoutApm
       nil
     end
 
-    def add_traces(*)
-      # noop
-    end
-
-    def traces
-      ScoutApm::TraceSet.new
-    end
 
     #######################################################################
     #  Many methods don't make any sense on a limited layer. Raise errors  #

data/lib/scout_apm/metric_stats.rb

@@ -9,7 +9,6 @@ class MetricStats
   attr_accessor :sum_of_squares
   attr_accessor :queue
   attr_accessor :latency
-  attr_accessor :traces
 
   def initialize(scoped = false)
     @scoped = scoped
@@ -19,7 +18,6 @@ class MetricStats
     self.min_call_time = 0.0
     self.max_call_time = 0.0
     self.sum_of_squares = 0.0
-    self.traces = []
   end
 
   # Note, that you must include exclusive_time if you wish to set
@@ -41,10 +39,6 @@ class MetricStats
     self
   end
 
-  def add_traces(traces)
-    self.traces += Array(traces)
-  end
-
   # combines data from another MetricStats object
   def combine!(other)
     self.call_count += other.call_count
@@ -53,7 +47,6 @@ class MetricStats
     self.min_call_time = other.min_call_time if self.min_call_time.zero? or other.min_call_time < self.min_call_time
     self.max_call_time = other.max_call_time if other.max_call_time > self.max_call_time
     self.sum_of_squares += other.sum_of_squares
-    self.traces = Array(self.traces) + Array(other.traces)
     self
   end
 
@@ -64,7 +57,6 @@ class MetricStats
       :min_call_time,
       :total_call_time,
       :total_exclusive_time,
-      :traces,
     ]
     ScoutApm::AttributeArranger.call(self, json_attributes)
   end

data/lib/scout_apm/middleware.rb

@@ -26,7 +26,7 @@ module ScoutApm
       ScoutApm::Agent.instance.start
       @started = ScoutApm::Agent.instance.context.started? && ScoutApm::Agent.instance.background_worker_running?
     rescue => e
-      ScoutApm::Agent.instance.context.logger("Failed to start via Middleware: #{e.message}\n\t#{e.backtrace.join("\n\t")}")
+      ScoutApm::Agent.instance.context.logger.info("Failed to start via Middleware: #{e.message}\n\t#{e.backtrace.join("\n\t")}")
     end
   end
 end

data/lib/scout_apm/periodic_work.rb

@@ -12,10 +12,29 @@ module ScoutApm
       ScoutApm::Debug.instance.call_periodic_hooks
       @reporting.process_metrics
       clean_old_percentiles
+
+      if context.config.value('auto_instruments')
+        log_autoinstrument_significant_counts rescue nil
+      end
     end
 
     private
 
+    def log_autoinstrument_significant_counts
+      # Ex key/value -
+      # "/Users/dlite/projects/scout/apm/app/controllers/application_controller.rb"=>[[0.0, 689], [1.0, 16]]
+      hists = context.auto_instruments_layer_histograms.as_json
+      hists_summary = hists.map { |file, buckets|
+        total = buckets.map(&:last).inject(0) { |sum, count| sum + count }
+        significant = (buckets.last.last / total.to_f).round(2)
+        [
+          file,
+          {:total => total, :significant => significant}
+        ]
+      }.to_h
+      context.logger.debug("AutoInstrument Significant Layer Histograms: #{hists_summary.pretty_inspect}")
+    end
+
     # XXX: Move logic into a RequestHistogramsByTime class that can keep the timeout logic in it
     def clean_old_percentiles
       context.

data/lib/scout_apm/remote/message.rb

@@ -17,6 +17,10 @@ module ScoutApm
 
       def encode
         Marshal.dump(self)
+      rescue
+        ScoutApm::Agent.instance.logger.info("Failed Marshalling Remote::Message")
+        ScoutApm::Agent.instance.logger.info(ScoutApm::Utils::MarshalLogging.new(self).dive) rescue nil
+        raise
       end
     end
   end

data/lib/scout_apm/reporter.rb

@@ -2,7 +2,6 @@ require 'openssl'
 
 module ScoutApm
   class Reporter
-    CA_FILE     = File.join( File.dirname(__FILE__), *%w[.. .. data cacert.pem] )
     VERIFY_MODE = OpenSSL::SSL::VERIFY_PEER | OpenSSL::SSL::VERIFY_FAIL_IF_NO_PEER_CERT
 
     attr_reader :type
@@ -23,6 +22,7 @@ module ScoutApm
       context.logger
     end
 
+    # The fully serialized string payload to be sent
     def report(payload, headers = {})
       hosts = determine_hosts
 
@@ -36,6 +36,7 @@ module ScoutApm
         logger.debug("Original Size: #{original_payload_size} Compressed Size: #{compress_payload_size}")
       end
 
+      logger.info("Posting payload to #{hosts.inspect}")
       post_payload(hosts, payload, headers)
     end
 
@@ -52,6 +53,8 @@ module ScoutApm
         URI.parse("#{host}/apps/deploy.scout?key=#{key}&name=#{encoded_app_name}")
       when :instant_trace
         URI.parse("#{host}/apps/instant_trace.scout?key=#{key}&name=#{encoded_app_name}&instant_key=#{instant_key}")
+      when :errors
+        URI.parse("#{host}/apps/error.scout?key=#{key}&name=#{encoded_app_name}")
       end.tap { |u| logger.debug("Posting to #{u}") }
     end
 
@@ -90,7 +93,7 @@ module ScoutApm
       logger.debug "got response: #{response.inspect}"
       case response
       when Net::HTTPSuccess, Net::HTTPNotModified
-        logger.debug "/#{type} OK"
+        logger.debug "#{type} OK"
       when Net::HTTPBadRequest
         logger.warn "/#{type} FAILED: The Account Key [#{config.value('key')}] is invalid."
       when Net::HTTPUnprocessableEntity
@@ -123,7 +126,7 @@ module ScoutApm
                               proxy_uri.password).new(url.host, url.port)
       if url.is_a?(URI::HTTPS)
         http.use_ssl = true
-        http.ca_file = CA_FILE
+        http.ca_file = config.value("ssl_cert_file")
         http.verify_mode = VERIFY_MODE
       end
       http
@@ -142,6 +145,8 @@ module ScoutApm
     def determine_hosts
       if [:deploy_hook, :instant_trace].include?(type)
         config.value('direct_host')
+      elsif [:errors].include?(type)
+        config.value('errors_host')
       else
         config.value('host')
       end

data/lib/scout_apm/reporting.rb

@@ -83,10 +83,11 @@ module ScoutApm
       slow_jobs = reporting_period.slow_jobs_payload
       histograms = reporting_period.histograms
       db_query_metrics = reporting_period.db_query_metrics_payload
+      traces = (slow_transactions.map(&:span_trace) + slow_jobs.map(&:span_trace)).compact
 
       log_deliver(metrics, slow_transactions, metadata, slow_jobs, histograms)
 
-      payload = ScoutApm::Serializers::PayloadSerializer.serialize(metadata, metrics, slow_transactions, jobs, slow_jobs, histograms, db_query_metrics)
+      payload = ScoutApm::Serializers::PayloadSerializer.serialize(metadata, metrics, slow_transactions, jobs, slow_jobs, histograms, db_query_metrics, traces)
       logger.debug("Sending payload w/ Headers: #{headers.inspect}")
 
       reporter.report(payload, headers)

data/lib/scout_apm/request_histograms.rb

@@ -22,6 +22,14 @@ module ScoutApm
       @histograms.keys.each { |n| yield n }
     end
 
+    def as_json
+      Hash[
+        @histograms.map{ |key, histogram|
+          [key, histogram.as_json]
+        }
+      ]
+    end
+
     def add(item, value)
       @histograms[item].add(value)
     end

data/lib/scout_apm/serializers/app_server_load_serializer.rb

@@ -5,6 +5,10 @@ module ScoutApm
     class AppServerLoadSerializer
       def self.serialize(data)
         Marshal.dump(data)
+      rescue
+        ScoutApm::Agent.instance.logger.info("Failed Marshalling AppServerLoad")
+        ScoutApm::Agent.instance.logger.info(ScoutApm::Utils::MarshalLogging.new(data).dive) rescue nil
+        raise
       end
 
       def self.deserialize(data)

data/lib/scout_apm/serializers/directive_serializer.rb

@@ -5,6 +5,10 @@ module ScoutApm
     class DirectiveSerializer
       def self.serialize(data)
         Marshal.dump(data)
+      rescue
+        ScoutApm::Agent.instance.logger.info("Failed Marshalling Directive")
+        ScoutApm::Agent.instance.logger.info(ScoutApm::Utils::MarshalLogging.new(data).dive) rescue nil
+        raise
       end
 
       def self.deserialize(data)