scout_apm 2.5.1 → 5.3.2

data/lib/scout_apm/remote/server.rb

@@ -16,8 +16,20 @@ module ScoutApm
         @server = nil
       end
 
-      def start
+      def require_webrick
         require 'webrick'
+        true
+      rescue LoadError
+        @logger.warn(
+          %q|Could not require Webrick. Ruby 3.0 stopped bundling it
+             automatically, but it is required to instrument Resque. Please add
+             Webrick to your Gemfile.|
+        )
+        false
+      end
+
+      def start
+        return false unless require_webrick
 
         @server = WEBrick::HTTPServer.new(
           :BindAddress => bind,

data/lib/scout_apm/reporter.rb

@@ -2,7 +2,6 @@ require 'openssl'
 
 module ScoutApm
   class Reporter
-    CA_FILE     = File.join( File.dirname(__FILE__), *%w[.. .. data cacert.pem] )
     VERIFY_MODE = OpenSSL::SSL::VERIFY_PEER | OpenSSL::SSL::VERIFY_FAIL_IF_NO_PEER_CERT
 
     attr_reader :type
@@ -23,6 +22,7 @@ module ScoutApm
       context.logger
     end
 
+    # The fully serialized string payload to be sent
     def report(payload, headers = {})
       hosts = determine_hosts
 
@@ -36,6 +36,7 @@ module ScoutApm
         logger.debug("Original Size: #{original_payload_size} Compressed Size: #{compress_payload_size}")
       end
 
+      logger.info("Posting payload to #{hosts.inspect}")
       post_payload(hosts, payload, headers)
     end
 
@@ -52,6 +53,8 @@ module ScoutApm
         URI.parse("#{host}/apps/deploy.scout?key=#{key}&name=#{encoded_app_name}")
       when :instant_trace
         URI.parse("#{host}/apps/instant_trace.scout?key=#{key}&name=#{encoded_app_name}&instant_key=#{instant_key}")
+      when :errors
+        URI.parse("#{host}/apps/error.scout?key=#{key}&name=#{encoded_app_name}")
       end.tap { |u| logger.debug("Posting to #{u}") }
     end
 
@@ -90,7 +93,7 @@ module ScoutApm
       logger.debug "got response: #{response.inspect}"
       case response
       when Net::HTTPSuccess, Net::HTTPNotModified
-        logger.debug "/#{type} OK"
+        logger.debug "#{type} OK"
       when Net::HTTPBadRequest
         logger.warn "/#{type} FAILED: The Account Key [#{config.value('key')}] is invalid."
       when Net::HTTPUnprocessableEntity
@@ -123,7 +126,7 @@ module ScoutApm
                               proxy_uri.password).new(url.host, url.port)
       if url.is_a?(URI::HTTPS)
         http.use_ssl = true
-        http.ca_file = CA_FILE
+        http.ca_file = config.value("ssl_cert_file")
         http.verify_mode = VERIFY_MODE
       end
       http
@@ -142,6 +145,8 @@ module ScoutApm
     def determine_hosts
       if [:deploy_hook, :instant_trace].include?(type)
         config.value('direct_host')
+      elsif [:errors].include?(type)
+        config.value('errors_host')
       else
         config.value('host')
       end

data/lib/scout_apm/reporting.rb

@@ -83,11 +83,12 @@ module ScoutApm
       slow_jobs = reporting_period.slow_jobs_payload
       histograms = reporting_period.histograms
       db_query_metrics = reporting_period.db_query_metrics_payload
+      external_service_metrics = reporting_period.external_service_metrics_payload
       traces = (slow_transactions.map(&:span_trace) + slow_jobs.map(&:span_trace)).compact
 
       log_deliver(metrics, slow_transactions, metadata, slow_jobs, histograms)
 
-      payload = ScoutApm::Serializers::PayloadSerializer.serialize(metadata, metrics, slow_transactions, jobs, slow_jobs, histograms, db_query_metrics, traces)
+      payload = ScoutApm::Serializers::PayloadSerializer.serialize(metadata, metrics, slow_transactions, jobs, slow_jobs, histograms, db_query_metrics, external_service_metrics, traces)
       logger.debug("Sending payload w/ Headers: #{headers.inspect}")
 
       reporter.report(payload, headers)

data/lib/scout_apm/request_histograms.rb

@@ -22,6 +22,14 @@ module ScoutApm
       @histograms.keys.each { |n| yield n }
     end
 
+    def as_json
+      Hash[
+        @histograms.map{ |key, histogram|
+          [key, histogram.as_json]
+        }
+      ]
+    end
+
     def add(item, value)
       @histograms[item].add(value)
     end

data/lib/scout_apm/serializers/app_server_load_serializer.rb

@@ -5,6 +5,10 @@ module ScoutApm
     class AppServerLoadSerializer
       def self.serialize(data)
         Marshal.dump(data)
+      rescue
+        ScoutApm::Agent.instance.logger.info("Failed Marshalling AppServerLoad")
+        ScoutApm::Agent.instance.logger.info(ScoutApm::Utils::MarshalLogging.new(data).dive) rescue nil
+        raise
       end
 
       def self.deserialize(data)

data/lib/scout_apm/serializers/directive_serializer.rb

@@ -5,6 +5,10 @@ module ScoutApm
     class DirectiveSerializer
       def self.serialize(data)
         Marshal.dump(data)
+      rescue
+        ScoutApm::Agent.instance.logger.info("Failed Marshalling Directive")
+        ScoutApm::Agent.instance.logger.info(ScoutApm::Utils::MarshalLogging.new(data).dive) rescue nil
+        raise
       end
 
       def self.deserialize(data)

data/lib/scout_apm/serializers/external_service_serializer_to_json.rb

@@ -0,0 +1,15 @@
+module ScoutApm
+  module Serializers
+    class ExternalServiceSerializerToJson
+      attr_reader :external_service_metrics
+
+      def initialize(external_service_metrics)
+        @external_service_metrics = external_service_metrics
+      end
+
+      def as_json
+        external_service_metrics.map{|metric| metric.as_json }
+      end
+    end
+  end
+end

data/lib/scout_apm/serializers/payload_serializer.rb

@@ -2,9 +2,9 @@
 module ScoutApm
   module Serializers
     class PayloadSerializer
-      def self.serialize(metadata, metrics, slow_transactions, jobs, slow_jobs, histograms, db_query_metrics, traces)
+      def self.serialize(metadata, metrics, slow_transactions, jobs, slow_jobs, histograms, db_query_metrics, external_service_metrics, traces)
         if ScoutApm::Agent.instance.context.config.value("report_format") == 'json'
-          ScoutApm::Serializers::PayloadSerializerToJson.serialize(metadata, metrics, slow_transactions, jobs, slow_jobs, histograms, db_query_metrics, traces)
+          ScoutApm::Serializers::PayloadSerializerToJson.serialize(metadata, metrics, slow_transactions, jobs, slow_jobs, histograms, db_query_metrics, external_service_metrics, traces)
         else
           metadata = metadata.dup
           metadata.default = nil
@@ -22,7 +22,8 @@ module ScoutApm
                        # payloads. At this point, the marshal code branch is
                        # very rarely used anyway.
                        :histograms        => HistogramsSerializerToJson.new(histograms).as_json,
-                       :db_query_metrics  => db_query_metrics)
+                       :db_query_metrics  => db_query_metrics,
+                       :external_service_metrics  => external_service_metrics)
         end
       end
 

data/lib/scout_apm/serializers/payload_serializer_to_json.rb

@@ -2,7 +2,7 @@ module ScoutApm
   module Serializers
     module PayloadSerializerToJson
       class << self
-        def serialize(metadata, metrics, slow_transactions, jobs, slow_jobs, histograms, db_query_metrics, traces)
+        def serialize(metadata, metrics, slow_transactions, jobs, slow_jobs, histograms, db_query_metrics, external_service_metrics, traces)
           metadata.merge!({:payload_version => 2})
 
           jsonify_hash({:metadata => metadata,
@@ -14,6 +14,9 @@ module ScoutApm
                         :db_metrics => {
                           :query => DbQuerySerializerToJson.new(db_query_metrics).as_json,
                         },
+                        :es_metrics => {
+                          :http => ExternalServiceSerializerToJson.new(external_service_metrics).as_json,
+                        },
                         :span_traces => traces.map{ |t| t.as_json },
           })
         end
@@ -46,17 +49,21 @@ module ScoutApm
         end
 
         ESCAPE_MAPPINGS = {
+          # Stackoverflow answer on gsub matches and backslashes
+          # https://stackoverflow.com/a/4149087/2705125
+          '\\' => '\\\\\\\\',
           "\b" => '\\b',
           "\t" => '\\t',
           "\n" => '\\n',
           "\f" => '\\f',
           "\r" => '\\r',
           '"'  => '\\"',
-          '\\' => '\\\\',
         }
 
         def escape(string)
-          ESCAPE_MAPPINGS.inject(string.to_s) {|s, (bad, good)| s.gsub(bad, good) }
+          ESCAPE_MAPPINGS.inject(string.to_s) {|s, (bad, good)| 
+            s.gsub(bad, good)
+          }
         end
 
         def format_by_type(formatee)

data/lib/scout_apm/slow_policy/age_policy.rb

@@ -0,0 +1,33 @@
+require 'scout_apm/slow_policy/policy'
+
+module ScoutApm::SlowPolicy
+  class AgePolicy < Policy
+    # For each minute we haven't seen an endpoint
+    POINT_MULTIPLIER_AGE = 0.25
+
+    # A hash of Endpoint Name to the last time we stored a slow transaction for it.
+    #
+    # Defaults to a start time that is pretty close to application boot time.
+    # So the "age" of an endpoint we've never seen is the time the application
+    # has been running.
+    attr_reader :last_seen
+
+    def initialize(context)
+      super
+
+      zero_time = Time.now
+      @last_seen = Hash.new { |h, k| h[k] = zero_time }
+    end
+
+    def call(request)
+      # How long has it been since we've seen this?
+      age = Time.now - last_seen[request.unique_name]
+
+      age / 60.0 * POINT_MULTIPLIER_AGE
+    end
+
+    def stored!(request)
+      last_seen[request.unique_name] = Time.now
+    end
+  end
+end

data/lib/scout_apm/slow_policy/percent_policy.rb

@@ -0,0 +1,22 @@
+require 'scout_apm/slow_policy/policy'
+
+module ScoutApm::SlowPolicy
+  class PercentPolicy < Policy
+    # Points for an endpoint's who's throughput * response time is a large % of
+    # overall time spent processing requests
+    POINT_MULTIPLIER_PERCENT_TIME = 2.5
+
+    # Of the total time spent handling endpoints in this app, if this endpoint
+    # is a higher percent, it should get more points.
+    #
+    # A: 20 calls @ 100ms each => 2 seconds of total time
+    # B: 10 calls @ 100ms each => 1 second of total time
+    #
+    # Then A is 66% of the total call time
+    def call(request) # Scale 0.0 - 1.0
+      percent = context.transaction_time_consumed.percent_of_total(request.unique_name)
+
+      percent * POINT_MULTIPLIER_PERCENT_TIME
+    end
+  end
+end

data/lib/scout_apm/slow_policy/percentile_policy.rb

@@ -0,0 +1,24 @@
+require 'scout_apm/slow_policy/policy'
+
+module ScoutApm::SlowPolicy
+  class PercentilePolicy < Policy
+    def call(request)
+      # What approximate percentile was this request?
+      total_time = request.root_layer.total_call_time
+      percentile = context.request_histograms.approximate_quantile_of_value(request.unique_name, total_time)
+
+      if percentile < 40
+        0.4 # Don't put much emphasis on capturing low percentiles.
+      elsif percentile < 60
+        1.4 # Highest here to get mean traces
+      elsif percentile < 90
+        0.7 # Between 60 & 90% is fine.
+      elsif percentile >= 90
+        1.4 # Highest here to get 90+%ile traces
+      else
+        # impossible.
+        percentile
+      end
+    end
+  end
+end

data/lib/scout_apm/slow_policy/policy.rb

@@ -0,0 +1,21 @@
+# Note that this is semi-internal API. You should not need this, and if you do
+# we're here to help at support@scoutapm.com. TrackedRequest doesn't change
+# often, but we can't promise a perfectly stable API for it either.
+module ScoutApm::SlowPolicy
+  class Policy
+    attr_reader :context
+
+    def initialize(context)
+      @context = context
+    end
+
+    def call(request)
+      raise NotImplementedError
+    end
+
+    # Override in subclasses to execute some behavior if the request gets a
+    # slot in the ScoredItemSet. Defaults to no-op
+    def stored!(request)
+    end
+  end
+end

data/lib/scout_apm/slow_policy/speed_policy.rb

@@ -0,0 +1,16 @@
+require 'scout_apm/slow_policy/policy'
+
+module ScoutApm::SlowPolicy
+  class SpeedPolicy < Policy
+    # Adjust speed points. See the function
+    POINT_MULTIPLIER_SPEED = 0.25
+
+    # Time in seconds
+    # Logarithm keeps huge times from swamping the other metrics.
+    # 1+ is necessary to keep the log function in positive territory.
+    def call(request)
+      total_time = request.root_layer.total_call_time
+      Math.log(1 + total_time) * POINT_MULTIPLIER_SPEED
+    end
+  end
+end

data/lib/scout_apm/slow_request_policy.rb

@@ -3,43 +3,29 @@
 
 module ScoutApm
   class SlowRequestPolicy
-    CAPTURE_TYPES = [
-      CAPTURE_DETAIL  = "capture_detail",
-      CAPTURE_NONE    = "capture_none",
-    ]
-
-    # Adjust speed points. See the function
-    POINT_MULTIPLIER_SPEED = 0.25
-
-    # For each minute we haven't seen an endpoint
-    POINT_MULTIPLIER_AGE = 0.25
-
-    # Outliers are worth up to "1000ms" of weight
-    POINT_MULTIPLIER_PERCENTILE = 1.0
-
-    # Points for an endpoint's who's throughput * response time is a large % of
-    # overall time spent processing requests
-    POINT_MULTIPLIER_PERCENT_TIME = 2.5
-
-    # A hash of Endpoint Name to the last time we stored a slow transaction for it.
-    #
-    # Defaults to a start time that is pretty close to application boot time.
-    # So the "age" of an endpoint we've never seen is the time the application
-    # has been running.
-    attr_reader :last_seen
-
     # The AgentContext we're running in
     attr_reader :context
+    attr_reader :policies
 
     def initialize(context)
       @context = context
+      @policies = []
+    end
 
-      zero_time = Time.now
-      @last_seen = Hash.new { |h, k| h[k] = zero_time }
+    def add_default_policies
+      add(SlowPolicy::SpeedPolicy.new(context))
+      add(SlowPolicy::PercentilePolicy.new(context))
+      add(SlowPolicy::AgePolicy.new(context))
+      add(SlowPolicy::PercentilePolicy.new(context))
     end
 
-    def stored!(request)
-      last_seen[request.unique_name] = Time.now
+    # policy is an object that behaves like a policy (responds to .call(req) for the score, and .store!(req))
+    def add(policy)
+      unless policy.respond_to?(:call) && policy.respond_to?(:stored!)
+        raise "SlowRequestPolicy must implement policy api call(req) and stored!(req)"
+      end
+
+      @policies << policy
     end
 
     # Determine if this request trace should be fully analyzed by scoring it
@@ -56,56 +42,11 @@ module ScoutApm
         return -1 # A negative score, should never be good enough to store.
       end
 
-      total_time = request.root_layer.total_call_time
-
-      # How long has it been since we've seen this?
-      age = Time.now - last_seen[unique_name]
-
-      # What approximate percentile was this request?
-      percentile = context.request_histograms.approximate_quantile_of_value(unique_name, total_time)
-
-      percent_of_total_time = context.transaction_time_consumed.percent_of_total(unique_name)
-
-      return speed_points(total_time) + percentile_points(percentile) + age_points(age) + percent_time_points(percent_of_total_time)
-    end
-
-    private
-
-    # Time in seconds
-    # Logarithm keeps huge times from swamping the other metrics.
-    # 1+ is necessary to keep the log function in positive territory.
-    def speed_points(time)
-      Math.log(1 + time) * POINT_MULTIPLIER_SPEED
-    end
-
-    def percentile_points(percentile)
-      if percentile < 40
-        0.4 # Don't put much emphasis on capturing low percentiles.
-      elsif percentile < 60
-        1.4 # Highest here to get mean traces
-      elsif percentile < 90
-        0.7 # Between 60 & 90% is fine.
-      elsif percentile >= 90
-        1.4 # Highest here to get 90+%ile traces
-      else
-        # impossible.
-        percentile
-      end
-    end
-
-    def age_points(age)
-      age / 60.0 * POINT_MULTIPLIER_AGE
+      policies.map{ |p| p.call(request) }.sum
     end
 
-    # Of the total time spent handling endpoints in this app, if this endpoint
-    # is a higher percent, it should get more points.
-    #
-    # A: 20 calls @ 100ms each => 2 seconds of total time
-    # B: 10 calls @ 100ms each => 1 second of total time
-    #
-    # Then A is 66% of the total call time
-    def percent_time_points(percent) # Scale 0.0 - 1.0
-      percent * POINT_MULTIPLIER_PERCENT_TIME
+    def stored!(request)
+      policies.each{ |p| p.stored!(request) }
     end
   end
 end

data/lib/scout_apm/store.rb

@@ -52,6 +52,13 @@ module ScoutApm
       }
     end
 
+    def track_external_service_metrics!(external_service_metric_set, options={})
+      @mutex.synchronize {
+        period = find_period(options[:timestamp])
+        period.merge_external_service_metrics!(external_service_metric_set)
+      }
+    end
+
     def track_one!(type, name, value, options={})
       meta = MetricMeta.new("#{type}/#{name}")
       stat = MetricStats.new(false)
@@ -206,8 +213,11 @@ module ScoutApm
 
     attr_reader :db_query_metric_set
 
+    attr_reader :external_service_metric_set
+
     def initialize(timestamp, context)
       @timestamp = timestamp
+      @context = context
 
       @request_traces = ScoredItemSet.new(context.config.value('max_traces'))
       @job_traces = ScoredItemSet.new(context.config.value('max_traces'))
@@ -216,10 +226,16 @@ module ScoutApm
 
       @metric_set = MetricSet.new
       @db_query_metric_set = DbQueryMetricSet.new(context)
+      @external_service_metric_set = ExternalServiceMetricSet.new(context)
 
       @jobs = Hash.new
     end
 
+    def logger
+      @context.logger
+    end
+    private :logger
+
     # Merges another StoreReportingPeriod into this one
     def merge(other)
       self.
@@ -228,7 +244,8 @@ module ScoutApm
         merge_jobs!(other.jobs).
         merge_slow_jobs!(other.slow_jobs_payload).
         merge_histograms!(other.histograms).
-        merge_db_query_metrics!(other.db_query_metric_set)
+        merge_db_query_metrics!(other.db_query_metric_set).
+        merge_external_service_metrics!(other.external_service_metric_set)
       self
     end
 
@@ -254,6 +271,15 @@ module ScoutApm
       self
     end
 
+    def merge_external_service_metrics!(other_metric_set)
+      if other_metric_set.nil?
+        logger.debug("Missing other_metric_set for merge_external_service_metrics - skipping.")
+      else
+        external_service_metric_set.combine!(other_metric_set)
+      end
+      self
+    end
+
     def merge_slow_transactions!(new_transactions)
       Array(new_transactions).each do |one_transaction|
         request_traces << one_transaction
@@ -316,6 +342,10 @@ module ScoutApm
       db_query_metric_set.metrics_to_report
     end
 
+    def external_service_metrics_payload
+      external_service_metric_set.metrics_to_report
+    end
+
     #################################
     # Debug Helpers
     #################################

data/lib/scout_apm/tracer.rb

@@ -91,7 +91,7 @@ module ScoutApm
 
       def _instrumented_method_string(instrumented_name, uninstrumented_name, type, name, options={})
         method_str = <<-EOF
-        def #{instrumented_name}(*args, &block)
+        def #{instrumented_name}(*args#{", **kwargs" if ScoutApm::Agent.instance.context.environment.supports_kwarg_delegation?}, &block)
           name = begin
                    "#{name}"
                  rescue => e
@@ -103,7 +103,7 @@ module ScoutApm
                                name,
                                {:scope => #{options[:scope] || false}}
                              ) do
-            #{uninstrumented_name}(*args, &block)
+            #{uninstrumented_name}(*args#{", **kwargs" if ScoutApm::Agent.instance.context.environment.supports_kwarg_delegation?}, &block)
           end
         end
         EOF

data/lib/scout_apm/tracked_request.rb

@@ -52,6 +52,10 @@ module ScoutApm
     # see that on Sidekiq.
     REQUEST_TYPES = ["Controller", "Job"]
 
+    # Layers of type 'AutoInstrument' are not recorded if their total_call_time doesn't exceed this threshold.
+    # AutoInstrument layers are frequently of short duration. This throws out this deadweight that is unlikely to be optimized.
+    AUTO_INSTRUMENT_TIMING_THRESHOLD = 5/1_000.0 # units = seconds
+
     def initialize(agent_context, store)
       @agent_context = agent_context
       @store = store #this is passed in so we can use a real store (normal operation) or fake store (instant mode only)
@@ -110,7 +114,15 @@ module ScoutApm
       layer.record_stop_time!
       layer.record_allocations!
 
-      @layers[-1].add_child(layer) if @layers.any?
+      # Must follow layer.record_stop_time! as the total_call_time is used to determine if the layer is significant.
+      return if layer_insignificant?(layer)
+
+      # Check that the parent exists before calling a method on it, since some threading can get us into a weird state.
+      # this doesn't fix that state, but prevents exceptions from leaking out.
+      parent = @layers[-1]
+      if parent
+        parent.add_child(layer)
+      end
 
       # This must be called before checking if a backtrace should be collected as the call count influences our capture logic.
       # We call `#update_call_counts in stop layer to ensure the layer has a final desc. Layer#desc is updated during the AR instrumentation flow.
@@ -150,6 +162,10 @@ module ScoutApm
     def capture_backtrace?(layer)
       return if ignoring_request?
 
+      # A backtrace has already been recorded. This happens with autoinstruments as
+      # the partial backtrace is set when creating the layer.
+      return false if layer.backtrace
+
       # Never capture backtraces for this kind of layer. The backtrace will
       # always be 100% framework code.
       return false if BACKTRACE_BLACKLIST.include?(layer.type)
@@ -169,6 +185,20 @@ module ScoutApm
       false
     end
 
+    # Returns +true+ if the total call time of AutoInstrument layers exceeds +AUTO_INSTRUMENT_TIMING_THRESHOLD+ and
+    # records a Histogram of insignificant / significant layers by file name.
+    def layer_insignificant?(layer)
+      result = false # default is significant
+      if layer.type == 'AutoInstrument'
+        if layer.total_call_time < AUTO_INSTRUMENT_TIMING_THRESHOLD
+          result = true # not significant
+        end
+        # 0 = not significant, 1 = significant
+        @agent_context.auto_instruments_layer_histograms.add(layer.file_name, (result ? 0 : 1))
+      end
+      result
+    end
+
     # Maintains a lookup Hash of call counts by layer name. Used to determine if we should capture a backtrace.
     def update_call_counts!(layer)
       @call_set[layer.name].update!(layer.desc)
@@ -212,8 +242,8 @@ module ScoutApm
     def stop_request
       @stopping = true
 
-      if recorder
-        recorder.record!(self)
+      if @recorder
+        @recorder.record!(self)
       end
     end
 
@@ -290,6 +320,7 @@ module ScoutApm
         :queue_time => LayerConverters::RequestQueueTimeConverter,
         :job => LayerConverters::JobConverter,
         :db => LayerConverters::DatabaseConverter,
+        :external_service => LayerConverters::ExternalServiceConverter,
 
         :slow_job => LayerConverters::SlowJobConverter,
         :slow_req => LayerConverters::SlowRequestConverter,
@@ -308,7 +339,7 @@ module ScoutApm
         memo
       end
       walker.walk
-      converter_results = converter_instances.inject({}) do |memo, (slug,i)| 
+      converter_results = converter_instances.inject({}) do |memo, (slug,i)|
         memo[slug] = i.record!
         memo
       end

data/lib/scout_apm/utils/backtrace_parser.rb

@@ -11,6 +11,9 @@ module ScoutApm
 
       attr_reader :call_stack
 
+      # call_stack - an +Array+ of calls, typically generated via the +caller+ method. 
+      # Example single line: 
+      # "/Users/dlite/.rvm/rubies/ruby-2.4.5/lib/ruby/2.4.0/irb/workspace.rb:87:in `eval'"
       def initialize(call_stack, root=ScoutApm::Agent.instance.context.environment.root)
         @call_stack = call_stack
         # We can't use a constant as it'd be too early to fetch environment info