scout_apm 2.5.1 → 5.3.2

data/lib/scout_apm/instruments/moped.rb

@@ -16,32 +16,37 @@ module ScoutApm
         @installed
       end
 
-      def install
+      def install(prepend:)
         if defined?(::Moped)
           @installed = true
 
-          logger.info "Instrumenting Moped"
+          logger.info "Instrumenting Moped. Prepend: #{prepend}"
 
-          ::Moped::Node.class_eval do
-            include ScoutApm::Tracer
+          if prepend
+            ::Moped::Node.send(:include, ScoutApm::Tracer)
+            ::Moped::Node.send(:prepend, MopedInstrumentationPrepend)
+          else
+            ::Moped::Node.class_eval do
+              include ScoutApm::Tracer
 
-            def process_with_scout_instruments(operation, &callback)
-              if operation.respond_to?(:collection)
-                collection = operation.collection
-                name = "Process/#{collection}/#{operation.class.to_s.split('::').last}"
-                self.class.instrument("MongoDB", name, :annotate_layer => { :query => scout_sanitize_log(operation.log_inspect) }) do
-                  process_without_scout_instruments(operation, &callback)
+              def process_with_scout_instruments(operation, &callback)
+                if operation.respond_to?(:collection)
+                  collection = operation.collection
+                  name = "Process/#{collection}/#{operation.class.to_s.split('::').last}"
+                  self.class.instrument("MongoDB", name, :annotate_layer => { :query => scout_sanitize_log(operation.log_inspect) }) do
+                    process_without_scout_instruments(operation, &callback)
+                  end
                 end
               end
-            end
-            alias_method :process_without_scout_instruments, :process
-            alias_method :process, :process_with_scout_instruments
-
-            # replaces values w/ ?
-            def scout_sanitize_log(log)
-              return nil if log.length > 1000 # safeguard - don't sanitize large SQL statements
-              log.gsub(/(=>")((?:[^"]|"")*)"/) do
-                $1 + '?' + '"'
+              alias_method :process_without_scout_instruments, :process
+              alias_method :process, :process_with_scout_instruments
+
+              # replaces values w/ ?
+              def scout_sanitize_log(log)
+                return nil if log.length > 1000 # safeguard - don't sanitize large SQL statements
+                log.gsub(/(=>")((?:[^"]|"")*)"/) do
+                  $1 + '?' + '"'
+                end
               end
             end
           end
@@ -49,6 +54,26 @@ module ScoutApm
       end
 
     end
+
+    module MopedInstrumentationPrepend
+      def process(operation, &callback)
+        if operation.respond_to?(:collection)
+          collection = operation.collection
+          name = "Process/#{collection}/#{operation.class.to_s.split('::').last}"
+          self.class.instrument("MongoDB", name, :annotate_layer => { :query => scout_sanitize_log(operation.log_inspect) }) do
+            super(operation, &callback)
+          end
+        end
+      end
+
+      # replaces values w/ ?
+      def scout_sanitize_log(log)
+        return nil if log.length > 1000 # safeguard - don't sanitize large SQL statements
+        log.gsub(/(=>")((?:[^"]|"")*)"/) do
+          $1 + '?' + '"'
+        end
+      end
+    end
   end
 end
 

data/lib/scout_apm/instruments/net_http.rb

@@ -16,34 +16,69 @@ module ScoutApm
         @installed
       end
 
-      def install
+      def install(prepend:)
         if defined?(::Net) && defined?(::Net::HTTP)
           @installed = true
 
-          logger.info "Instrumenting Net::HTTP"
+          logger.info "Instrumenting Net::HTTP. Prepend: #{prepend}"
 
-          ::Net::HTTP.class_eval do
-            include ScoutApm::Tracer
+          if prepend
+            ::Net::HTTP.send(:include, ScoutApm::Tracer)
+            ::Net::HTTP.send(:prepend, NetHttpInstrumentationPrepend)
+          else
+            ::Net::HTTP.class_eval do
+              include ScoutApm::Tracer
 
-            def request_with_scout_instruments(*args,&block)
-              self.class.instrument("HTTP", "request", :ignore_children => true, :desc => request_scout_description(args.first)) do
-                request_without_scout_instruments(*args, &block)
+              def request_with_scout_instruments(*args, &block)
+                self.class.instrument("HTTP", "request", :ignore_children => true, :desc => request_scout_description(args.first)) do
+                  request_without_scout_instruments(*args, &block)
+                end
               end
-            end
 
-            def request_scout_description(req)
-              path = req.path
-              path = path.path if path.respond_to?(:path)
+              def request_scout_description(req)
+                path = req.path
+                path = path.path if path.respond_to?(:path)
 
-              max_length = ScoutApm::Agent.instance.context.config.value('instrument_http_url_length')
-              (@address + path.split('?').first)[0..(max_length - 1)]
-            end
+                # Protect against a nil address value
+                if @address.nil?
+                  return "No Address Found"
+                end
+
+                max_length = ScoutApm::Agent.instance.context.config.value('instrument_http_url_length')
+                (@address + path.split('?').first)[0..(max_length - 1)]
+              rescue
+                ""
+              end
 
-            alias request_without_scout_instruments request
-            alias request request_with_scout_instruments
+              alias request_without_scout_instruments request
+              alias request request_with_scout_instruments
+            end
           end
         end
       end
     end
+
+    module NetHttpInstrumentationPrepend
+      def request(request, *args, &block)
+        self.class.instrument("HTTP", "request", :ignore_children => true, :desc => request_scout_description(args.first)) do
+          super(request, *args, &block)
+        end
+      end
+
+      def request_scout_description(req)
+        path = req.path
+        path = path.path if path.respond_to?(:path)
+
+        # Protect against a nil address value
+        if @address.nil?
+          return "No Address Found"
+        end
+
+        max_length = ScoutApm::Agent.instance.context.config.value('instrument_http_url_length')
+        (@address + path.split('?').first)[0..(max_length - 1)]
+      rescue
+        ""
+      end
+    end
   end
 end

data/lib/scout_apm/instruments/rails_router.rb

@@ -16,7 +16,7 @@ module ScoutApm
         @installed
       end
 
-      def install
+      def install(prepend:)
         if defined?(ActionDispatch) && defined?(ActionDispatch::Routing) && defined?(ActionDispatch::Routing::RouteSet)
           @installed = true
 

data/lib/scout_apm/instruments/redis.rb

@@ -16,28 +16,43 @@ module ScoutApm
         @installed
       end
 
-      def install
-        if defined?(::Redis) && defined?(::Redis::Client)
+      def install(prepend:)
+        if defined?(::Redis) && defined?(::Redis::Client) && ::Redis::Client.instance_methods(false).include?(:call)
           @installed = true
 
-          logger.info "Instrumenting Redis"
+          logger.info "Instrumenting Redis. Prepend: #{prepend}"
 
-          ::Redis::Client.class_eval do
-            include ScoutApm::Tracer
+          if prepend
+            ::Redis::Client.send(:include, ScoutApm::Tracer)
+            ::Redis::Client.send(:prepend, RedisClientInstrumentationPrepend)
+          else
+            ::Redis::Client.class_eval do
+              include ScoutApm::Tracer
 
-            def call_with_scout_instruments(*args, &block)
-              command = args.first.first rescue "Unknown"
+              def call_with_scout_instruments(*args, &block)
+                command = args.first.first rescue "Unknown"
 
-              self.class.instrument("Redis", command) do
-                call_without_scout_instruments(*args, &block)
+                self.class.instrument("Redis", command) do
+                  call_without_scout_instruments(*args, &block)
+                end
               end
-            end
 
-            alias_method :call_without_scout_instruments, :call
-            alias_method :call, :call_with_scout_instruments
+              alias_method :call_without_scout_instruments, :call
+              alias_method :call, :call_with_scout_instruments
+            end
           end
         end
       end
     end
+
+    module RedisClientInstrumentationPrepend
+      def call(*args, &block)
+        command = args.first.first rescue "Unknown"
+
+        self.class.instrument("Redis", command) do
+          super(*args, &block)
+        end
+      end
+    end
   end
 end

data/lib/scout_apm/instruments/redis5.rb

@@ -0,0 +1,59 @@
+module ScoutApm
+    module Instruments
+      class Redis5
+        attr_reader :context
+  
+        def initialize(context)
+          @context = context
+          @installed = false
+        end
+  
+        def logger
+          context.logger
+        end
+  
+        def installed?
+          @installed
+        end
+  
+        def install(prepend:)
+          if defined?(::Redis) && defined?(::Redis::Client) && ::Redis::Client.instance_methods(false).include?(:call_v)
+            @installed = true
+  
+            logger.info "Instrumenting Redis5. Prepend: #{prepend}"
+  
+            if prepend
+              ::Redis::Client.send(:include, ScoutApm::Tracer)
+              ::Redis::Client.send(:prepend, Redis5ClientInstrumentationPrepend)
+            else
+              ::Redis::Client.class_eval do
+                include ScoutApm::Tracer
+  
+                def call_with_scout_instruments(args, &block)
+                  command = args.first rescue "Unknown"
+  
+                  self.class.instrument("Redis", command) do
+                    call_without_scout_instruments(args, &block)
+                  end
+                end
+  
+                alias_method :call_without_scout_instruments, :call_v
+                alias_method :call_v, :call_with_scout_instruments
+              end
+            end
+          end
+        end
+      end
+  
+      module Redis5ClientInstrumentationPrepend
+        def call(args, &block)
+          command = args.first rescue "Unknown"
+  
+          self.class.instrument("Redis", command) do
+            super(args, &block)
+          end
+        end
+      end
+    end
+  end
+  

data/lib/scout_apm/instruments/sinatra.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: false
+
 # XXX: Is this file used?
 module ScoutApm
   module Instruments
@@ -13,7 +15,7 @@ module ScoutApm
         @installed
       end
 
-      def install
+      def install(prepend:)
         if defined?(::Sinatra) && defined?(::Sinatra::Base) && ::Sinatra::Base.private_method_defined?(:dispatch!)
           @installed = true
 

data/lib/scout_apm/instruments/typhoeus.rb

@@ -0,0 +1,90 @@
+module ScoutApm
+  module Instruments
+    class Typhoeus
+      attr_reader :context
+
+      def initialize(context)
+        @context = context
+        @installed = false
+      end
+
+      def logger
+        context.logger
+      end
+
+      def installed?
+        @installed
+      end
+
+      def install(prepend:)
+        if defined?(::Typhoeus)
+          @installed = true
+
+          logger.info "Instrumenting Typhoeus"
+
+          ::Typhoeus::Request.send(:prepend, TyphoeusInstrumentation)
+          ::Typhoeus::Hydra.send(:prepend, TyphoeusHydraInstrumentation)
+        end
+      end
+
+      module TyphoeusHydraInstrumentation
+        def run(*args, &block)
+          layer = ScoutApm::Layer.new("HTTP", "Hydra")
+          layer.desc = scout_desc
+
+          req = ScoutApm::RequestManager.lookup
+          req.start_layer(layer)
+
+          begin
+            super(*args, &block)
+          ensure
+            req.stop_layer
+          end
+        end
+
+        def scout_desc
+          "#{self.queued_requests.count} requests"
+        rescue
+          ""
+        end
+      end
+
+      module TyphoeusInstrumentation
+        def run(*args, &block)
+          layer = ScoutApm::Layer.new("HTTP", scout_request_verb)
+          layer.desc = scout_desc(scout_request_verb, scout_request_url)
+
+          req = ScoutApm::RequestManager.lookup
+          req.start_layer(layer)
+
+          begin
+            super(*args, &block)
+          ensure
+            req.stop_layer
+          end
+        end
+
+        def scout_desc(verb, uri)
+          max_length = ScoutApm::Agent.instance.context.config.value('instrument_http_url_length')
+          (String(uri).split('?').first)[0..(max_length - 1)]
+        rescue
+          ""
+        end
+
+        def scout_request_url
+          self.url
+        rescue
+          ""
+        end
+
+        def scout_request_verb
+          self.options[:method].to_s
+        rescue
+          ""
+        end
+
+      end
+
+    end
+  end
+end

data/lib/scout_apm/job_record.rb

@@ -32,8 +32,10 @@ module ScoutApm
 
     # Modifies self and returns self, after merging in `other`.
     def combine!(other)
-      same_job = queue_name == other.queue_name && job_name == other.job_name
-      raise "Mismatched Merge of Background Job" unless same_job
+      if !self.eql?(other)
+        ScoutApm::Agent.instance.logger.debug("Mismatched Merge of Background Job: (Queue #{queue_name} == #{other.queue_name}) (Name #{job_name} == #{other.job_name}) (Hash #{hash} == #{other.hash})")
+        return self
+      end
 
       @errors += other.errors
       @metric_set = metric_set.combine!(other.metric_set)

data/lib/scout_apm/layaway_file.rb

@@ -30,6 +30,10 @@ module ScoutApm
 
     def serialize(data)
       Marshal.dump(data)
+    rescue
+      ScoutApm::Agent.instance.logger.info("Failed Marshalling LayawayFile")
+      ScoutApm::Agent.instance.logger.info(ScoutApm::Utils::MarshalLogging.new(data).dive) rescue nil
+      raise
     end
 
     def deserialize(data)

data/lib/scout_apm/layer.rb

@@ -36,7 +36,10 @@ module ScoutApm
 
     # If this layer took longer than a fixed amount of time, store the
     # backtrace of where it occurred.
-    attr_reader :backtrace
+    attr_accessor :backtrace
+
+    # The file name associated with the layer. Only used for autoinstruments overhead logging.
+    attr_accessor :file_name
 
     # As we go through a part of a request, instrumentation can store additional data
     # Known Keys:
@@ -113,7 +116,7 @@ module ScoutApm
     # In Ruby 2.0+, we can pass the range directly to the caller to reduce the memory footprint.
     def caller_array
       # omits the first several callers which are in the ScoutAPM stack.
-      if ScoutApm::Agent.instance.context.environment.ruby_2?
+      if ScoutApm::Agent.instance.context.environment.ruby_2? || ScoutApm::Agent.instance.context.environment.ruby_3? 
         caller(3...BACKTRACE_CALLER_LIMIT)
       else
         caller[3...BACKTRACE_CALLER_LIMIT]

data/lib/scout_apm/layer_children_set.rb

@@ -46,9 +46,15 @@ module ScoutApm
       set = child_set(metric_type)
 
       if set.size >= unique_cutoff
-        # find limited_layer
-        @limited_layers || init_limited_layers
-        @limited_layers[metric_type].absorb(child)
+        # find or create limited_layer
+        @limited_layers ||= Hash.new 
+        layer = if @limited_layers.has_key?(metric_type)
+                  @limited_layers[metric_type]
+                else
+                  @limited_layers[metric_type] = LimitedLayer.new(metric_type)
+                end
+
+        layer.absorb(child)
       else
         # we have space just add it
         set << child
@@ -76,10 +82,5 @@ module ScoutApm
     def size
       @children.size
     end
-
-    # hold off initializing this until we know we need it
-    def init_limited_layers
-      @limited_layers ||= Hash.new { |hash, key| hash[key] = LimitedLayer.new(key) }
-    end
   end
 end

data/lib/scout_apm/layer_converters/external_service_converter.rb

@@ -0,0 +1,65 @@
+module ScoutApm
+  module LayerConverters
+    class ExternalServiceConverter < ConverterBase
+      def initialize(*)
+        super
+        @external_service_metric_set = ExternalServiceMetricSet.new(context)
+      end
+
+      def register_hooks(walker)
+        super
+
+        return unless scope_layer
+
+        walker.on do |layer|
+          next if skip_layer?(layer)
+          stat = ExternalServiceMetricStats.new(
+            domain_name(layer),
+            operation_name(layer),          # operation name/verb. GET/POST/PUT etc.
+            scope_layer.legacy_metric_name, # controller_scope
+            1,                              # count, this is a single call, so 1
+            layer.total_call_time
+          )
+          @external_service_metric_set << stat
+        end
+      end
+
+      def record!
+        # Everything in the metric set here is from a single transaction, which
+        # we want to keep track of. (One web call did a User#find 10 times, but
+        # only due to 1 http request)
+        @external_service_metric_set.increment_transaction_count!
+        @store.track_external_service_metrics!(@external_service_metric_set)
+
+        nil # not returning anything in the layer results ... not used
+      end
+
+      def skip_layer?(layer)
+        layer.type != 'HTTP' ||
+          layer.limited? ||
+          super
+      end
+
+      private
+
+      # If we can't name the domain name, default to:
+      DEFAULT_DOMAIN = "Unknown"
+
+      def domain_name(layer)
+        domain = ""
+        desc_str = layer.desc.to_s
+        desc_str = 'http://' + desc_str unless desc_str =~ /^http/i
+        domain = URI.parse(desc_str).host
+      rescue
+        # Do nothing
+      ensure
+        domain = DEFAULT_DOMAIN if domain.to_s.blank?
+        domain
+      end
+
+      def operation_name(layer)
+        "all" # Hardcode to "all" until we support breakout by verb
+      end
+    end
+  end
+end

data/lib/scout_apm/layer_converters/find_layer_by_type.rb

@@ -5,6 +5,10 @@
 #       show
 #       render :update
 #     end
+
+# This doesn't cache the negative result when searching for a controller / job,
+# so that we can ask again later after more of the request has occurred and
+# correctly find it.
 module ScoutApm
   module LayerConverters
     class FindLayerByType

data/lib/scout_apm/layer_converters/request_queue_time_converter.rb

@@ -11,6 +11,8 @@ module ScoutApm
       def record!
         return unless request.web?
 
+        return unless context.config.value('record_queue_time')
+
         return unless headers
 
         raw_start = locate_timestamp

data/lib/scout_apm/layer_converters/trace_converter.rb

@@ -58,6 +58,9 @@ module ScoutApm
         code = "" # User#index for instance
 
         spans = create_spans(request.root_layer)
+        if limited?
+          tags[:"scout.reached_span_cap"] = true
+        end
 
         DetailedTrace.new(
           transaction_id,
@@ -117,7 +120,9 @@ module ScoutApm
           tags)
 
         layer.children.each do |child|
-          unless over_span_limit?(result)
+          # Don't create spans from limited layers. These don't have start/stop times and our processing can't
+          # handle these yet.
+          unless over_span_limit?(result) || child.is_a?(LimitedLayer)
             result += create_spans(child, span_id)
           end
         end
@@ -152,10 +157,8 @@ module ScoutApm
       # Limit Handling
       ################################################################################
 
-      # To prevent huge traces from being generated, we should stop collecting
+      # To prevent huge traces from being generated, we stop collecting
       # spans as we go beyond some reasonably large count.
-      # Until the root cause of https://github.com/scoutapp/scout_apm_ruby/issues/267 is addressed,
-      # keep `MAX_SPANS` less than `DEFAULT_UNIQUE_CUTOFF`.
       MAX_SPANS = 1500
 
       def over_span_limit?(spans)

data/lib/scout_apm/logger.rb

@@ -70,13 +70,17 @@ module ScoutApm
 
     def build_logger
       logger_class.new(@log_destination)
+    rescue => e
+      logger = ::Logger.new(STDERR)
+      logger.error("Error while building ScoutApm logger: #{e.message}. Falling back to STDERR")
+      logger
     end
 
     def logger_class
       klass = @opts.fetch(:logger_class, ::Logger)
       case klass
       when String
-        result = KlassHelper.lookup(klass)
+        result = Utils::KlassHelper.lookup(klass)
         if result == :missing_class
           ::Logger
         else

data/lib/scout_apm/middleware.rb

@@ -26,7 +26,7 @@ module ScoutApm
       ScoutApm::Agent.instance.start
       @started = ScoutApm::Agent.instance.context.started? && ScoutApm::Agent.instance.background_worker_running?
     rescue => e
-      ScoutApm::Agent.instance.context.logger("Failed to start via Middleware: #{e.message}\n\t#{e.backtrace.join("\n\t")}")
+      ScoutApm::Agent.instance.context.logger.info("Failed to start via Middleware: #{e.message}\n\t#{e.backtrace.join("\n\t")}")
     end
   end
 end

data/lib/scout_apm/periodic_work.rb

@@ -12,10 +12,29 @@ module ScoutApm
       ScoutApm::Debug.instance.call_periodic_hooks
       @reporting.process_metrics
       clean_old_percentiles
+
+      if context.config.value('auto_instruments')
+        log_autoinstrument_significant_counts rescue nil
+      end
     end
 
     private
 
+    def log_autoinstrument_significant_counts
+      # Ex key/value -
+      # "/Users/dlite/projects/scout/apm/app/controllers/application_controller.rb"=>[[0.0, 689], [1.0, 16]]
+      hists = context.auto_instruments_layer_histograms.as_json
+      hists_summary = hists.map { |file, buckets|
+        total = buckets.map(&:last).inject(0) { |sum, count| sum + count }
+        significant = (buckets.last.last / total.to_f).round(2)
+        [
+          file,
+          {:total => total, :significant => significant}
+        ]
+      }.to_h
+      context.logger.debug("AutoInstrument Significant Layer Histograms: #{hists_summary.pretty_inspect}")
+    end
+
     # XXX: Move logic into a RequestHistogramsByTime class that can keep the timeout logic in it
     def clean_old_percentiles
       context.

data/lib/scout_apm/remote/message.rb

@@ -17,6 +17,10 @@ module ScoutApm
 
       def encode
         Marshal.dump(self)
+      rescue
+        ScoutApm::Agent.instance.logger.info("Failed Marshalling Remote::Message")
+        ScoutApm::Agent.instance.logger.info(ScoutApm::Utils::MarshalLogging.new(self).dive) rescue nil
+        raise
       end
     end
   end