scout_apm 2.5.1 → 5.3.2

data/lib/scout_apm/auto_instrument/rails.rb

@@ -0,0 +1,174 @@
+
+require 'scout_apm/auto_instrument/layer'
+require 'scout_apm/auto_instrument/parser'
+
+module ScoutApm
+  module AutoInstrument
+    module Rails
+      # A general pattern to match Rails controller files:
+      CONTROLLER_FILE = /\/app\/controllers\/*\/.*_controller.rb$/.freeze
+
+      # Some gems (Devise) provide controllers that match CONTROLLER_FILE pattern.
+      # Try a simple match to see if it's a Gemfile
+      GEM_FILE = /\/gems?\//.freeze
+
+      # Whether the given path is likely to be a Rails controller and not provided by a Gem.
+      def self.controller_path? path
+        CONTROLLER_FILE.match(path) && !GEM_FILE.match(path)
+      end
+
+      # Autoinstruments increases overhead when applied to many code expressions that perform little work.
+      # You can exclude files from autoinstruments via the `auto_instruments_ignore` option.
+      def self.ignore?(path)
+        res = false
+        ScoutApm::Agent.instance.context.config.value('auto_instruments_ignore').each do |ignored_file_name|
+          if path.include?(ignored_file_name)
+            res = true
+            break
+          end
+        end
+        res
+      end
+
+      def self.rewrite(path, code = nil)
+        code ||= File.read(path)
+
+        ast = ::Parser::CurrentRuby.parse(code)
+
+        # pp ast
+
+        buffer = ::Parser::Source::Buffer.new(path)
+        buffer.source = code
+
+        rewriter = Rewriter.new
+
+        # Rewrite the AST, returns a String with the new form.
+        rewriter.rewrite(buffer, ast)
+      end
+
+      class Rewriter < ::Parser::TreeRewriter
+        def initialize
+          super
+
+          # Keeps track of the parent - child relationship between nodes:
+          @nesting = []
+
+          # The stack of method nodes (type :def):
+          @method = []
+
+          # The stack of class nodes:
+          @scope = []
+
+          @cache = Cache.new
+        end
+
+        def instrument(source, file_name, line)
+          # Don't log huge chunks of code... just the first line:
+          if lines = source.lines and lines.count > 1
+            source = lines.first.chomp + "..."
+          end
+
+          method_name = @method.last.children[0]
+          bt = ["#{file_name}:#{line}:in `#{method_name}'"]
+
+          return [
+            "::ScoutApm::AutoInstrument("+ source.dump + ",#{bt}){",
+            "}"
+          ]
+        end
+
+        # Look up 1 or more nodes to check if the parent exists and matches the given type.
+        # @param type [Symbol] the symbol type to match.
+        # @param up [Integer] how far up to look.
+        def parent_type?(type, up = 1)
+          parent = @nesting[@nesting.size - up - 1] and parent.type == type
+        end
+
+        def on_block(node)
+          # If we are not in a method, don't do any instrumentation:
+          return if @method.empty?
+
+          line = node.location.line || 'line?'
+          column = node.location.column || 'column?' # not used
+          method_name = node.children[0].children[1] || '*unknown*' # not used
+          file_name = @source_rewriter.source_buffer.name
+
+          wrap(node.location.expression, *instrument(node.location.expression.source, file_name, line))
+        end
+
+        def on_mlhs(node)
+          # Ignore / don't instrument multiple assignment (LHS).
+          return
+        end
+
+        def on_op_asgn(node)
+          process(node.children[2])
+        end
+
+        def on_or_asgn(node)
+          process(node.children[1])
+        end
+
+        def on_and_asgn(node)
+          process(node.children[1])
+        end
+
+        # Handle the method call AST node. If this method doesn't call `super`, no futher rewriting is applied to children.
+        def on_send(node)
+          # We aren't interested in top level function calls:
+          return if @method.empty?
+
+          if @cache.local_assignments?(node)
+            return super
+          end
+
+          # This ignores both initial block method invocation `*x*{}`, and subsequent nested invocations `x{*y*}`:
+          return if parent_type?(:block)
+
+          # Extract useful metadata for instrumentation:
+          line = node.location.line || 'line?'
+          column = node.location.column || 'column?' # not used
+          method_name = node.children[1] || '*unknown*' # not used
+          file_name = @source_rewriter.source_buffer.name
+
+          # Wrap the expression with instrumentation:
+          wrap(node.location.expression, *instrument(node.location.expression.source, file_name, line))
+        end
+
+        # def on_class(node)
+        #   class_name = node.children[1]
+        #
+        #   Kernel.const_get(class_name).ancestors.include? ActionController::Controller
+        #
+        #   if class_name =~ /.../
+        #     super # continue processing
+        #   end
+        # end
+
+        # Invoked for every AST node as it is processed top to bottom.
+        def process(node)
+          # We are nesting inside this node:
+          @nesting.push(node)
+
+          if node and node.type == :def
+            # If the node is a method, push it on the method stack as well:
+            @method.push(node)
+            super
+            @method.pop
+          elsif node and node.type == :class
+            @scope.push(node.children[0])
+            super
+            @scope.pop
+          else
+            super
+          end
+
+          @nesting.pop
+        end
+      end
+    end
+  end
+end
+
+# Force any lazy loading to occur here, before we patch iseq_load. Otherwise you might end up in an infinite loop when rewriting code.
+ScoutApm::AutoInstrument::Rails.rewrite('(preload)', '')

data/lib/scout_apm/auto_instrument.rb

@@ -0,0 +1,5 @@
+if RUBY_VERSION >= "2.3"
+  require 'scout_apm/auto_instrument/instruction_sequence'
+else
+  warn "ScoutApm::AutoInstrument requires Ruby >= v2.3. Skipping."
+end

data/lib/scout_apm/background_job_integrations/delayed_job.rb

@@ -11,7 +11,7 @@ module ScoutApm
       end
 
       def present?
-        defined?(::Delayed::Job)
+        defined?(::Delayed::Worker)
       end
 
       def forking?

data/lib/scout_apm/background_job_integrations/faktory.rb

@@ -0,0 +1,103 @@
+module ScoutApm
+  module BackgroundJobIntegrations
+    class Faktory
+      attr_reader :logger
+
+      def name
+        :faktory
+      end
+
+      def present?
+        defined?(::Faktory)
+      end
+
+      def forking?
+        false
+      end
+
+      def install
+        add_middleware
+        install_processor
+      end
+
+      def add_middleware
+        ::Faktory.configure_worker do |config|
+          config.worker_middleware do |chain|
+            chain.add FaktoryMiddleware
+          end
+        end
+      end
+
+      def install_processor
+        require 'faktory/processor' # sidekiq v4 has not loaded this file by this point
+
+        ::Faktory::Processor.class_eval do
+          def initialize_with_scout(*args)
+            agent = ::ScoutApm::Agent.instance
+            agent.start
+            initialize_without_scout(*args)
+          end
+
+          alias_method :initialize_without_scout, :initialize
+          alias_method :initialize, :initialize_with_scout
+        end
+      end
+    end
+
+    # We insert this middleware into the Sidekiq stack, to capture each job,
+    # and time them.
+    class FaktoryMiddleware
+      def call(worker_instance, job)
+        queue = job["queue"]
+
+        req = ScoutApm::RequestManager.lookup
+        req.annotate_request(:queue_latency => latency(job))
+
+        begin
+          req.start_layer(ScoutApm::Layer.new('Queue', queue))
+          started_queue = true
+          req.start_layer(ScoutApm::Layer.new('Job', job_class(job)))
+          started_job = true
+
+          yield
+        rescue
+          req.error!
+          raise
+        ensure
+          req.stop_layer if started_job
+          req.stop_layer if started_queue
+        end
+      end
+
+      UNKNOWN_CLASS_PLACEHOLDER = 'UnknownJob'.freeze
+      ACTIVE_JOB_KLASS = 'ActiveJob::QueueAdapters::FaktoryAdapter::JobWrapper'.freeze
+
+      def job_class(job)
+        job_class = job.fetch('jobtype', UNKNOWN_CLASS_PLACEHOLDER)
+
+        if job_class == ACTIVE_JOB_KLASS && job.key?('custom') && job['custom'].key?('wrapped')
+          begin
+            job_class = job['custom']['wrapped']
+          rescue
+            ACTIVE_JOB_KLASS
+          end
+        end
+
+        job_class
+      rescue
+        UNKNOWN_CLASS_PLACEHOLDER
+      end
+
+      def latency(job, time = Time.now)
+        created_at = Time.parse(job['enqueued_at'] || job['created_at'])
+        if created_at
+          (time - created_at)
+        else
+          0
+        end
+      rescue
+        0
+      end
+    end
+  end
+end

data/lib/scout_apm/background_job_integrations/legacy_sneakers.rb

@@ -0,0 +1,55 @@
+# This is different than other BackgroundJobIntegrations and must be prepended
+# manually in each job.
+#
+# class MyWorker
+#   prepend ScoutApm::BackgroundJobIntegrations::LegacySneakers
+#
+#   def work(msg)
+#     ...
+#   end
+# end
+module ScoutApm
+  module BackgroundJobIntegrations
+    module LegacySneakers
+      UNKNOWN_QUEUE_PLACEHOLDER = 'default'.freeze
+
+      def self.prepended(base)
+        ScoutApm::Agent.instance.logger.info("Prepended LegacySneakers in #{base}")
+      end
+
+      def initialize(*args)
+        super
+
+        # Save off the existing value to call the correct existing work
+        # function in the instrumentation. But then override Sneakers to always
+        # use the extra-argument version, which has data Scout needs
+        @call_work = respond_to?(:work)
+      end
+
+      def work_with_params(msg, delivery_info, metadata)
+        queue = delivery_info[:routing_key] || UNKNOWN_QUEUE_PLACEHOLDER
+        job_class = self.class.name
+        req = ScoutApm::RequestManager.lookup
+
+        begin
+          req.start_layer(ScoutApm::Layer.new('Queue', queue))
+          started_queue = true
+          req.start_layer(ScoutApm::Layer.new('Job', job_class))
+          started_job = true
+
+          if @call_work
+            work(msg)
+          else
+            super
+          end
+        rescue Exception
+          req.error!
+          raise
+        ensure
+          req.stop_layer if started_job
+          req.stop_layer if started_queue
+        end
+      end
+    end
+  end
+end

data/lib/scout_apm/background_job_integrations/que.rb

@@ -0,0 +1,134 @@
+module ScoutApm
+  module BackgroundJobIntegrations
+    class Que
+
+      UNKNOWN_CLASS_PLACEHOLDER = 'UnknownJob'.freeze
+      ACTIVE_JOB_KLASS = 'ActiveJob::QueueAdapters::QueAdapter::JobWrapper'.freeze
+      UNKNOWN_QUEUE_PLACEHOLDER = 'UnknownQueue'.freeze
+
+      attr_reader :logger
+
+      def name
+        :que
+      end
+
+      def present?
+        if defined?(::Que) && File.basename($PROGRAM_NAME).start_with?('que')
+          # 0.x releases used "Version" constant.
+          # 1.x releases used "VERSION" constant.
+          return defined?(::Que::Version)
+        end
+      end
+
+      def forking?
+        false
+      end
+
+      def install
+        install_tracer
+        install_worker
+        install_job
+      end
+
+      def install_tracer
+        ::Que::Job.class_eval do
+          include ScoutApm::Tracer
+        end
+      end
+
+      def install_worker
+        ::Que::Worker.class_eval do
+          def initialize_with_scout(*args)
+            agent = ::ScoutApm::Agent.instance
+            agent.start
+            initialize_without_scout(*args)
+          end
+
+          alias_method :initialize_without_scout, :initialize
+          alias_method :initialize, :initialize_with_scout
+        end
+      end
+
+      def install_job
+        ::Que::Job.class_eval do
+          # attrs = {
+          #   "queue"=>"",
+          #   "priority"=>100,
+          #   "run_at"=>2018-12-19 15:12:32 -0700,
+          #   "job_id"=>4,
+          #   "job_class"=>"ExampleJob",
+          #   "args"=>[{"key"=>"value"}],
+          #   "error_count"=>0
+          # }
+          #
+          # With ActiveJob, v0.14.3:
+          # attrs = {
+          #   "queue"=>"",
+          #   "priority"=>100,
+          #   "run_at"=>2018-12-19 15:29:18 -0700,
+          #   "job_id"=>6,
+          #   "job_class"=>"ActiveJob::QueueAdapters::QueAdapter::JobWrapper",
+          #   "args"=> [{
+          #     "job_class"=>"ExampleJob",
+          #     "job_id"=>"60798b45-4b55-436e-806d-693939266c97",
+          #     "provider_job_id"=>nil,
+          #     "queue_name"=>"default",
+          #     "priority"=>nil,
+          #     "arguments"=>[],
+          #     "executions"=>0,
+          #     "locale"=>"en"
+          #   }],
+          #   "error_count"=>0
+          # }
+          #
+          # With ActiveJob, v1.0:
+          #   There are changes here to make Que more compatible with ActiveJob
+          #   and this probably needs to be revisited.
+
+          def _run_with_scout(*args)
+            # default queue unless specifed is a blank string
+            queue = attrs['queue'] || UNKNOWN_QUEUE_PLACEHOLDER
+
+            job_class = begin
+                          if self.class == ActiveJob::QueueAdapters::QueAdapter::JobWrapper
+                            Array(attrs['args']).first['job_class'] || UNKNOWN_CLASS_PLACEHOLDER
+                          else
+                            self.class.name
+                          end
+                        rescue => e
+                          UNKNOWN_CLASS_PLACEHOLDER
+                        end
+
+            latency = begin
+                        Time.now - attrs['run_at']
+                      rescue
+                        0
+                      end
+
+            req = ScoutApm::RequestManager.lookup
+            req.annotate_request(:queue_latency => latency)
+
+            begin
+              req.start_layer(ScoutApm::Layer.new('Queue', queue))
+              started_queue = true
+              req.start_layer(ScoutApm::Layer.new('Job', job_class))
+              started_job = true
+
+              _run_without_scout(*args)
+            rescue Exception => e
+              req.error!
+              raise
+            ensure
+              req.stop_layer if started_job
+              req.stop_layer if started_queue
+            end
+          end
+
+          alias_method :_run_without_scout, :_run
+          alias_method :_run, :_run_with_scout
+        end
+      end
+    end
+
+  end
+end

data/lib/scout_apm/background_job_integrations/shoryuken.rb

@@ -37,6 +37,8 @@ module ScoutApm
       end
 
       def install_processor
+        # celluloid has not loaded by this point and older versions of `shorykuen/processor` assume that it did
+        require 'celluloid' if defined?(::Shoryuken::VERSION) && ::Shoryuken::VERSION < '3'
         require 'shoryuken/processor' # sidekiq v4 has not loaded this file by this point
 
         ::Shoryuken::Processor.class_eval do

data/lib/scout_apm/background_job_integrations/sidekiq.rb

@@ -37,17 +37,11 @@ module ScoutApm
       end
 
       def install_processor
-        require 'sidekiq/processor' # sidekiq v4 has not loaded this file by this point
-
-        ::Sidekiq::Processor.class_eval do
-          def initialize_with_scout(boss)
+        ::Sidekiq.configure_server do |config|
+          config.on(:startup) do
             agent = ::ScoutApm::Agent.instance
             agent.start
-            initialize_without_scout(boss)
           end
-
-          alias_method :initialize_without_scout, :initialize
-          alias_method :initialize, :initialize_with_scout
         end
       end
     end
@@ -80,12 +74,23 @@ module ScoutApm
       DELAYED_WRAPPER_KLASS = 'Sidekiq::Extensions::DelayedClass'.freeze
 
 
+      # Capturing the class name is a little tricky, since we need to handle several cases:
+      # 1. ActiveJob, with the class in the key 'wrapped'
+      # 2. ActiveJob, but the 'wrapped' key is wrong (due to YAJL serializing weirdly), find it in args.job_class
+      # 3. DelayedJob wrapper, deserializing using YAML into the real object, which can be introspected
+      # 4. No wrapper, just sidekiq's class
       def job_class(msg)
         job_class = msg.fetch('class', UNKNOWN_CLASS_PLACEHOLDER)
 
-        if job_class == ACTIVE_JOB_KLASS && msg.key?('wrapped')
+        if job_class == ACTIVE_JOB_KLASS && msg.key?('wrapped') && msg['wrapped'].is_a?(String)
+          begin
+            job_class = msg['wrapped'].to_s
+          rescue
+            ACTIVE_JOB_KLASS
+          end
+        elsif job_class == ACTIVE_JOB_KLASS && msg.try(:[], 'args').try(:[], 'job_class')
           begin
-            job_class = msg['wrapped']
+            job_class = msg['args']['job_class'].to_s
           rescue
             ACTIVE_JOB_KLASS
           end

data/lib/scout_apm/background_job_integrations/sneakers.rb

@@ -1,14 +1,22 @@
 module ScoutApm
   module BackgroundJobIntegrations
     class Sneakers
-      attr_reader :logger
-
       def name
         :sneakers
       end
 
       def present?
-        defined?(::Sneakers)
+        defined?(::Sneakers) && supported_version?
+      end
+      
+      # Only support Sneakers 2.7 and up
+      def supported_version?
+        result = Gem::Version.new(::Sneakers::VERSION) > Gem::Version.new("2.7.0")
+        ScoutApm::Agent.instance.logger.info("Skipping Sneakers instrumentation. Only versions 2.7+ are supported. See docs or contact support@scoutapm.com for instrumentation of older versions.")
+        result
+      rescue
+        ScoutApm::Agent.instance.logger.info("Failed comparing Sneakers Version. Skipping")
+        false
       end
 
       def forking?
@@ -69,14 +77,6 @@ module ScoutApm
           alias_method :process_work_without_scout, :process_work
           alias_method :process_work, :process_work_with_scout
         end
-
-        # msg = {
-        #   "job_class":"DummyWorker",
-        #   "job_id":"ea23ba1c-3022-4e05-870b-c3bcb1c4f328",
-        #   "queue_name":"default",
-        #   "arguments":["fjdkl"],
-        #   "locale":"en"
-        # }
       end
 
       ACTIVE_JOB_KLASS = 'ActiveJob::QueueAdapters::SneakersAdapter::JobWrapper'.freeze