scout_apm 3.0.0.pre27 → 4.0.3

data/lib/scout_apm/background_job_integrations/que.rb

@@ -0,0 +1,134 @@
+module ScoutApm
+  module BackgroundJobIntegrations
+    class Que
+
+      UNKNOWN_CLASS_PLACEHOLDER = 'UnknownJob'.freeze
+      ACTIVE_JOB_KLASS = 'ActiveJob::QueueAdapters::QueAdapter::JobWrapper'.freeze
+      UNKNOWN_QUEUE_PLACEHOLDER = 'UnknownQueue'.freeze
+
+      attr_reader :logger
+
+      def name
+        :que
+      end
+
+      def present?
+        if defined?(::Que) && File.basename($PROGRAM_NAME).start_with?('que')
+          # 0.x releases used "Version" constant.
+          # 1.x releases used "VERSION" constant.
+          return defined?(::Que::Version)
+        end
+      end
+
+      def forking?
+        false
+      end
+
+      def install
+        install_tracer
+        install_worker
+        install_job
+      end
+
+      def install_tracer
+        ::Que::Job.class_eval do
+          include ScoutApm::Tracer
+        end
+      end
+
+      def install_worker
+        ::Que::Worker.class_eval do
+          def initialize_with_scout(*args)
+            agent = ::ScoutApm::Agent.instance
+            agent.start
+            initialize_without_scout(*args)
+          end
+
+          alias_method :initialize_without_scout, :initialize
+          alias_method :initialize, :initialize_with_scout
+        end
+      end
+
+      def install_job
+        ::Que::Job.class_eval do
+          # attrs = {
+          #   "queue"=>"",
+          #   "priority"=>100,
+          #   "run_at"=>2018-12-19 15:12:32 -0700,
+          #   "job_id"=>4,
+          #   "job_class"=>"ExampleJob",
+          #   "args"=>[{"key"=>"value"}],
+          #   "error_count"=>0
+          # }
+          #
+          # With ActiveJob, v0.14.3:
+          # attrs = {
+          #   "queue"=>"",
+          #   "priority"=>100,
+          #   "run_at"=>2018-12-19 15:29:18 -0700,
+          #   "job_id"=>6,
+          #   "job_class"=>"ActiveJob::QueueAdapters::QueAdapter::JobWrapper",
+          #   "args"=> [{
+          #     "job_class"=>"ExampleJob",
+          #     "job_id"=>"60798b45-4b55-436e-806d-693939266c97",
+          #     "provider_job_id"=>nil,
+          #     "queue_name"=>"default",
+          #     "priority"=>nil,
+          #     "arguments"=>[],
+          #     "executions"=>0,
+          #     "locale"=>"en"
+          #   }],
+          #   "error_count"=>0
+          # }
+          #
+          # With ActiveJob, v1.0:
+          #   There are changes here to make Que more compatible with ActiveJob
+          #   and this probably needs to be revisited.
+
+          def _run_with_scout(*args)
+            # default queue unless specifed is a blank string
+            queue = attrs['queue'] || UNKNOWN_QUEUE_PLACEHOLDER
+
+            job_class = begin
+                          if self.class == ActiveJob::QueueAdapters::QueAdapter::JobWrapper
+                            Array(attrs['args']).first['job_class'] || UNKNOWN_CLASS_PLACEHOLDER
+                          else
+                            self.class.name
+                          end
+                        rescue => e
+                          UNKNOWN_CLASS_PLACEHOLDER
+                        end
+
+            latency = begin
+                        Time.now - attrs['run_at']
+                      rescue
+                        0
+                      end
+
+            req = ScoutApm::RequestManager.lookup
+            req.annotate_request(:queue_latency => latency)
+
+            begin
+              req.start_layer(ScoutApm::Layer.new('Queue', queue))
+              started_queue = true
+              req.start_layer(ScoutApm::Layer.new('Job', job_class))
+              started_job = true
+
+              _run_without_scout(*args)
+            rescue Exception => e
+              req.error!
+              raise
+            ensure
+              req.stop_layer if started_job
+              req.stop_layer if started_queue
+            end
+          end
+
+          alias_method :_run_without_scout, :_run
+          alias_method :_run, :_run_with_scout
+        end
+      end
+    end
+
+  end
+end

data/lib/scout_apm/background_job_integrations/resque.rb

@@ -26,8 +26,12 @@ module ScoutApm
       def install_before_fork
         ::Resque.before_first_fork do
           begin
-            ScoutApm::Agent.instance.start
-            ScoutApm::Agent.instance.context.start_remote_server!(bind, port)
+            if ScoutApm::Agent.instance.context.config.value('start_resque_server_instrument')
+              ScoutApm::Agent.instance.start
+              ScoutApm::Agent.instance.context.start_remote_server!(bind, port)
+            else
+              logger.info("Not starting remote server due to 'start_resque_server_instrument' setting")
+            end
           rescue Errno::EADDRINUSE
             ScoutApm::Agent.instance.context.logger.warn "Error while Installing Resque Instruments, Port #{port} already in use. Set via the `remote_agent_port` configuration option"
           rescue => e

data/lib/scout_apm/background_job_integrations/shoryuken.rb

@@ -0,0 +1,124 @@
+module ScoutApm
+  module BackgroundJobIntegrations
+    class Shoryuken
+      attr_reader :logger
+
+      def name
+        :shoryuken
+      end
+
+      def present?
+        defined?(::Shoryuken) && File.basename($PROGRAM_NAME).start_with?('shoryuken')
+      end
+
+      def forking?
+        false
+      end
+
+      def install
+        install_tracer
+        add_middleware
+        install_processor
+      end
+
+      def install_tracer
+        # ScoutApm::Tracer is not available when this class is defined
+        ShoryukenMiddleware.class_eval do
+          include ScoutApm::Tracer
+        end
+      end
+
+      def add_middleware
+        ::Shoryuken.configure_server do |config|
+          config.server_middleware do |chain|
+            chain.add ShoryukenMiddleware
+          end
+        end
+      end
+
+      def install_processor
+        require 'shoryuken/processor' # sidekiq v4 has not loaded this file by this point
+
+        ::Shoryuken::Processor.class_eval do
+          def initialize_with_scout(*args)
+            agent = ::ScoutApm::Agent.instance
+            agent.start
+            initialize_without_scout(*args)
+          end
+
+          alias_method :initialize_without_scout, :initialize
+          alias_method :initialize, :initialize_with_scout
+        end
+      end
+    end
+
+    # We insert this middleware into the Shoryuken stack, to capture each job,
+    # and time them.
+    class ShoryukenMiddleware
+      def call(worker_instance, queue, msg, body)
+        job_class =
+          begin
+            if worker_instance.class.to_s == ACTIVE_JOB_KLASS
+              body["job_class"]
+            else
+              worker_instance.class.to_s
+            end
+          rescue
+            UNKNOWN_CLASS_PLACEHOLDER
+          end
+
+        req = ScoutApm::RequestManager.lookup
+        req.annotate_request(:queue_latency => latency(msg))
+
+        begin
+          req.start_layer(ScoutApm::Layer.new('Queue', queue))
+          started_queue = true
+          req.start_layer(ScoutApm::Layer.new('Job', job_class))
+          started_job = true
+
+          yield
+        rescue Exception => e
+          req.error!
+          raise
+        ensure
+          req.stop_layer if started_job
+          req.stop_layer if started_queue
+        end
+      end
+
+      UNKNOWN_CLASS_PLACEHOLDER = 'UnknownJob'.freeze
+      ACTIVE_JOB_KLASS = 'ActiveJob::QueueAdapters::ShoryukenAdapter::JobWrapper'.freeze
+
+      def latency(msg, time = Time.now.to_f)
+        ms_since_epoch_str = msg.attributes.fetch('SentTimestamp', 0)
+        return 0 if ms_since_epoch_str.nil?
+
+        # Convert from ms to seconds as a float
+        created_at = ms_since_epoch_str.to_i / 1000.0
+
+        time - created_at
+      rescue => e
+        0
+      end
+    end
+  end
+end
+
+# MSG: #<Shoryuken::Message:0x007fb742a96950 @client=#<Aws::SQS::Client>
+# @data=#<struct Aws::SQS::Types::Message message_id="7a2ef0af-2bbd-4368-9c39-34cc89e4da15"
+# receipt_handle="AQEB8YK4+TCyvCM3p0EanmhZiTbBCM6uMeyCn7zibNn+XZcMnjZp2Z8D8yoUs4mMX9vJqlQvaS8gRUGBYG7ciq+BthmEqDWfxbcJ8jN+Vp/PXIyyTgYL3vvlnHcQajDz3H7Bd30UmLu80sqeLSjXXNEiKolcIxdGuIsIdSM4aUEPXsecr5eH7o8pZHcDV+bGcLuE7VbvKRZT3A2HeezW7wWwkYve/wt6asS1bYB+VJurAORY0y26xgCooEMNbs5yqxcnSD/CiNT822hkmw0eHNpTHOjF9WLgxLbkpITnQl1lsfK5TsM/ukE1oB1F9nN5ZkCBVCDeFYJDBAo81VvVV9G16knxyCYzjnmpwhvHg2BqTA56iV6r9KZYbiwOaMPdH5ealKLRnWhFoLOEPNA4yjG1yw=="
+# md5_of_body="8b3be018857a74f9e46b4c6ef3c3f515"
+# body="Unique Person #7532"
+# attributes={"SenderId"=>"AIDAJZXBVF26MLZPE6FOO"
+# "ApproximateFirstReceiveTimestamp"=>"1534873932213"
+# "ApproximateReceiveCount"=>"1"
+# "SentTimestamp"=>"1534873927868"}
+# md5_of_message_attributes="c70f52a6566cf42ec5e61e81877132dd"
+# message_attributes={"shoryuken_class"=>#<struct Aws::SQS::Types::MessageAttributeValue string_value="DummyWorker"
+    # binary_value=nil
+    # string_list_values=[]
+    # binary_list_values=[]
+    # data_type="String">}>
+# @queue_url="https://sqs.us-west-2.amazonaws.com/023109228371/shoryuken_test"
+# @queue_name="shoryuken_test">
+

data/lib/scout_apm/background_job_integrations/sidekiq.rb

@@ -40,10 +40,10 @@ module ScoutApm
         require 'sidekiq/processor' # sidekiq v4 has not loaded this file by this point
 
         ::Sidekiq::Processor.class_eval do
-          def initialize_with_scout(boss)
+          def initialize_with_scout(*args)
             agent = ::ScoutApm::Agent.instance
             agent.start
-            initialize_without_scout(boss)
+            initialize_without_scout(*args)
           end
 
           alias_method :initialize_without_scout, :initialize
@@ -59,20 +59,10 @@ module ScoutApm
         req = ScoutApm::RequestManager.lookup
         req.annotate_request(:queue_latency => latency(msg))
 
-        queue_layer = ScoutApm::Layer.new('Queue', queue)
-        job_layer = ScoutApm::Layer.new('Job', job_class(msg))
-
-        if ScoutApm::Agent.instance.context.config.value('profile') && SidekiqMiddleware.version_supports_profiling?
-          # Capture ScoutProf if we can
-          #req.enable_profiled_thread!
-          #job_layer.set_root_class(job_class)
-          #job_layer.traced!
-        end
-
         begin
-          req.start_layer(queue_layer)
+          req.start_layer(ScoutApm::Layer.new('Queue', queue))
           started_queue = true
-          req.start_layer(job_layer)
+          req.start_layer(ScoutApm::Layer.new('Job', job_class(msg)))
           started_job = true
 
           yield
@@ -131,10 +121,6 @@ module ScoutApm
       rescue
         0
       end
-
-      def self.version_supports_profiling?
-        @@sidekiq_supports_profling ||= defined?(::Sidekiq::VERSION) && Gem::Dependency.new('', '~> 4.0').match?('', ::Sidekiq::VERSION.to_s)
-      end
-    end # SidekiqMiddleware
+    end
   end
 end

data/lib/scout_apm/background_job_integrations/sneakers.rb

@@ -0,0 +1,87 @@
+module ScoutApm
+  module BackgroundJobIntegrations
+    class Sneakers
+      def name
+        :sneakers
+      end
+
+      def present?
+        defined?(::Sneakers) && supported_version?
+      end
+      
+      # Only support Sneakers 2.7 and up
+      def supported_version?
+        result = Gem::Version.new(::Sneakers::VERSION) > Gem::Version.new("2.7.0")
+        ScoutApm::Agent.instance.logger.info("Skipping Sneakers instrumentation. Only versions 2.7+ are supported. See docs or contact support@scoutapm.com for instrumentation of older versions.")
+        result
+      rescue
+        ScoutApm::Agent.instance.logger.info("Failed comparing Sneakers Version. Skipping")
+        false
+      end
+
+      def forking?
+        false
+      end
+
+      def install
+        install_worker_override
+      end
+
+      def install_worker_override
+        ::Sneakers::Worker.module_eval do
+          def initialize_with_scout(*args)
+            agent = ::ScoutApm::Agent.instance
+            agent.start
+            initialize_without_scout(*args)
+          end
+
+          alias_method :initialize_without_scout, :initialize
+          alias_method :initialize, :initialize_with_scout
+
+          def process_work_with_scout(*args)
+            delivery_info, _metadata, msg, _handler = args
+
+            queue = delivery_info[:routing_key] || UNKNOWN_QUEUE_PLACEHOLDER
+
+            job_class = begin
+              if self.class == ActiveJob::QueueAdapters::SneakersAdapter::JobWrapper
+                msg["job_class"] || UNKNOWN_CLASS_PLACEHOLDER
+              else
+                self.class.name
+              end
+            rescue => e
+              UNKNOWN_CLASS_PLACEHOLDER
+            end
+
+            req = ScoutApm::RequestManager.lookup
+
+            # RabbitMQ does not store a created-at timestamp
+            # req.annotate_request(:queue_latency => latency(msg))
+
+            begin
+              req.start_layer(ScoutApm::Layer.new('Queue', queue))
+              started_queue = true
+              req.start_layer(ScoutApm::Layer.new('Job', job_class))
+              started_job = true
+
+              process_work_without_scout(*args)
+            rescue Exception => e
+              req.error!
+              raise
+            ensure
+              req.stop_layer if started_job
+              req.stop_layer if started_queue
+            end
+          end
+
+          alias_method :process_work_without_scout, :process_work
+          alias_method :process_work, :process_work_with_scout
+        end
+      end
+
+      ACTIVE_JOB_KLASS = 'ActiveJob::QueueAdapters::SneakersAdapter::JobWrapper'.freeze
+      UNKNOWN_CLASS_PLACEHOLDER = 'UnknownJob'.freeze
+      UNKNOWN_QUEUE_PLACEHOLDER = 'default'.freeze
+    end
+  end
+end

data/lib/scout_apm/config.rb

@@ -6,10 +6,11 @@ require 'scout_apm/environment'
 # Valid Config Options:
 #
 # This list is complete, but some are old and unused, or for developers of
-# scout_apm itself. See the documentation at http://help.apm.scoutapp.com for
+# scout_apm itself. See the documentation at https://docs.scoutapm.com for
 # customer-focused documentation.
 #
 # application_root - override the detected directory of the application
+# collect_remote_ip - automatically capture user's IP into a Trace's Context
 # compress_payload - true/false to enable gzipping of payload
 # data_file        - override the default temporary storage location. Must be a location in a writable directory
 # dev_trace        - true or false. Enables always-on tracing in development environmen only
@@ -30,6 +31,10 @@ require 'scout_apm/environment'
 # uri_reporting    - 'path' or 'full_path' default is 'full_path', which reports URL params as well as the path.
 # remote_agent_host - Internal: What host to bind to, and also send messages to for remote. Default: 127.0.0.1.
 # remote_agent_port - What port to bind the remote webserver to
+# start_resque_server_instrument - Used in special situations with certain Resque installs
+# timeline_traces - true/false to enable sending of of the timeline trace format.
+# auto_instruments - true/false whether to install autoinstruments. Only installed if on a supported Ruby version.
+# auto_instruments_ignore - An array of file names to exclude from autoinstruments (Ex: ['application_controller']).
 #
 # Any of these config settings can be set with an environment variable prefixed
 # by SCOUT_ and uppercasing the key: SCOUT_LOG_LEVEL for instance.
@@ -39,6 +44,7 @@ module ScoutApm
     KNOWN_CONFIG_OPTIONS = [
         'application_root',
         'async_recording',
+        'collect_remote_ip',
         'compress_payload',
         'config_file',
         'data_file',
@@ -66,9 +72,21 @@ module ScoutApm
         'remote_agent_host',
         'remote_agent_port',
         'report_format',
+        'revision_sha',
         'scm_subdirectory',
+        'start_resque_server_instrument',
+        'ssl_cert_file',
         'uri_reporting',
         'instrument_http_url_length',
+        'timeline_traces',
+        'auto_instruments',
+        'auto_instruments_ignore',
+
+        # Error Service Related Configuration
+        'errors_enabled',
+        'errors_ignored_exceptions',
+        'errors_filtered_params',
+        'errors_host',
     ]
 
     ################################################################################
@@ -148,16 +166,25 @@ module ScoutApm
 
 
     SETTING_COERCIONS = {
-      "async_recording"        => BooleanCoercion.new,
-      "detailed_middleware"    => BooleanCoercion.new,
-      "dev_trace"              => BooleanCoercion.new,
-      "enable_background_jobs" => BooleanCoercion.new,
-      "ignore"                 => JsonCoercion.new,
-      "max_traces"             => IntegerCoercion.new,
-      "monitor"                => BooleanCoercion.new,
+      'async_recording' => BooleanCoercion.new,
+      'detailed_middleware' => BooleanCoercion.new,
+      'dev_trace' => BooleanCoercion.new,
+      'enable_background_jobs' => BooleanCoercion.new,
+      'ignore' => JsonCoercion.new,
+      'max_traces' => IntegerCoercion.new,
+      'monitor' => BooleanCoercion.new,
+      'collect_remote_ip' => BooleanCoercion.new,
+      'compress_payload' => BooleanCoercion.new,
       'database_metric_limit'  => IntegerCoercion.new,
       'database_metric_report_limit' => IntegerCoercion.new,
       'instrument_http_url_length' => IntegerCoercion.new,
+      'start_resque_server_instrument' => BooleanCoercion.new,
+      'timeline_traces' => BooleanCoercion.new,
+      'auto_instruments' => BooleanCoercion.new,
+      'auto_instruments_ignore' => JsonCoercion.new,
+      'errors_enabled' => BooleanCoercion.new,
+      'errors_ignored_exceptions' => JsonCoercion.new,
+      'errors_filtered_params' => JsonCoercion.new,
     }
 
 
@@ -263,6 +290,16 @@ module ScoutApm
         'database_metric_limit'  => 5000, # The hard limit on db metrics
         'database_metric_report_limit' => 1000,
         'instrument_http_url_length' => 300,
+        'start_resque_server_instrument' => true, # still only starts if Resque is detected
+        'collect_remote_ip' => true,
+        'timeline_traces' => true,
+        'auto_instruments' => false,
+        'auto_instruments_ignore' => [],
+        'ssl_cert_file' => File.join( File.dirname(__FILE__), *%w[.. .. data cacert.pem] ),
+        'errors_enabled' => false,
+        'errors_ignored_exceptions' => %w(ActiveRecord::RecordNotFound ActionController::RoutingError),
+        'errors_filtered_params' => %w(password s3-key),
+        'errors_host' => 'https://errors.scoutapm.com',
       }.freeze
 
       def value(key)