scout_apm 2.2.0.pre3 → 2.3.0.pre

data/lib/scout_apm/agent.rb

@@ -10,9 +10,9 @@ module ScoutApm
 
     # Accessors below are for associated classes
     attr_accessor :store
+    attr_reader :recorder
     attr_accessor :layaway
     attr_accessor :config
-    attr_accessor :capacity
     attr_accessor :logger
     attr_accessor :log_file # path to the log file
     attr_accessor :options # options passed to the agent when +#start+ is called.
@@ -42,6 +42,9 @@ module ScoutApm
       @process_start_time = Time.now
       @options ||= options
 
+      # until the agent is started, there's no recorder
+      @recorder = nil
+
       # Start up without attempting to load a configuration file. We need to be
       # able to lookup configuration options like "application_root" which would
       # then in turn influence where the configuration file came from.
@@ -55,10 +58,10 @@ module ScoutApm
       @request_histograms_by_time = Hash.new { |h, k| h[k] = ScoutApm::RequestHistograms.new }
 
       @store          = ScoutApm::Store.new
+
       @layaway        = ScoutApm::Layaway.new(config, environment)
       @metric_lookup  = Hash.new
 
-      @capacity       = ScoutApm::Capacity.new
       @installed_instruments = []
     end
 
@@ -86,6 +89,11 @@ module ScoutApm
         return false unless force?
       end
 
+      if environment.interactive?
+        logger.warn "Agent attempting to load in interactive mode. #{force? ? 'Forcing agent to start' : 'Not starting agent'}"
+        return false unless force?
+      end
+
       if app_server_missing?(options) && background_job_missing?
         if force?
           logger.warn "Agent starting (forced)"
@@ -119,22 +127,27 @@ module ScoutApm
       init_logger
       logger.info "Attempting to start Scout Agent [#{ScoutApm::VERSION}] on [#{environment.hostname}]"
 
-      @ignored_uris = ScoutApm::IgnoredUris.new(config.value('ignore'))
+      @recorder = create_recorder
 
-      if environment.deploy_integration
-        logger.info "Starting monitoring for [#{environment.deploy_integration.name}]]."
-        return environment.deploy_integration.install
-      end
+      @config.log_settings
+
+      @ignored_uris = ScoutApm::IgnoredUris.new(config.value('ignore'))
 
       load_instruments if should_load_instruments?(options)
 
+      if !@config.any_keys_found?
+        logger.info("No configuration file loaded, and no configuration found in ENV. " +
+                    "For assistance configuring Scout, visit " +
+                    "http://help.apm.scoutapp.com/#configuration-options")
+      end
+
       return false unless preconditions_met?(options)
       @started = true
       logger.info "Starting monitoring for [#{environment.application_name}]. Framework [#{environment.framework}] App Server [#{environment.app_server}] Background Job Framework [#{environment.background_job_name}]."
 
       [ ScoutApm::Instruments::Process::ProcessCpu.new(environment.processors, logger),
         ScoutApm::Instruments::Process::ProcessMemory.new(logger),
-        ScoutApm::Instruments::PercentileSampler.new(logger, 95),
+        ScoutApm::Instruments::PercentileSampler.new(logger, request_histograms_by_time),
       ].each { |s| store.add_sampler(s) }
 
       app_server_load_hook
@@ -202,20 +215,21 @@ module ScoutApm
     # It does not attempt to actually report metrics.
     def shutdown
       logger.info "Shutting down ScoutApm"
+
       return if !started?
 
+      return if @shutdown
+      @shutdown = true
+
       if @background_worker
         logger.info("Stopping background worker")
         @background_worker.stop
         store.write_to_layaway(layaway, :force)
+        if @background_worker_thread.alive?
+          @background_worker_thread.wakeup
+          @background_worker_thread.join
+        end
       end
-
-      logger.debug "Joining background worker thread"
-      if @background_worker_thread
-        @background_worker_thread.wakeup
-        @background_worker_thread.join
-      end
-      ScoutApm::Instruments::Stacks.uninstall
     end
 
     def started?
@@ -234,7 +248,10 @@ module ScoutApm
     end
 
     def background_worker_running?
-      !! @background_worker_thread
+      @background_worker_thread          &&
+        @background_worker_thread.alive? &&
+        @background_worker               &&
+        @background_worker.running?
     end
 
     # Creates the worker thread. The worker thread is a loop that runs continuously. It sleeps for +Agent#period+ and when it wakes,
@@ -249,15 +266,13 @@ module ScoutApm
 
       install_exit_handler
 
-      if ScoutApm::Agent.instance.config.value('profile')
-        # After we fork, setup the handlers here.
-        ScoutApm::Instruments::Stacks.install
-        ScoutApm::Instruments::Stacks.start
-      end
+      @recorder = create_recorder
+      logger.info("recorder is now: #{@recorder.class}")
 
       @background_worker = ScoutApm::BackgroundWorker.new
       @background_worker_thread = Thread.new do
         @background_worker.start {
+          ScoutApm::Debug.instance.call_periodic_hooks
           ScoutApm::Agent.instance.process_metrics
           clean_old_percentiles
         }
@@ -281,22 +296,21 @@ module ScoutApm
 
     # Loads the instrumention logic.
     def load_instruments
-      if !background_job_missing?
-        case environment.background_job_name
-        when :delayed_job
-          install_instrument(ScoutApm::Instruments::DelayedJob)
-        end
-      else
-        case environment.framework
-        when :rails       then install_instrument(ScoutApm::Instruments::ActionControllerRails2)
-        when :rails3_or_4 then
-          install_instrument(ScoutApm::Instruments::ActionControllerRails3Rails4)
+      case environment.framework
+      when :rails then
+        install_instrument(ScoutApm::Instruments::ActionControllerRails2)
+      when :rails3_or_4 then
+        install_instrument(ScoutApm::Instruments::ActionControllerRails3Rails4)
+        install_instrument(ScoutApm::Instruments::RailsRouter)
+
+        if config.value("detailed_middleware")
+          install_instrument(ScoutApm::Instruments::MiddlewareDetailed)
+        else
           install_instrument(ScoutApm::Instruments::MiddlewareSummary)
-          install_instrument(ScoutApm::Instruments::RailsRouter)
-        # when :sinatra     then install_instrument(ScoutApm::Instruments::Sinatra)
         end
       end
 
+      install_instrument(ScoutApm::Instruments::ActionView)
       install_instrument(ScoutApm::Instruments::ActiveRecord)
       install_instrument(ScoutApm::Instruments::Moped)
       install_instrument(ScoutApm::Instruments::Mongoid)
@@ -328,10 +342,6 @@ module ScoutApm
       instance.install
     end
 
-    def deploy_integration
-      environment.deploy_integration
-    end
-
     def app_server_missing?(options = {})
       !environment.app_server_integration(true).found? && !options[:skip_app_server_check]
     end
@@ -339,5 +349,47 @@ module ScoutApm
     def background_job_missing?(options = {})
       environment.background_job_integration.nil? && !options[:skip_background_job_check]
     end
+
+    def clear_recorder
+      @recorder = nil
+    end
+
+    def create_recorder
+      if @recorder
+        return @recorder
+      end
+
+      if config.value("async_recording")
+        logger.debug("Using asynchronous recording")
+        ScoutApm::BackgroundRecorder.new(logger).start
+      else
+        logger.debug("Using synchronous recording")
+        ScoutApm::SynchronousRecorder.new(logger).start
+      end
+    end
+
+    def start_remote_server(bind, port)
+      return if @remote_server && @remote_server.running?
+
+      logger.info("Starting Remote Agent Server")
+
+      # Start the listening web server only in parent process.
+      @remote_server = ScoutApm::Remote::Server.new(
+        bind,
+        port,
+        ScoutApm::Remote::Router.new(ScoutApm::SynchronousRecorder.new(logger), logger),
+        logger
+      )
+
+      @remote_server.start
+    end
+
+    # Execute this in the child process of a remote agent. The parent is
+    # expected to have its accepting webserver up and running
+    def use_remote_recorder(host, port)
+      logger.debug("Becoming Remote Agent (reporting to: #{host}:#{port})")
+      @recorder = ScoutApm::Remote::Recorder.new(host, port, logger)
+      @store = ScoutApm::FakeStore.new
+    end
   end
 end

data/lib/scout_apm/agent/logging.rb

@@ -6,7 +6,12 @@ module ScoutApm
         "#{environment.root}/log"
       end
 
-      def init_logger
+      def init_logger(opts={})
+        if opts[:force]
+          @log_file = nil
+          @logger = nil
+        end
+
         begin
           @log_file ||= determine_log_destination
         rescue => e

data/lib/scout_apm/agent/reporting.rb

@@ -24,7 +24,7 @@ module ScoutApm
         report_to_server
       end
 
-      # In a running app, one process will get one period ready for delivery, the others will see 0.
+      # In a running app, one process will get the period ready for delivery, the others will see 0.
       def report_to_server
         period_to_report = ScoutApm::StoreReportingPeriodTimestamp.minutes_ago(2)
 
@@ -58,6 +58,8 @@ module ScoutApm
         slow_transactions = reporting_period.slow_transactions_payload
         jobs = reporting_period.jobs
         slow_jobs = reporting_period.slow_jobs_payload
+        histograms = reporting_period.histograms
+        db_query_metrics = reporting_period.db_query_metrics_payload
 
         metadata = {
           :app_root      => ScoutApm::Environment.instance.root.to_s,
@@ -68,10 +70,10 @@ module ScoutApm
           :platform      => "ruby",
         }
 
-        log_deliver(metrics, slow_transactions, metadata, slow_jobs)
+        log_deliver(metrics, slow_transactions, metadata, slow_jobs, histograms)
 
-        payload = ScoutApm::Serializers::PayloadSerializer.serialize(metadata, metrics, slow_transactions, jobs, slow_jobs)
-        # logger.debug("Payload: #{payload}")
+        payload = ScoutApm::Serializers::PayloadSerializer.serialize(metadata, metrics, slow_transactions, jobs, slow_jobs, histograms, db_query_metrics)
+        logger.debug("Sending payload w/ Headers: #{headers.inspect}")
 
         reporter.report(payload, headers)
       rescue => e
@@ -80,7 +82,7 @@ module ScoutApm
         logger.debug e.backtrace
       end
 
-      def log_deliver(metrics, slow_transactions, metadata, jobs_traces)
+      def log_deliver(metrics, slow_transactions, metadata, jobs_traces, histograms)
         total_request_count = metrics.
           select { |meta,stats| meta.metric_name =~ /\AController/ }.
           inject(0) {|sum, (_, stat)| sum + stat.call_count }
@@ -97,9 +99,10 @@ module ScoutApm
         metrics_clause    = "#{metrics.length} Metrics for #{total_request_count} requests"
         slow_trans_clause = "#{slow_transactions.length} Slow Transaction Traces"
         job_clause        = "#{jobs_traces.length} Job Traces"
+        histogram_clause  = "#{histograms.length} Histograms"
 
         logger.info "#{time_clause} Delivering #{metrics_clause} and #{slow_trans_clause} and #{job_clause}, #{process_log_str}."
-        # logger.debug("Metrics: #{metrics.pretty_inspect}\nSlowTrans: #{slow_transactions.pretty_inspect}\nMetadata: #{metadata.inspect.pretty_inspect}")
+        logger.debug("\n\nMetrics: #{metrics.pretty_inspect}\nSlowTrans: #{slow_transactions.pretty_inspect}\nMetadata: #{metadata.inspect.pretty_inspect}\n\n")
       end
 
       # TODO: Move this into PayloadSerializer?

data/lib/scout_apm/app_server_load.rb

@@ -27,19 +27,30 @@ module ScoutApm
     end
 
     def data
-      { :server_time        => Time.now,
-        :framework          => ScoutApm::Environment.instance.framework_integration.human_name,
-        :framework_version  => ScoutApm::Environment.instance.framework_integration.version,
-        :environment        => ScoutApm::Environment.instance.framework_integration.env,
-        :app_server         => ScoutApm::Environment.instance.app_server,
+      { :server_time        => to_s_safe(Time.now),
+        :framework          => to_s_safe(ScoutApm::Environment.instance.framework_integration.human_name),
+        :framework_version  => to_s_safe(ScoutApm::Environment.instance.framework_integration.version),
+        :environment        => to_s_safe(ScoutApm::Environment.instance.framework_integration.env),
+        :app_server         => to_s_safe(ScoutApm::Environment.instance.app_server),
         :ruby_version       => RUBY_VERSION,
-        :hostname           => ScoutApm::Environment.instance.hostname,
-        :database_engine    => ScoutApm::Environment.instance.database_engine,      # Detected
-        :database_adapter   => ScoutApm::Environment.instance.raw_database_adapter, # Raw
-        :application_name   => ScoutApm::Environment.instance.application_name,
+        :hostname           => to_s_safe(ScoutApm::Environment.instance.hostname),
+        :database_engine    => to_s_safe(ScoutApm::Environment.instance.database_engine),      # Detected
+        :database_adapter   => to_s_safe(ScoutApm::Environment.instance.raw_database_adapter), # Raw
+        :application_name   => to_s_safe(ScoutApm::Environment.instance.application_name),
         :libraries          => ScoutApm::Utils::InstalledGems.new.run,
-        :paas               => ScoutApm::Environment.instance.platform_integration.name
+        :paas               => to_s_safe(ScoutApm::Environment.instance.platform_integration.name),
+        :git_sha            => to_s_safe(ScoutApm::Environment.instance.git_revision.sha)
       }
     end
+
+    # Calls `.to_s` on the object passed in.
+    # Returns literal string 'to_s error' if the object does not respond to .to_s
+    def to_s_safe(obj)
+      if obj.respond_to?(:to_s)
+        obj.to_s
+      else
+        'to_s error'
+      end
+    end
   end
 end

data/lib/scout_apm/attribute_arranger.rb

@@ -5,8 +5,6 @@ module ScoutApm
     def self.call(subject, attributes_list)
       attributes_list.inject({}) do |attribute_hash, attribute|
         case attribute
-        when :traces
-          attribute_hash[attribute] = subject.traces.to_a
         when Array
           attribute_hash[attribute[0]] = subject.send(attribute[1])
         when :bucket
@@ -14,7 +12,12 @@ module ScoutApm
         when :name
           attribute_hash[attribute] = subject.bucket_name
         when Symbol
-          attribute_hash[attribute] = subject.send(attribute)
+          data = subject.send(attribute)
+          if data.respond_to?(:as_json)
+            attribute_hash[attribute] = data.as_json
+          else
+            attribute_hash[attribute] = data
+          end
         end
         attribute_hash
       end

data/lib/scout_apm/background_job_integrations/delayed_job.rb

@@ -1,6 +1,9 @@
 module ScoutApm
   module BackgroundJobIntegrations
     class DelayedJob
+      ACTIVE_JOB_KLASS = 'ActiveJob::QueueAdapters::DelayedJobAdapter::JobWrapper'.freeze
+      DJ_PERFORMABLE_METHOD = 'Delayed::PerformableMethod'.freeze
+
       attr_reader :logger
 
       def name
@@ -8,12 +11,79 @@ module ScoutApm
       end
 
       def present?
-        defined?(::Delayed::Job) && (File.basename($0) =~ /\Adelayed_job/)
+        defined?(::Delayed::Job)
       end
 
       def forking?
         false
       end
+
+      def install
+        plugin = Class.new(Delayed::Plugin) do
+          require 'delayed_job'
+
+          callbacks do |lifecycle|
+            lifecycle.around(:invoke_job) do |job, *args, &block|
+              ScoutApm::Agent.instance.start_background_worker unless ScoutApm::Agent.instance.background_worker_running?
+
+              name = begin
+                       case job.payload_object.class.to_s
+
+                       # ActiveJob's class wraps the actual job class
+                       when ACTIVE_JOB_KLASS
+                         job.payload_object.job_data["job_class"]
+
+                       # An adhoc job, called like `@user.delay.fib(10)`.
+                       # returns a string like "User#fib"
+                       when DJ_PERFORMABLE_METHOD
+                         job.name
+
+                       # A "real" job called like `Delayed::Job.enqueue(MyJob.new)`
+                       # returns "MyJob"
+                       else
+                         job.payload_object.class.to_s
+                       end
+                     rescue
+                       # Fall back to whatever DJ thinks the name is.
+                       job.name
+                     end
+
+              queue = job.queue || "default"
+
+              req = ScoutApm::RequestManager.lookup
+              req.job!
+
+              begin
+                latency = Time.now - job.created_at
+                req.annotate_request(:queue_latency => latency)
+              rescue
+              end
+
+              queue_layer = ScoutApm::Layer.new('Queue', queue)
+              job_layer = ScoutApm::Layer.new('Job', name)
+
+              begin
+                req.start_layer(queue_layer)
+                started_queue = true
+                req.start_layer(job_layer)
+                started_job = true
+
+                # Call the job itself.
+                block.call(job, *args)
+              rescue
+                req.error!
+                raise
+              ensure
+                req.stop_layer if started_job
+                req.stop_layer if started_queue
+              end
+            end
+          end
+        end
+
+        Delayed::Worker.plugins << plugin # ScoutApm::BackgroundJobIntegrations::DelayedJobPlugin
+      end
     end
   end
 end
+