scout_apm 2.6.10 → 3.0.0.pre0

data/lib/scout_apm/agent.rb

@@ -1,127 +1,227 @@
 module ScoutApm
-  # The entry-point for the ScoutApm Agent.
+  # The agent gathers performance data from a Ruby application. One Agent instance is created per-Ruby process.
   #
-  # Only one Agent instance is created per-Ruby process, and it coordinates the lifecycle of the monitoring.
-  #   - initializes various data stores
-  #   - coordinates configuration & logging
-  #   - starts background threads, running periodically
-  #   - installs shutdown hooks
+  # Each Agent object creates a worker thread (unless monitoring is disabled or we're forking).
+  # The worker thread wakes up every +Agent#period+, merges in-memory metrics w/those saved to disk,
+  # saves tshe merged data to disk, and sends it to the Scout server.
   class Agent
     # see self.instance
     @@instance = nil
 
-    attr_reader :context
-
+    # Accessors below are for associated classes
+    attr_accessor :store
+    attr_accessor :layaway
+    attr_accessor :config
+    attr_accessor :capacity
+    attr_accessor :logger
+    attr_accessor :log_file # path to the log file
     attr_accessor :options # options passed to the agent when +#start+ is called.
+    attr_accessor :metric_lookup # Hash used to lookup metric ids based on their name and scope
+    attr_reader :slow_request_policy
+    attr_reader :slow_job_policy
+    attr_reader :process_start_time # used when creating slow transactions to report how far from startup the transaction was recorded.
+    attr_reader :ignored_uris
+
+    # Histogram of the cumulative requests since the start of the process
+    attr_reader :request_histograms
 
-    attr_reader :instrument_manager
+    # Histogram of the requests, distinct by reporting period (minute)
+    # { StoreReportingPeriodTimestamp => RequestHistograms }
+    attr_reader :request_histograms_by_time
 
     # All access to the agent is thru this class method to ensure multiple Agent instances are not initialized per-Ruby process.
     def self.instance(options = {})
       @@instance ||= self.new(options)
     end
 
-    # First call of the agent. Does very little so that the object can be created, and exist.
+    # Note - this doesn't start instruments or the worker thread. This is handled via +#start+ as we don't
+    # want to start the worker thread or install instrumentation if (1) disabled for this environment (2) a worker thread shouldn't
+    # be started (when forking).
     def initialize(options = {})
-      @options = options
-      @context = ScoutApm::AgentContext.new
+      @started = false
+      @process_start_time = Time.now
+      @options ||= options
+
+      # Start up without attempting to load a configuration file. We need to be
+      # able to lookup configuration options like "application_root" which would
+      # then in turn influence where the configuration file came from.
+      #
+      # Later in initialization, we reset @config to include the file.
+      @config = ScoutApm::Config.without_file
+
+      @slow_request_policy = ScoutApm::SlowRequestPolicy.new
+      @slow_job_policy = ScoutApm::SlowJobPolicy.new
+      @request_histograms = ScoutApm::RequestHistograms.new
+      @request_histograms_by_time = Hash.new { |h, k| h[k] = ScoutApm::RequestHistograms.new }
+
+      @store          = ScoutApm::Store.new
+      @layaway        = ScoutApm::Layaway.new(config, environment)
+      @metric_lookup  = Hash.new
+
+      @capacity       = ScoutApm::Capacity.new
+      @installed_instruments = []
     end
 
-    def logger
-      context.logger
+    def environment
+      ScoutApm::Environment.instance
     end
 
-    # Finishes setting up the instrumentation, configuration, and attempts to start the agent.
-    def install(force=false)
-      context.config = ScoutApm::Config.with_file(context, context.config.value("config_file"))
+    def apm_enabled?
+      config.value('monitor')
+    end
 
-      logger.info "Scout Agent [#{ScoutApm::VERSION}] Initialized"
+    # If true, the agent will start regardless of safety checks. Currently just used for testing.
+    def force?
+      @options[:force]
+    end
 
-      if should_load_instruments? || force
-        instrument_manager.install!
-        install_background_job_integrations
-        install_app_server_integration
-      else
-        logger.info "Not Loading Instruments"
+    def preconditions_met?(options={})
+      if !apm_enabled?
+        logger.warn "Monitoring isn't enabled for the [#{environment.env}] environment. #{force? ? 'Forcing agent to start' : 'Not starting agent'}"
+        return false unless force?
       end
 
-      logger.info "Scout Agent [#{ScoutApm::VERSION}] Installed"
+      if !environment.application_name
+        logger.warn "An application name could not be determined. Specify the :name value in scout_apm.yml. #{force? ? 'Forcing agent to start' : 'Not starting agent'}."
+        return false unless force?
+      end
 
-      context.installed!
+      if app_server_missing?(options) && background_job_missing?
+        if force?
+          logger.warn "Agent starting (forced)"
+        else
+          logger.warn "Deferring agent start. Standing by for first request"
+        end
+        return false unless force?
+      end
+
+      if started?
+        logger.warn "Already started agent."
+        return false
+      end
 
-      @preconditions = ScoutApm::Agent::Preconditions.new
-      if @preconditions.check?(context) || force
-        start
+      if defined?(::ScoutRails)
+        logger.warn "ScoutAPM is incompatible with the old Scout Rails plugin. Please remove scout_rails from your Gemfile"
+        return false unless force?
       end
+
+      true
     end
 
-    # Unconditionally starts the agent. This includes verifying instruments are
-    # installed, and starting the background worker.
-    #
-    # The monitor precondition is checked explicitly, and we will *never* start with monitor = false
-    #
-    # This does not attempt to start twice
-    def start(opts={})
-      return unless context.config.value('monitor')
+    # This is called via +ScoutApm::Agent.instance.start+ when ScoutApm is required in a Ruby application.
+    # It initializes the agent and starts the worker thread (if appropiate).
+    def start(options = {})
+      @options.merge!(options)
 
-      if context.started?
-        start_background_worker unless background_worker_running?
-        start_error_service_background_worker unless error_service_background_worker_running?
-        return
+      @config = ScoutApm::Config.with_file(@config.value("config_file"))
+      layaway.config = config
+
+      init_logger
+      logger.info "Attempting to start Scout Agent [#{ScoutApm::VERSION}] on [#{environment.hostname}]"
+
+      @ignored_uris = ScoutApm::IgnoredUris.new(config.value('ignore'))
+
+      if environment.deploy_integration
+        logger.info "Starting monitoring for [#{environment.deploy_integration.name}]]."
+        return environment.deploy_integration.install
       end
 
-      install unless context.installed?
+      load_instruments if should_load_instruments?(options)
 
-      instrument_manager.install! if should_load_instruments?
+      return false unless preconditions_met?(options)
+      @started = true
+      logger.info "Starting monitoring for [#{environment.application_name}]. Framework [#{environment.framework}] App Server [#{environment.app_server}] Background Job Framework [#{environment.background_job_name}]."
 
-      context.started!
+      [ ScoutApm::Instruments::Process::ProcessCpu.new(environment.processors, logger),
+        ScoutApm::Instruments::Process::ProcessMemory.new(logger),
+        ScoutApm::Instruments::PercentileSampler.new(logger, 95),
+      ].each { |s| store.add_sampler(s) }
 
-      log_environment
+      app_server_load_hook
 
-      # Save it into a variable to prevent it from ever running twice
-      @app_server_load ||= AppServerLoad.new(context).run
+      if environment.background_job_integration
+        environment.background_job_integration.install
+        logger.info "Installed Background Job Integration [#{environment.background_job_name}]"
+      end
 
-      start_background_worker
-      start_error_service_background_worker
+      # start_background_worker? is true on non-forking servers, and directly
+      # starts the background worker.  On forking servers, a server-specific
+      # hook is inserted to start the background worker after forking.
+      if start_background_worker?
+        start_background_worker
+        logger.info "Scout Agent [#{ScoutApm::VERSION}] Initialized"
+      else
+        environment.app_server_integration.install
+        logger.info "Scout Agent [#{ScoutApm::VERSION}] loaded in [#{environment.app_server}] master process. Monitoring will start after server forks its workers."
+      end
     end
 
-    def instrument_manager
-      @instrument_manager ||= ScoutApm::InstrumentManager.new(context)
+    # Sends a ping to APM right away, smoothes out onboarding
+    # Collects up any relevant info (framework, app server, system time, ruby version, etc)
+    def app_server_load_hook
+      AppServerLoad.new.run
     end
 
-    def log_environment
-      bg_names = context.environment.background_job_integrations.map{|bg| bg.name }.join(", ")
-
-      logger.info(
-        "Scout Agent [#{ScoutApm::VERSION}] starting for [#{context.environment.application_name}] " +
-        "Framework [#{context.environment.framework}] " +
-        "App Server [#{context.environment.app_server}] " +
-        "Background Job Framework [#{bg_names}] " +
-        "Hostname [#{context.environment.hostname}]"
-      )
+    def exit_handler_supported?
+      if environment.sinatra?
+        logger.debug "Exit handler not supported for Sinatra"
+        false
+      elsif environment.jruby?
+        logger.debug "Exit handler not supported for JRuby"
+        false
+      elsif environment.rubinius?
+        logger.debug "Exit handler not supported for Rubinius"
+        false
+      else
+        true
+      end
     end
 
-    # Attempts to install all background job integrations. This can come up if
-    # an app has both Resque and Sidekiq - we want both to be installed if
-    # possible, it's no harm to have the "wrong" one also installed while running.
-    def install_background_job_integrations
-      context.environment.background_job_integrations.each do |int|
-        int.install
-        logger.info "Installed Background Job Integration [#{int.name}]"
+    # at_exit, calls Agent#shutdown to wrapup metric reporting.
+    def install_exit_handler
+      logger.debug "Shutdown handler not supported" and return unless exit_handler_supported?
+      logger.debug "Installing Shutdown Handler"
+
+      at_exit do
+        logger.info "Shutting down Scout Agent"
+        # MRI 1.9 bug drops exit codes.
+        # http://bugs.ruby-lang.org/issues/5218
+        if environment.ruby_19?
+          status = $!.status if $!.is_a?(SystemExit)
+          shutdown
+          exit status if status
+        else
+          shutdown
+        end
       end
     end
 
-    # This sets up the background worker thread to run at the correct time,
-    # either immediately, or after a fork into the actual unicorn/puma/etc
-    # worker
-    def install_app_server_integration
-      context.environment.app_server_integration.install
-      logger.info "Installed Application Server Integration [#{context.environment.app_server}]."
+    # Called via an at_exit handler, it:
+    # (1) Stops the background worker
+    # (2) Stores metrics locally (forcing current-minute metrics to be written)
+    # It does not attempt to actually report metrics.
+    def shutdown
+      logger.info "Shutting down ScoutApm"
+
+      return if !started?
+
+      return if @shutdown
+      @shutdown = true
+
+      if @background_worker
+        logger.info("Stopping background worker")
+        @background_worker.stop
+        store.write_to_layaway(layaway, :force)
+        if @background_worker_thread.alive?
+          @background_worker_thread.wakeup
+          @background_worker_thread.join
+        end
+      end
+      ScoutApm::Instruments::Stacks.uninstall
     end
 
-    # If true, the agent will start regardless of safety checks.
-    def force?
-      @options[:force]
+    def started?
+      @started
     end
 
     # The worker thread will automatically start UNLESS:
@@ -129,96 +229,117 @@ module ScoutApm
     # * A supported application server is detected, but it forks. In this case,
     #   the agent is started in the forked process.
     def start_background_worker?
+      return true if environment.app_server == :thin
+      return true if environment.app_server == :webrick
       return true if force?
-      return !context.environment.forking?
+      return !environment.forking?
     end
 
-    # monitor is the key configuration here. If it is true, then we want the
-    # instruments. If it is false, we mostly don't want them, unless you're
-    # asking for devtrace (ie. not reporting to apm servers as a real app, but
-    # only for local browsers).
-    def should_load_instruments?
-      return true if context.config.value('dev_trace')
-      context.config.value('monitor')
+    def background_worker_running?
+      !! @background_worker_thread
     end
 
-    #################################
-    #  Background Worker Lifecycle  #
-    #################################
-
     # Creates the worker thread. The worker thread is a loop that runs continuously. It sleeps for +Agent#period+ and when it wakes,
     # processes data, either saving it to disk or reporting to Scout.
-    # => true if thread & worker got started
-    # => false if it wasn't started (either due to already running, or other preconditions)
-    def start_background_worker(quiet=false)
-      if !context.config.value('monitor')
-        logger.debug "Not starting background worker as monitoring isn't enabled." unless quiet
-        return false
-      end
-
-      if background_worker_running?
-        logger.info "Not starting background worker, already started" unless quiet
+    def start_background_worker
+      if !apm_enabled?
+        logger.debug "Not starting background worker as monitoring isn't enabled."
         return false
       end
-
-      if context.shutting_down?
-        logger.info "Not starting background worker, already in process of shutting down" unless quiet
-        return false
-      end
-
+      logger.info "Not starting background worker, already started" and return if background_worker_running?
       logger.info "Initializing worker thread."
 
-      ScoutApm::Agent::ExitHandler.new(context).install
+      install_exit_handler
 
-      periodic_work = ScoutApm::PeriodicWork.new(context)
+      if ScoutApm::Agent.instance.config.value('profile')
+        # After we fork, setup the handlers here.
+        ScoutApm::Instruments::Stacks.install
+        ScoutApm::Instruments::Stacks.start
+      end
 
-      @background_worker = ScoutApm::BackgroundWorker.new(context)
+      @background_worker = ScoutApm::BackgroundWorker.new
       @background_worker_thread = Thread.new do
         @background_worker.start {
-          periodic_work.run
+          ScoutApm::Agent.instance.process_metrics
+          clean_old_percentiles
         }
       end
+    end
 
-      return true
+    def clean_old_percentiles
+      request_histograms_by_time.
+        keys.
+        select {|timestamp| timestamp.age_in_seconds > 60 * 10 }.
+        each {|old_timestamp| request_histograms_by_time.delete(old_timestamp) }
     end
 
-    def stop_background_worker
-      if @background_worker
-        logger.info("Stopping background worker")
-        @background_worker.stop
-        context.store.write_to_layaway(context.layaway, :force)
-        if @background_worker_thread.alive?
-          @background_worker_thread.wakeup
-          @background_worker_thread.join
+    # If we want to skip the app_server_check, then we must load it.
+    def should_load_instruments?(options={})
+      return true if options[:skip_app_server_check]
+      return true if config.value('dev_trace')
+      return false if !apm_enabled?
+      environment.app_server_integration.found? || !background_job_missing?
+    end
+
+    # Loads the instrumention logic.
+    def load_instruments
+      if !background_job_missing?
+        case environment.background_job_name
+        when :delayed_job
+          install_instrument(ScoutApm::Instruments::DelayedJob)
+        end
+      else
+        case environment.framework
+        when :rails       then install_instrument(ScoutApm::Instruments::ActionControllerRails2)
+        when :rails3_or_4 then
+          install_instrument(ScoutApm::Instruments::ActionControllerRails3Rails4)
+          install_instrument(ScoutApm::Instruments::MiddlewareSummary)
+          install_instrument(ScoutApm::Instruments::RailsRouter)
+        # when :sinatra     then install_instrument(ScoutApm::Instruments::Sinatra)
         end
       end
-    end
 
-    def background_worker_running?
-      @background_worker_thread          &&
-        @background_worker_thread.alive? &&
-        @background_worker               &&
-        @background_worker.running?
+      install_instrument(ScoutApm::Instruments::ActiveRecord)
+      install_instrument(ScoutApm::Instruments::Moped)
+      install_instrument(ScoutApm::Instruments::Mongoid)
+      install_instrument(ScoutApm::Instruments::NetHttp)
+      install_instrument(ScoutApm::Instruments::HttpClient)
+      install_instrument(ScoutApm::Instruments::Redis)
+      install_instrument(ScoutApm::Instruments::InfluxDB)
+      install_instrument(ScoutApm::Instruments::Elasticsearch)
+      install_instrument(ScoutApm::Instruments::Grape)
+    rescue
+      logger.warn "Exception loading instruments:"
+      logger.warn $!.message
+      logger.warn $!.backtrace
     end
 
-    # seconds to batch error reports
-    ERROR_SEND_FREQUENCY = 5
-    def start_error_service_background_worker
-      periodic_work = ScoutApm::ErrorService::PeriodicWork.new(context)
+    def install_instrument(instrument_klass)
+      # Don't attempt to install the same instrument twice
+      return if @installed_instruments.any? { |already_installed_instrument| instrument_klass === already_installed_instrument }
 
-      @error_service_background_worker = ScoutApm::BackgroundWorker.new(context, ERROR_SEND_FREQUENCY)
-      @error_service_background_worker_thread = Thread.new do
-        @error_service_background_worker.start { 
-          periodic_work.run
-        }
+      # Allow users to skip individual instruments via the config file
+      instrument_short_name = instrument_klass.name.split("::").last
+      if (config.value("disabled_instruments") || []).include?(instrument_short_name)
+        logger.info "Skipping Disabled Instrument: #{instrument_short_name} - To re-enable, change `disabled_instruments` key in scout_apm.yml"
+        return
       end
+
+      instance = instrument_klass.new
+      @installed_instruments << instance
+      instance.install
+    end
+
+    def deploy_integration
+      environment.deploy_integration
+    end
+
+    def app_server_missing?(options = {})
+      !environment.app_server_integration(true).found? && !options[:skip_app_server_check]
     end
 
-    def error_service_background_worker_running?
-      @error_service_background_worker_thread          &&
-        @error_service_background_worker_thread.alive? &&
-        @error_service_background_worker               &&
-        @error_service_background_worker.running?
+    def background_job_missing?(options = {})
+      environment.background_job_integration.nil? && !options[:skip_background_job_check]
     end
   end
 end

data/lib/scout_apm/app_server_load.rb

@@ -1,11 +1,9 @@
 module ScoutApm
   class AppServerLoad
     attr_reader :logger
-    attr_reader :context
 
-    def initialize(context)
-      @context = context
-      @logger = context.logger
+    def initialize(logger=Agent.instance.logger)
+      @logger = logger
     end
 
     def run
@@ -15,7 +13,7 @@ module ScoutApm
           logger.debug("Full Application Startup Info: #{data.inspect}")
 
           payload = ScoutApm::Serializers::AppServerLoadSerializer.serialize(data)
-          reporter = Reporter.new(context, :app_server_load)
+          reporter = Reporter.new(:app_server_load)
           reporter.report(payload)
 
           logger.debug("Finished sending Startup Info")
@@ -29,43 +27,19 @@ module ScoutApm
     end
 
     def data
-      { 
-        :language           => 'ruby',
-        :language_version   => RUBY_VERSION,
-        :ruby_version       => RUBY_VERSION, # Deprecated.
-
-        :framework          => to_s_safe(environment.framework_integration.human_name),
-        :framework_version  => to_s_safe(environment.framework_integration.version),
-
-        :server_time        => to_s_safe(Time.now),
-        :environment        => to_s_safe(environment.framework_integration.env),
-        :app_server         => to_s_safe(environment.app_server),
-        :hostname           => to_s_safe(environment.hostname),
-        :database_engine    => to_s_safe(environment.database_engine),      # Detected
-        :database_adapter   => to_s_safe(environment.raw_database_adapter), # Raw
-        :application_name   => to_s_safe(environment.application_name),
-        :libraries          => ScoutApm::Utils::InstalledGems.new(context).run,
-        :paas               => to_s_safe(environment.platform_integration.name),
-        :git_sha            => to_s_safe(environment.git_revision.sha)
+      { :server_time        => Time.now,
+        :framework          => ScoutApm::Environment.instance.framework_integration.human_name,
+        :framework_version  => ScoutApm::Environment.instance.framework_integration.version,
+        :environment        => ScoutApm::Environment.instance.framework_integration.env,
+        :app_server         => ScoutApm::Environment.instance.app_server,
+        :ruby_version       => RUBY_VERSION,
+        :hostname           => ScoutApm::Environment.instance.hostname,
+        :database_engine    => ScoutApm::Environment.instance.database_engine,      # Detected
+        :database_adapter   => ScoutApm::Environment.instance.raw_database_adapter, # Raw
+        :application_name   => ScoutApm::Environment.instance.application_name,
+        :libraries          => ScoutApm::Utils::InstalledGems.new.run,
+        :paas               => ScoutApm::Environment.instance.platform_integration.name
       }
-    ensure
-      # Sometimes :database_engine and :database_adapter can cause a reference to an AR connection.
-      # Make sure we release all AR connections held by this thread.
-      ActiveRecord::Base.clear_active_connections! if Utils::KlassHelper.defined?("ActiveRecord::Base")
-    end
-
-    # Calls `.to_s` on the object passed in.
-    # Returns literal string 'to_s error' if the object does not respond to .to_s
-    def to_s_safe(obj)
-      if obj.respond_to?(:to_s)
-        obj.to_s
-      else
-        'to_s error'
-      end
-    end
-
-    def environment
-      context.environment
     end
   end
 end

data/lib/scout_apm/attribute_arranger.rb

@@ -5,6 +5,8 @@ module ScoutApm
     def self.call(subject, attributes_list)
       attributes_list.inject({}) do |attribute_hash, attribute|
         case attribute
+        when :traces
+          attribute_hash[attribute] = subject.traces.to_a
         when Array
           attribute_hash[attribute[0]] = subject.send(attribute[1])
         when :bucket
@@ -12,20 +14,7 @@ module ScoutApm
         when :name
           attribute_hash[attribute] = subject.bucket_name
         when Symbol
-          data = subject.send(attribute)
-
-          # Never try to `as_json` a time object, since it'll break if the
-          # app has the Oj gem set to mimic_JSON, and there's never
-          # anything interesting nested inside of a Time obj. We just want
-          # the ISO8601 string (which happens later in the payload
-          # serializing process)
-          if data.is_a?(Time)
-            attribute_hash[attribute] = data
-          elsif data.respond_to?(:as_json)
-            attribute_hash[attribute] = data.as_json
-          else
-            attribute_hash[attribute] = data
-          end
+          attribute_hash[attribute] = subject.send(attribute)
         end
         attribute_hash
       end