scout_apm 2.3.5 → 2.4.0.pre

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7f02661ce9cf968ffd2fc3efdd96f81a82b49f05
-  data.tar.gz: 2f47f1ddebab08773b82679dad0f81a29442c7f3
+  metadata.gz: cd45ad6c823f8163081d1a4083f1740823351551
+  data.tar.gz: 822bdbc9113c78762a31258b166fe71786bb3448
 SHA512:
-  metadata.gz: 52a809924c15851078d2f8e3ec820021b6ec018f767d08240d1481975a18747e27442c2742bc72be51692d5c35cf9a0d47188be6d386832bbd8d4bdb473a34d4
-  data.tar.gz: 9d4359d4285532afef1e043412459360430e7ec95392f9fa4a705e3b428d8bddcab953425d1cf23335a8cdd9e9ec39d1e4ce79ef1406208b6d62246b975d43d4
+  metadata.gz: 6c041d9e89c7dd58588755d805ff88dce9d55ddc38733143e92b32cb1cc8c2d53f577540f69dcc6af8aabb217903ef11c8dc8fe8981e191edc5239c42a1e9317
+  data.tar.gz: 4d70647935de5bcd27fbaa04f3b4a5e657cb01f13e7384ea17f57ee6bbcf5c73b30bfeb7c37c66bda2688069c4b7c5af61e29098a61c293ba06e1f2a3c1358aa

data/CHANGELOG.markdown

@@ -1,26 +1,3 @@
-# 2.3.5
-
-* More robust recovery from stale layaway files
-* Quiet logging when hitting unusual layaway file limits
-* Better naming for Sidekiq delayed method jobs
-* Webrick is only required if actually needed
-
-# 2.3.4
-
-* Capture 300 characters of a url from net/http and httpclient instruments (up from 100).
-
-# 2.3.3
-
-* Capture ActiveRecord calls that generate more complex queries
-* More aggressively determine names of complex queries (to determine "User/find", "Account/create" and similar)
-* Increases the maximum size of SQL queries that are sanitized to 16KB from 4 KB
-* Captures all SQL individual queries generated in a given AR call (previous only a single query was captured)
-
-# 2.3.2
-
-* More robust startup sequence when using `rails server` vs. directly launching an app server
-* Avoid incompatibility with 3rd party gems that aggressively obtain database connections
-
 # 2.3.1
 
 * Fix DevTrace bug

data/lib/scout_apm.rb

@@ -14,6 +14,7 @@ require 'socket'
 require 'thread'
 require 'time'
 require 'yaml'
+require 'webrick'
 
 #####################################
 # Gem Requires
@@ -79,16 +80,18 @@ require 'scout_apm/instruments/elasticsearch'
 require 'scout_apm/instruments/active_record'
 require 'scout_apm/instruments/action_controller_rails_2'
 require 'scout_apm/instruments/action_controller_rails_3_rails4'
+require 'scout_apm/instruments/action_view'
 require 'scout_apm/instruments/middleware_summary'
 require 'scout_apm/instruments/middleware_detailed' # Disabled by default, see the file for more details
 require 'scout_apm/instruments/rails_router'
 require 'scout_apm/instruments/grape'
 require 'scout_apm/instruments/sinatra'
+require 'allocations'
+
 require 'scout_apm/instruments/process/process_cpu'
 require 'scout_apm/instruments/process/process_memory'
 require 'scout_apm/instruments/percentile_sampler'
-require 'scout_apm/instruments/action_view'
-require 'allocations'
+require 'scout_apm/instruments/samplers'
 
 require 'scout_apm/app_server_load'
 
@@ -97,7 +100,6 @@ require 'scout_apm/utils/active_record_metric_name'
 require 'scout_apm/utils/backtrace_parser'
 require 'scout_apm/utils/installed_gems'
 require 'scout_apm/utils/klass_helper'
-require 'scout_apm/utils/null_logger'
 require 'scout_apm/utils/scm'
 require 'scout_apm/utils/sql_sanitizer'
 require 'scout_apm/utils/time'
@@ -108,8 +110,8 @@ require 'scout_apm/utils/gzip_helper'
 require 'scout_apm/config'
 require 'scout_apm/environment'
 require 'scout_apm/agent'
-require 'scout_apm/agent/logging'
-require 'scout_apm/agent/reporting'
+require 'scout_apm/logger'
+require 'scout_apm/reporting'
 require 'scout_apm/layaway'
 require 'scout_apm/layaway_file'
 require 'scout_apm/reporter'
@@ -162,6 +164,14 @@ require 'scout_apm/remote/message'
 require 'scout_apm/remote/recorder'
 require 'scout_apm/instruments/resque'
 
+require 'scout_apm/agent_context'
+require 'scout_apm/instrument_manager'
+require 'scout_apm/periodic_work'
+require 'scout_apm/agent/preconditions'
+require 'scout_apm/agent/exit_handler'
+require 'scout_apm/tasks/doctor'
+require 'scout_apm/tasks/support'
+
 if defined?(Rails) && defined?(Rails::VERSION) && defined?(Rails::VERSION::MAJOR) && Rails::VERSION::MAJOR >= 3 && defined?(Rails::Railtie)
   module ScoutApm
     class Railtie < Rails::Railtie
@@ -172,17 +182,18 @@ if defined?(Rails) && defined?(Rails::VERSION) && defined?(Rails::VERSION::MAJOR
         app.middleware.use ScoutApm::Middleware
 
         # Attempt to start right away, this will work best for preloading apps, Unicorn & Puma & similar
-        ScoutApm::Agent.instance.start
-      end
-    end
-    class Railtie < Rails::Railtie
-      initializer 'scout_apm.start' do |app|
+        ScoutApm::Agent.instance.install
+
         # Install the middleware every time in development mode.
         # The middleware is a noop if dev_trace is not enabled in config
         if Rails.env.development?
           app.middleware.use ScoutApm::Instant::Middleware
         end
       end
+
+      rake_tasks do
+        load "tasks/doctor.rake"
+      end
     end
   end
 else

data/lib/scout_apm/agent.rb

@@ -1,244 +1,110 @@
 module ScoutApm
-  # The agent gathers performance data from a Ruby application. One Agent instance is created per-Ruby process.
+  # The entry-point for the ScoutApm Agent.
   #
-  # Each Agent object creates a worker thread (unless monitoring is disabled or we're forking).
-  # The worker thread wakes up every +Agent#period+, merges in-memory metrics w/those saved to disk,
-  # saves tshe merged data to disk, and sends it to the Scout server.
+  # Only one Agent instance is created per-Ruby process, and it coordinates the lifecycle of the monitoring.
+  #   - initializes various data stores
+  #   - coordinates configuration & logging
+  #   - starts background threads, running periodically
+  #   - installs shutdown hooks
   class Agent
     # see self.instance
     @@instance = nil
 
-    # Accessors below are for associated classes
-    attr_accessor :store
-    attr_reader :recorder
-    attr_accessor :layaway
-    attr_accessor :config
-    attr_accessor :logger
-    attr_accessor :log_file # path to the log file
-    attr_accessor :options # options passed to the agent when +#start+ is called.
-    attr_accessor :metric_lookup # Hash used to lookup metric ids based on their name and scope
-    attr_reader :slow_request_policy
-    attr_reader :slow_job_policy
-    attr_reader :process_start_time # used when creating slow transactions to report how far from startup the transaction was recorded.
-    attr_reader :ignored_uris
+    attr_reader :context
 
-    # Histogram of the cumulative requests since the start of the process
-    attr_reader :request_histograms
+    attr_accessor :options # options passed to the agent when +#start+ is called.
 
-    # Histogram of the requests, distinct by reporting period (minute)
-    # { StoreReportingPeriodTimestamp => RequestHistograms }
-    attr_reader :request_histograms_by_time
+    attr_reader :instrument_manager
 
     # All access to the agent is thru this class method to ensure multiple Agent instances are not initialized per-Ruby process.
     def self.instance(options = {})
       @@instance ||= self.new(options)
     end
 
-    # Note - this doesn't start instruments or the worker thread. This is handled via +#start+ as we don't
-    # want to start the worker thread or install instrumentation if (1) disabled for this environment (2) a worker thread shouldn't
-    # be started (when forking).
+    # First call of the agent. Does very little so that the object can be created, and exist.
     def initialize(options = {})
-      @started = false
-      @shutting_down = false
-      @process_start_time = Time.now
-      @options ||= options
-
-      # until the agent is started, there's no recorder
-      @recorder = nil
-
-      # Start up without attempting to load a configuration file. We need to be
-      # able to lookup configuration options like "application_root" which would
-      # then in turn influence where the configuration file came from.
-      #
-      # Later in initialization, we reset @config to include the file.
-      @config = ScoutApm::Config.without_file
-
-      @slow_request_policy = ScoutApm::SlowRequestPolicy.new
-      @slow_job_policy = ScoutApm::SlowJobPolicy.new
-      @request_histograms = ScoutApm::RequestHistograms.new
-      @request_histograms_by_time = Hash.new { |h, k| h[k] = ScoutApm::RequestHistograms.new }
-
-      @store          = ScoutApm::Store.new
-
-      @layaway        = ScoutApm::Layaway.new(config, environment)
-      @metric_lookup  = Hash.new
-
-      @installed_instruments = []
+      @options = options
+      @context = ScoutApm::AgentContext.new
     end
 
-    def environment
-      ScoutApm::Environment.instance
+    def logger
+      context.logger
     end
 
-    def apm_enabled?
-      config.value('monitor')
-    end
+    # Finishes setting up the instrumentation, configuration, and attempts to start the agent.
+    def install(force=false)
+      context.config = ScoutApm::Config.with_file(context, context.config.value("config_file"))
 
-    # If true, the agent will start regardless of safety checks. Currently just used for testing.
-    def force?
-      @options[:force]
-    end
+      context.logger.info "Scout Agent [#{ScoutApm::VERSION}] Initialized"
 
-    def preconditions_met?(options={})
-      if !apm_enabled?
-        logger.warn "Monitoring isn't enabled for the [#{environment.env}] environment. #{force? ? 'Forcing agent to start' : 'Not starting agent'}"
-        return false unless force?
-      end
+      @instrument_manager = ScoutApm::InstrumentManager.new(context)
+      @instrument_manager.install! if should_load_instruments? || force
 
-      if !environment.application_name
-        logger.warn "An application name could not be determined. Specify the :name value in scout_apm.yml. #{force? ? 'Forcing agent to start' : 'Not starting agent'}."
-        return false unless force?
-      end
+      install_background_job_integrations
+      install_app_server_integration
 
-      if environment.interactive?
-        logger.warn "Agent attempting to load in interactive mode. #{force? ? 'Forcing agent to start' : 'Not starting agent'}"
-        return false unless force?
-      end
+      # XXX: Should this happen at application start?
+      # Should this ever happen after fork?
+      # We start a thread in this, which can screw stuff up when we then fork.
+      AppServerLoad.new(context).run
 
-      if app_server_missing?(options) && background_job_missing?
-        if force?
-          logger.warn "Agent starting (forced)"
-        else
-          logger.warn "Deferring agent start. Standing by for first request"
-        end
-        return false unless force?
-      end
+      logger.info "Scout Agent [#{ScoutApm::VERSION}] installed"
 
-      if started?
-        logger.warn "Already started agent."
-        return false
-      end
+      context.installed!
 
-      if defined?(::ScoutRails)
-        logger.warn "ScoutAPM is incompatible with the old Scout Rails plugin. Please remove scout_rails from your Gemfile"
-        return false unless force?
+      if ScoutApm::Agent::Preconditions.check?(context) || force
+        start
       end
-
-      true
     end
 
-    # This is called via +ScoutApm::Agent.instance.start+ when ScoutApm is required in a Ruby application.
-    # It initializes the agent and starts the worker thread (if appropiate).
-    def start(options = {})
-      @options.merge!(options)
+    # Unconditionally starts the agent. This includes verifying instruments are
+    # installed, and starting the background worker.
+    #
+    # Does not attempt to start twice.
+    def start
+      return if context.started?
+      install unless context.installed?
 
-      @config = ScoutApm::Config.with_file(@config.value("config_file"))
-      layaway.config = config
+      context.started!
 
-      init_logger
-      logger.info "Attempting to start Scout Agent [#{ScoutApm::VERSION}] on [#{environment.hostname}]"
-
-      @recorder = create_recorder
-
-      @config.log_settings
-
-      @ignored_uris = ScoutApm::IgnoredUris.new(config.value('ignore'))
-
-      load_instruments if should_load_instruments?(options)
-
-      if !@config.any_keys_found?
-        logger.info("No configuration file loaded, and no configuration found in ENV. " +
-                    "For assistance configuring Scout, visit " +
-                    "http://help.apm.scoutapp.com/#configuration-options")
-      end
+      log_environment
 
-      return false unless preconditions_met?(options)
-      @started = true
-      logger.info "Starting monitoring for [#{environment.application_name}]. Framework [#{environment.framework}] App Server [#{environment.app_server}] Background Job Framework [#{environment.background_job_name}]."
-
-      [ ScoutApm::Instruments::Process::ProcessCpu.new(environment.processors, logger),
-        ScoutApm::Instruments::Process::ProcessMemory.new(logger),
-        ScoutApm::Instruments::PercentileSampler.new(logger, request_histograms_by_time),
-      ].each { |s| store.add_sampler(s) }
-
-      app_server_load_hook
-
-      if environment.background_job_integration
-        environment.background_job_integration.install
-        logger.info "Installed Background Job Integration [#{environment.background_job_name}]"
-      end
-
-      # start_background_worker? is true on non-forking servers, and directly
-      # starts the background worker.  On forking servers, a server-specific
-      # hook is inserted to start the background worker after forking.
-      if start_background_worker?
-        start_background_worker
-        logger.info "Scout Agent [#{ScoutApm::VERSION}] Initialized"
-      else
-        environment.app_server_integration.install
-        logger.info "Scout Agent [#{ScoutApm::VERSION}] loaded in [#{environment.app_server}] master process. Monitoring will start after server forks its workers."
-      end
+      start_background_worker
     end
 
-    # Sends a ping to APM right away, smoothes out onboarding
-    # Collects up any relevant info (framework, app server, system time, ruby version, etc)
-    def app_server_load_hook
-      @app_server_load ||= AppServerLoad.new.run
-    end
+    def log_environment
+      bg_names = context.environment.background_job_integrations.map{|bg| bg.name }.join(", ")
 
-    def exit_handler_supported?
-      if environment.sinatra?
-        logger.debug "Exit handler not supported for Sinatra"
-        false
-      elsif environment.jruby?
-        logger.debug "Exit handler not supported for JRuby"
-        false
-      elsif environment.rubinius?
-        logger.debug "Exit handler not supported for Rubinius"
-        false
-      else
-        true
-      end
+      logger.info(
+        "Scout Agent [#{ScoutApm::VERSION}] starting for [#{context.environment.application_name}] " +
+        "Framework [#{context.environment.framework}] " +
+        "App Server [#{context.environment.app_server}] " +
+        "Background Job Framework [#{bg_names}] " +
+        "Hostname [#{context.environment.hostname}]"
+      )
     end
 
-    # at_exit, calls Agent#shutdown to wrapup metric reporting.
-    def install_exit_handler
-      logger.debug "Shutdown handler not supported" and return unless exit_handler_supported?
-      logger.debug "Installing Shutdown Handler"
-
-      at_exit do
-        logger.info "Shutting down Scout Agent"
-        # MRI 1.9 bug drops exit codes.
-        # http://bugs.ruby-lang.org/issues/5218
-        if environment.ruby_19?
-          status = $!.status if $!.is_a?(SystemExit)
-          shutdown
-          exit status if status
-        else
-          shutdown
-        end
+    # Attempts to install all background job integrations. This can come up if
+    # an app has both Resque and Sidekiq - we want both to be installed if
+    # possible, it's no harm to have the "wrong" one also installed while running.
+    def install_background_job_integrations
+      context.environment.background_job_integrations.each do |int|
+        int.install
+        logger.info "Installed Background Job Integration [#{int.name}]"
       end
     end
 
-    # Called via an at_exit handler, it:
-    # (1) Stops the background worker
-    # (2) Stores metrics locally (forcing current-minute metrics to be written)
-    # It does not attempt to actually report metrics.
-    def shutdown
-      logger.info "Shutting down ScoutApm"
-
-      return if !started?
-
-      return if @shutdown
-      @shutdown = true
-
-      if @background_worker
-        logger.info("Stopping background worker")
-        @background_worker.stop
-        store.write_to_layaway(layaway, :force)
-        if @background_worker_thread.alive?
-          @background_worker_thread.wakeup
-          @background_worker_thread.join
-        end
-      end
-    end
-
-    def started?
-      @started
+    # This sets up the background worker thread to run at the correct time,
+    # either immediately, or after a fork into the actual unicorn/puma/etc
+    # worker
+    def install_app_server_integration
+      context.environment.app_server_integration.install
+      logger.info "Installed Application Server Integration [#{context.environment.app_server}]."
     end
 
-    def shutting_down?
-      @shutdown == true
+    # If true, the agent will start regardless of safety checks.
+    def force?
+      @options[:force]
     end
 
     # The worker thread will automatically start UNLESS:
@@ -246,167 +112,63 @@ module ScoutApm
     # * A supported application server is detected, but it forks. In this case,
     #   the agent is started in the forked process.
     def start_background_worker?
-      return true if environment.app_server == :thin
-      return true if environment.app_server == :webrick
       return true if force?
-      return !environment.forking?
+      return !context.environment.forking?
     end
 
-    def background_worker_running?
-      @background_worker_thread          &&
-        @background_worker_thread.alive? &&
-        @background_worker               &&
-        @background_worker.running?
+    def should_load_instruments?
+      return true if context.config.value('dev_trace')
+      # XXX: If monitor is true, we want to install, right?
+      # return false if context.config.value('monitor')
+      context.environment.app_server_integration.found? || context.environment.background_job_integration
     end
 
-    # Creates the worker thread. The worker thread is a loop that runs continuously. It sleeps for +Agent#period+ and when it wakes,
-    # processes data, either saving it to disk or reporting to Scout.
-    # => true if thread & worker got started
-    # => false if it wasn't started (either due to already running, or other preconditions)
-    def start_background_worker(quiet=false)
-      if !apm_enabled?
-        logger.debug "Not starting background worker as monitoring isn't enabled." unless quiet
+    #################################
+    #  Background Worker Lifecycle  #
+    #################################
+
+    def start_background_worker
+      if !context.config.value('monitor')
+        logger.debug "Not starting background worker as monitoring isn't enabled."
         return false
       end
+
       if background_worker_running?
-        logger.info "Not starting background worker, already started" unless quiet
-        return false
-      end
-      if shutting_down?
-        logger.info "Not starting background worker, already in process of shutting down" unless quiet
-        return false
+        logger.info "Not starting background worker, already started"
+        return
       end
 
       logger.info "Initializing worker thread."
 
-      install_exit_handler
+      ScoutApm::Agent::ExitHandler.new(context).install
 
-      @recorder = create_recorder
-      logger.info("Recorder is now: #{@recorder.class}")
+      periodic_work = ScoutApm::PeriodicWork.new(context)
 
-      @background_worker = ScoutApm::BackgroundWorker.new
+      @background_worker = ScoutApm::BackgroundWorker.new(context)
       @background_worker_thread = Thread.new do
         @background_worker.start {
-          ScoutApm::Debug.instance.call_periodic_hooks
-          ScoutApm::Agent.instance.process_metrics
-          clean_old_percentiles
+          periodic_work.run
         }
       end
-
-      return true
-    end
-
-    def clean_old_percentiles
-      request_histograms_by_time.
-        keys.
-        select {|timestamp| timestamp.age_in_seconds > 60 * 10 }.
-        each {|old_timestamp| request_histograms_by_time.delete(old_timestamp) }
-    end
-
-    # If we want to skip the app_server_check, then we must load it.
-    def should_load_instruments?(options={})
-      return true if options[:skip_app_server_check]
-      return true if config.value('dev_trace')
-      return false if !apm_enabled?
-      environment.app_server_integration.found? || !background_job_missing?
     end
 
-    # Loads the instrumention logic.
-    def load_instruments
-      case environment.framework
-      when :rails then
-        install_instrument(ScoutApm::Instruments::ActionControllerRails2)
-      when :rails3_or_4 then
-        install_instrument(ScoutApm::Instruments::ActionControllerRails3Rails4)
-        install_instrument(ScoutApm::Instruments::RailsRouter)
-
-        if config.value("detailed_middleware")
-          install_instrument(ScoutApm::Instruments::MiddlewareDetailed)
-        else
-          install_instrument(ScoutApm::Instruments::MiddlewareSummary)
+    def stop_background_worker
+      if @background_worker
+        logger.info("Stopping background worker")
+        @background_worker.stop
+        context.store.write_to_layaway(context.layaway, :force)
+        if @background_worker_thread.alive?
+          @background_worker_thread.wakeup
+          @background_worker_thread.join
         end
       end
-
-      install_instrument(ScoutApm::Instruments::ActionView)
-      install_instrument(ScoutApm::Instruments::ActiveRecord)
-      install_instrument(ScoutApm::Instruments::Moped)
-      install_instrument(ScoutApm::Instruments::Mongoid)
-      install_instrument(ScoutApm::Instruments::NetHttp)
-      install_instrument(ScoutApm::Instruments::HttpClient)
-      install_instrument(ScoutApm::Instruments::Redis)
-      install_instrument(ScoutApm::Instruments::InfluxDB)
-      install_instrument(ScoutApm::Instruments::Elasticsearch)
-      install_instrument(ScoutApm::Instruments::Grape)
-    rescue
-      logger.warn "Exception loading instruments:"
-      logger.warn $!.message
-      logger.warn $!.backtrace
     end
 
-    def install_instrument(instrument_klass)
-      # Don't attempt to install the same instrument twice
-      return if @installed_instruments.any? { |already_installed_instrument| instrument_klass === already_installed_instrument }
-
-      # Allow users to skip individual instruments via the config file
-      instrument_short_name = instrument_klass.name.split("::").last
-      if (config.value("disabled_instruments") || []).include?(instrument_short_name)
-        logger.info "Skipping Disabled Instrument: #{instrument_short_name} - To re-enable, change `disabled_instruments` key in scout_apm.yml"
-        return
-      end
-
-      instance = instrument_klass.new
-      @installed_instruments << instance
-      instance.install
-    end
-
-    def app_server_missing?(options = {})
-      !environment.app_server_integration(true).found? && !options[:skip_app_server_check]
-    end
-
-    def background_job_missing?(options = {})
-      environment.background_job_integration.nil? && !options[:skip_background_job_check]
-    end
-
-    def clear_recorder
-      @recorder = nil
-    end
-
-    def create_recorder
-      if @recorder
-        return @recorder
-      end
-
-      if config.value("async_recording")
-        logger.debug("Using asynchronous recording")
-        ScoutApm::BackgroundRecorder.new(logger).start
-      else
-        logger.debug("Using synchronous recording")
-        ScoutApm::SynchronousRecorder.new(logger).start
-      end
-    end
-
-    def start_remote_server(bind, port)
-      return if @remote_server && @remote_server.running?
-
-      logger.info("Starting Remote Agent Server")
-
-      # Start the listening web server only in parent process.
-      @remote_server = ScoutApm::Remote::Server.new(
-        bind,
-        port,
-        ScoutApm::Remote::Router.new(ScoutApm::SynchronousRecorder.new(logger), logger),
-        logger
-      )
-
-      @remote_server.start
-    end
-
-    # Execute this in the child process of a remote agent. The parent is
-    # expected to have its accepting webserver up and running
-    def use_remote_recorder(host, port)
-      logger.debug("Becoming Remote Agent (reporting to: #{host}:#{port})")
-      @recorder = ScoutApm::Remote::Recorder.new(host, port, logger)
-      @store = ScoutApm::FakeStore.new
+    def background_worker_running?
+      @background_worker_thread          &&
+        @background_worker_thread.alive? &&
+        @background_worker               &&
+        @background_worker.running?
     end
   end
 end