scout_apm 2.1.32 → 2.2.0.pre0

data/lib/scout_apm.rb

@@ -14,7 +14,7 @@ require 'socket'
 require 'thread'
 require 'time'
 require 'yaml'
-require 'webrick'
+require 'rbconfig'
 
 #####################################
 # Gem Requires
@@ -26,11 +26,8 @@ require 'rusage'
 #####################################
 require 'scout_apm/version'
 
-require 'scout_apm/debug'
 require 'scout_apm/tracked_request'
 require 'scout_apm/layer'
-require 'scout_apm/limited_layer'
-require 'scout_apm/layer_children_set'
 require 'scout_apm/request_manager'
 require 'scout_apm/call_set'
 
@@ -54,7 +51,6 @@ require 'scout_apm/server_integrations/null'
 
 require 'scout_apm/background_job_integrations/sidekiq'
 require 'scout_apm/background_job_integrations/delayed_job'
-require 'scout_apm/background_job_integrations/resque'
 
 require 'scout_apm/framework_integrations/rails_2'
 require 'scout_apm/framework_integrations/rails_3_or_4'
@@ -67,6 +63,9 @@ require 'scout_apm/platform_integrations/server'
 
 require 'scout_apm/histogram'
 
+require 'scout_apm/deploy_integrations/capistrano_3'
+# require 'scout_apm/deploy_integrations/capistrano_2'
+
 require 'scout_apm/instruments/net_http'
 require 'scout_apm/instruments/http_client'
 require 'scout_apm/instruments/moped'
@@ -74,20 +73,26 @@ require 'scout_apm/instruments/mongoid'
 require 'scout_apm/instruments/redis'
 require 'scout_apm/instruments/influxdb'
 require 'scout_apm/instruments/elasticsearch'
+require 'scout_apm/instruments/delayed_job'
 require 'scout_apm/instruments/active_record'
 require 'scout_apm/instruments/action_controller_rails_2'
 require 'scout_apm/instruments/action_controller_rails_3_rails4'
 require 'scout_apm/instruments/middleware_summary'
-require 'scout_apm/instruments/middleware_detailed' # Disabled by default, see the file for more details
+# require 'scout_apm/instruments/middleware_detailed' # Currently disabled functionality, see the file for details.
 require 'scout_apm/instruments/rails_router'
 require 'scout_apm/instruments/grape'
 require 'scout_apm/instruments/sinatra'
 require 'scout_apm/instruments/process/process_cpu'
 require 'scout_apm/instruments/process/process_memory'
 require 'scout_apm/instruments/percentile_sampler'
-require 'scout_apm/instruments/action_view'
 require 'allocations'
 
+begin
+  require 'stacks'
+rescue LoadError
+  require 'scout_apm/utils/fake_stacks'
+end
+
 require 'scout_apm/app_server_load'
 
 require 'scout_apm/ignored_uris.rb'
@@ -96,12 +101,9 @@ require 'scout_apm/utils/backtrace_parser'
 require 'scout_apm/utils/installed_gems'
 require 'scout_apm/utils/klass_helper'
 require 'scout_apm/utils/null_logger'
-require 'scout_apm/utils/scm'
 require 'scout_apm/utils/sql_sanitizer'
 require 'scout_apm/utils/time'
 require 'scout_apm/utils/unique_id'
-require 'scout_apm/utils/numbers'
-require 'scout_apm/utils/gzip_helper'
 
 require 'scout_apm/config'
 require 'scout_apm/environment'
@@ -120,43 +122,35 @@ require 'scout_apm/fake_store'
 require 'scout_apm/tracer'
 require 'scout_apm/context'
 require 'scout_apm/instant_reporting'
-require 'scout_apm/background_recorder'
-require 'scout_apm/synchronous_recorder'
+require 'scout_apm/trace_compactor'
 
 require 'scout_apm/metric_meta'
 require 'scout_apm/metric_stats'
 require 'scout_apm/slow_transaction'
 require 'scout_apm/slow_job_record'
+require 'scout_apm/slow_item_set'
 require 'scout_apm/scored_item_set'
 require 'scout_apm/slow_request_policy'
 require 'scout_apm/slow_job_policy'
 require 'scout_apm/job_record'
 require 'scout_apm/request_histograms'
 
+require 'scout_apm/capacity'
 require 'scout_apm/attribute_arranger'
-require 'scout_apm/git_revision'
 
 require 'scout_apm/serializers/payload_serializer'
 require 'scout_apm/serializers/payload_serializer_to_json'
 require 'scout_apm/serializers/jobs_serializer_to_json'
 require 'scout_apm/serializers/slow_jobs_serializer_to_json'
 require 'scout_apm/serializers/metrics_to_json_serializer'
-require 'scout_apm/serializers/histograms_serializer_to_json'
 require 'scout_apm/serializers/directive_serializer'
 require 'scout_apm/serializers/app_server_load_serializer'
+require 'scout_apm/serializers/deploy_serializer'
 
 require 'scout_apm/middleware'
 
 require 'scout_apm/instant/middleware'
 
-require 'scout_apm/rack'
-
-require 'scout_apm/remote/server'
-require 'scout_apm/remote/router'
-require 'scout_apm/remote/message'
-require 'scout_apm/remote/recorder'
-require 'scout_apm/instruments/resque'
-
 if defined?(Rails) && defined?(Rails::VERSION) && defined?(Rails::VERSION::MAJOR) && Rails::VERSION::MAJOR >= 3 && defined?(Rails::Railtie)
   module ScoutApm
     class Railtie < Rails::Railtie
@@ -172,8 +166,6 @@ if defined?(Rails) && defined?(Rails::VERSION) && defined?(Rails::VERSION::MAJOR
     end
     class Railtie < Rails::Railtie
       initializer 'scout_apm.start' do |app|
-        # Install the middleware every time in development mode.
-        # The middleware is a noop if dev_trace is not enabled in config
         if Rails.env.development?
           app.middleware.use ScoutApm::Instant::Middleware
         end

data/lib/scout_apm/agent.rb

@@ -10,9 +10,9 @@ module ScoutApm
 
     # Accessors below are for associated classes
     attr_accessor :store
-    attr_reader :recorder
     attr_accessor :layaway
     attr_accessor :config
+    attr_accessor :capacity
     attr_accessor :logger
     attr_accessor :log_file # path to the log file
     attr_accessor :options # options passed to the agent when +#start+ is called.
@@ -42,9 +42,6 @@ module ScoutApm
       @process_start_time = Time.now
       @options ||= options
 
-      # until the agent is started, there's no recorder
-      @recorder = nil
-
       # Start up without attempting to load a configuration file. We need to be
       # able to lookup configuration options like "application_root" which would
       # then in turn influence where the configuration file came from.
@@ -58,10 +55,10 @@ module ScoutApm
       @request_histograms_by_time = Hash.new { |h, k| h[k] = ScoutApm::RequestHistograms.new }
 
       @store          = ScoutApm::Store.new
-
-      @layaway        = ScoutApm::Layaway.new(config, environment)
+      @layaway        = ScoutApm::Layaway.new
       @metric_lookup  = Hash.new
 
+      @capacity       = ScoutApm::Capacity.new
       @installed_instruments = []
     end
 
@@ -89,11 +86,6 @@ module ScoutApm
         return false unless force?
       end
 
-      if environment.interactive?
-        logger.warn "Agent attempting to load in interactive mode. #{force? ? 'Forcing agent to start' : 'Not starting agent'}"
-        return false unless force?
-      end
-
       if app_server_missing?(options) && background_job_missing?
         if force?
           logger.warn "Agent starting (forced)"
@@ -120,34 +112,26 @@ module ScoutApm
     # It initializes the agent and starts the worker thread (if appropiate).
     def start(options = {})
       @options.merge!(options)
-
       @config = ScoutApm::Config.with_file(@config.value("config_file"))
-      layaway.config = config
-
       init_logger
       logger.info "Attempting to start Scout Agent [#{ScoutApm::VERSION}] on [#{environment.hostname}]"
 
-      @recorder = create_recorder
-
-      @config.log_settings
-
       @ignored_uris = ScoutApm::IgnoredUris.new(config.value('ignore'))
 
-      load_instruments if should_load_instruments?(options)
-
-      if !@config.any_keys_found?
-        logger.info("No configuration file loaded, and no configuration found in ENV. " +
-                    "For assistance configuring Scout, visit " +
-                    "http://help.apm.scoutapp.com/#configuration-options")
+      if environment.deploy_integration
+        logger.info "Starting monitoring for [#{environment.deploy_integration.name}]]."
+        return environment.deploy_integration.install
       end
 
+      load_instruments if should_load_instruments?(options)
+
       return false unless preconditions_met?(options)
       @started = true
       logger.info "Starting monitoring for [#{environment.application_name}]. Framework [#{environment.framework}] App Server [#{environment.app_server}] Background Job Framework [#{environment.background_job_name}]."
 
       [ ScoutApm::Instruments::Process::ProcessCpu.new(environment.processors, logger),
         ScoutApm::Instruments::Process::ProcessMemory.new(logger),
-        ScoutApm::Instruments::PercentileSampler.new(logger, request_histograms_by_time),
+        ScoutApm::Instruments::PercentileSampler.new(logger, 95),
       ].each { |s| store.add_sampler(s) }
 
       app_server_load_hook
@@ -215,21 +199,20 @@ module ScoutApm
     # It does not attempt to actually report metrics.
     def shutdown
       logger.info "Shutting down ScoutApm"
-
       return if !started?
 
-      return if @shutdown
-      @shutdown = true
-
       if @background_worker
         logger.info("Stopping background worker")
         @background_worker.stop
         store.write_to_layaway(layaway, :force)
-        if @background_worker_thread.alive?
-          @background_worker_thread.wakeup
-          @background_worker_thread.join
-        end
       end
+
+      logger.debug "Joining background worker thread"
+      if @background_worker_thread
+        @background_worker_thread.wakeup
+        @background_worker_thread.join
+      end
+      ScoutApm::Instruments::Stacks.uninstall
     end
 
     def started?
@@ -248,10 +231,7 @@ module ScoutApm
     end
 
     def background_worker_running?
-      @background_worker_thread          &&
-        @background_worker_thread.alive? &&
-        @background_worker               &&
-        @background_worker.running?
+      !! @background_worker_thread
     end
 
     # Creates the worker thread. The worker thread is a loop that runs continuously. It sleeps for +Agent#period+ and when it wakes,
@@ -266,13 +246,15 @@ module ScoutApm
 
       install_exit_handler
 
-      @recorder = create_recorder
-      logger.info("recorder is now: #{@recorder.class}")
+      if ScoutApm::Agent.instance.config.value('profile')
+        # After we fork, setup the handlers here.
+        ScoutApm::Instruments::Stacks.install
+        ScoutApm::Instruments::Stacks.start
+      end
 
       @background_worker = ScoutApm::BackgroundWorker.new
       @background_worker_thread = Thread.new do
         @background_worker.start {
-          ScoutApm::Debug.instance.call_periodic_hooks
           ScoutApm::Agent.instance.process_metrics
           clean_old_percentiles
         }
@@ -296,21 +278,22 @@ module ScoutApm
 
     # Loads the instrumention logic.
     def load_instruments
-      case environment.framework
-      when :rails then
-        install_instrument(ScoutApm::Instruments::ActionControllerRails2)
-      when :rails3_or_4 then
-        install_instrument(ScoutApm::Instruments::ActionControllerRails3Rails4)
-        install_instrument(ScoutApm::Instruments::RailsRouter)
-
-        if config.value("detailed_middleware")
-          install_instrument(ScoutApm::Instruments::MiddlewareDetailed)
-        else
+      if !background_job_missing?
+        case environment.background_job_name
+        when :delayed_job
+          install_instrument(ScoutApm::Instruments::DelayedJob)
+        end
+      else
+        case environment.framework
+        when :rails       then install_instrument(ScoutApm::Instruments::ActionControllerRails2)
+        when :rails3_or_4 then
+          install_instrument(ScoutApm::Instruments::ActionControllerRails3Rails4)
           install_instrument(ScoutApm::Instruments::MiddlewareSummary)
+          install_instrument(ScoutApm::Instruments::RailsRouter)
+        # when :sinatra     then install_instrument(ScoutApm::Instruments::Sinatra)
         end
       end
 
-      install_instrument(ScoutApm::Instruments::ActionView)
       install_instrument(ScoutApm::Instruments::ActiveRecord)
       install_instrument(ScoutApm::Instruments::Moped)
       install_instrument(ScoutApm::Instruments::Mongoid)
@@ -342,6 +325,10 @@ module ScoutApm
       instance.install
     end
 
+    def deploy_integration
+      environment.deploy_integration
+    end
+
     def app_server_missing?(options = {})
       !environment.app_server_integration(true).found? && !options[:skip_app_server_check]
     end
@@ -349,47 +336,5 @@ module ScoutApm
     def background_job_missing?(options = {})
       environment.background_job_integration.nil? && !options[:skip_background_job_check]
     end
-
-    def clear_recorder
-      @recorder = nil
-    end
-
-    def create_recorder
-      if @recorder
-        return @recorder
-      end
-
-      if config.value("async_recording")
-        logger.debug("Using asynchronous recording")
-        ScoutApm::BackgroundRecorder.new(logger).start
-      else
-        logger.debug("Using synchronous recording")
-        ScoutApm::SynchronousRecorder.new(logger).start
-      end
-    end
-
-    def start_remote_server(bind, port)
-      return if @remote_server && @remote_server.running?
-
-      logger.info("Starting Remote Agent Server")
-
-      # Start the listening web server only in parent process.
-      @remote_server = ScoutApm::Remote::Server.new(
-        bind,
-        port,
-        ScoutApm::Remote::Router.new(ScoutApm::SynchronousRecorder.new(logger), logger),
-        logger
-      )
-
-      @remote_server.start
-    end
-
-    # Execute this in the child process of a remote agent. The parent is
-    # expected to have its accepting webserver up and running
-    def use_remote_recorder(host, port)
-      logger.debug("Becoming Remote Agent (reporting to: #{host}:#{port})")
-      @recorder = ScoutApm::Remote::Recorder.new(host, port, logger)
-      @store = ScoutApm::FakeStore.new
-    end
   end
 end

data/lib/scout_apm/agent/logging.rb

@@ -6,12 +6,7 @@ module ScoutApm
         "#{environment.root}/log"
       end
 
-      def init_logger(opts={})
-        if opts[:force]
-          @log_file = nil
-          @logger = nil
-        end
-
+      def init_logger
         begin
           @log_file ||= determine_log_destination
         rescue => e

data/lib/scout_apm/agent/reporting.rb

@@ -24,7 +24,7 @@ module ScoutApm
         report_to_server
       end
 
-      # In a running app, one process will get the period ready for delivery, the others will see 0.
+      # In a running app, one process will get one period ready for delivery, the others will see 0.
       def report_to_server
         period_to_report = ScoutApm::StoreReportingPeriodTimestamp.minutes_ago(2)
 
@@ -58,7 +58,6 @@ module ScoutApm
         slow_transactions = reporting_period.slow_transactions_payload
         jobs = reporting_period.jobs
         slow_jobs = reporting_period.slow_jobs_payload
-        histograms = reporting_period.histograms
 
         metadata = {
           :app_root      => ScoutApm::Environment.instance.root.to_s,
@@ -69,10 +68,10 @@ module ScoutApm
           :platform      => "ruby",
         }
 
-        log_deliver(metrics, slow_transactions, metadata, slow_jobs, histograms)
+        log_deliver(metrics, slow_transactions, metadata, slow_jobs)
 
-        payload = ScoutApm::Serializers::PayloadSerializer.serialize(metadata, metrics, slow_transactions, jobs, slow_jobs, histograms)
-        logger.debug("Sending payload w/ Headers: #{headers.inspect}")
+        payload = ScoutApm::Serializers::PayloadSerializer.serialize(metadata, metrics, slow_transactions, jobs, slow_jobs)
+        # logger.debug("Payload: #{payload}")
 
         reporter.report(payload, headers)
       rescue => e
@@ -81,7 +80,7 @@ module ScoutApm
         logger.debug e.backtrace
       end
 
-      def log_deliver(metrics, slow_transactions, metadata, jobs_traces, histograms)
+      def log_deliver(metrics, slow_transactions, metadata, jobs_traces)
         total_request_count = metrics.
           select { |meta,stats| meta.metric_name =~ /\AController/ }.
           inject(0) {|sum, (_, stat)| sum + stat.call_count }
@@ -98,10 +97,9 @@ module ScoutApm
         metrics_clause    = "#{metrics.length} Metrics for #{total_request_count} requests"
         slow_trans_clause = "#{slow_transactions.length} Slow Transaction Traces"
         job_clause        = "#{jobs_traces.length} Job Traces"
-        histogram_clause  = "#{histograms.length} Histograms"
 
         logger.info "#{time_clause} Delivering #{metrics_clause} and #{slow_trans_clause} and #{job_clause}, #{process_log_str}."
-        logger.debug("\n\nMetrics: #{metrics.pretty_inspect}\nSlowTrans: #{slow_transactions.pretty_inspect}\nMetadata: #{metadata.inspect.pretty_inspect}\n\n")
+        # logger.debug("Metrics: #{metrics.pretty_inspect}\nSlowTrans: #{slow_transactions.pretty_inspect}\nMetadata: #{metadata.inspect.pretty_inspect}")
       end
 
       # TODO: Move this into PayloadSerializer?

data/lib/scout_apm/app_server_load.rb

@@ -27,30 +27,19 @@ module ScoutApm
     end
 
     def data
-      { :server_time        => to_s_safe(Time.now),
-        :framework          => to_s_safe(ScoutApm::Environment.instance.framework_integration.human_name),
-        :framework_version  => to_s_safe(ScoutApm::Environment.instance.framework_integration.version),
-        :environment        => to_s_safe(ScoutApm::Environment.instance.framework_integration.env),
-        :app_server         => to_s_safe(ScoutApm::Environment.instance.app_server),
+      { :server_time        => Time.now,
+        :framework          => ScoutApm::Environment.instance.framework_integration.human_name,
+        :framework_version  => ScoutApm::Environment.instance.framework_integration.version,
+        :environment        => ScoutApm::Environment.instance.framework_integration.env,
+        :app_server         => ScoutApm::Environment.instance.app_server,
         :ruby_version       => RUBY_VERSION,
-        :hostname           => to_s_safe(ScoutApm::Environment.instance.hostname),
-        :database_engine    => to_s_safe(ScoutApm::Environment.instance.database_engine),      # Detected
-        :database_adapter   => to_s_safe(ScoutApm::Environment.instance.raw_database_adapter), # Raw
-        :application_name   => to_s_safe(ScoutApm::Environment.instance.application_name),
+        :hostname           => ScoutApm::Environment.instance.hostname,
+        :database_engine    => ScoutApm::Environment.instance.database_engine,      # Detected
+        :database_adapter   => ScoutApm::Environment.instance.raw_database_adapter, # Raw
+        :application_name   => ScoutApm::Environment.instance.application_name,
         :libraries          => ScoutApm::Utils::InstalledGems.new.run,
-        :paas               => to_s_safe(ScoutApm::Environment.instance.platform_integration.name),
-        :git_sha            => to_s_safe(ScoutApm::Environment.instance.git_revision.sha)
+        :paas               => ScoutApm::Environment.instance.platform_integration.name
       }
     end
-
-    # Calls `.to_s` on the object passed in.
-    # Returns literal string 'to_s error' if the object does not respond to .to_s
-    def to_s_safe(obj)
-      if obj.respond_to?(:to_s)
-        obj.to_s
-      else
-        'to_s error'
-      end
-    end
   end
 end