scout_apm 1.6.8 → 2.0.0.pre

data/lib/scout_apm/config.rb

@@ -7,103 +7,147 @@ require 'scout_apm/environment'
 #
 # application_root - override the detected directory of the application
 # data_file        - override the default temporary storage location. Must be a location in a writable directory
-# host             - override the default hostname detection. Default varies by environment - either system hostname, or PAAS hostname
-# direct_host      - override the default "direct" host. The direct_host bypasses the ingestion pipeline and goes directly to the webserver, and is primarily used for features under development.
+# hostname         - override the default hostname detection. Default varies by environment - either system hostname, or PAAS hostname
 # key              - the account key with Scout APM. Found in Settings in the Web UI
 # log_file_path    - either a directory or "STDOUT".
 # log_level        - DEBUG / INFO / WARN as usual
 # monitor          - true or false.  False prevents any instrumentation from starting
 # name             - override the name reported to APM. This is the name that shows in the Web UI
 # uri_reporting    - 'path' or 'full_path' default is 'full_path', which reports URL params as well as the path.
-# report_format    - 'json' or 'marshal'. Marshal is legacy and will be removed.
+# report_format    - 'json' or 'marshal'. Json is default, Marshal is deprecated
 #
 # Any of these config settings can be set with an environment variable prefixed
 # by SCOUT_ and uppercasing the key: SCOUT_LOG_LEVEL for instance.
 
+
+# Config - Made up of config overlay
+# Default -> File -> Environment Var
+# QUESTION: How to embed arrays or hashes into ENV?
+
 module ScoutApm
   class Config
-    DEFAULTS =  {
-        'host'      => 'https://checkin.scoutapp.com',
-        'direct_host' => 'https://apm.scoutapp.com',
-        'log_level' => 'info',
-        'uri_reporting' => 'full_path',
-        'report_format' => 'json',
-        'disabled_instruments' => [],
-        'enable_background_jobs' => true,
-        'ignore_traces' => [],
-    }.freeze
-
-    def initialize(config_path = nil)
-      @config_path = config_path
+    # Load up a config instance without attempting to load a file.
+    # Useful for bootstrapping.
+    def self.without_file
+      overlays = [
+        ConfigEnvironment.new,
+        ConfigDefaults.new,
+      ]
+      new(overlays)
     end
 
-    def config_file_exists?
-      File.exist?(config_path)
+    # Load up a config instance, attempting to load a yaml file.  Allows a
+    # definite location if requested, or will attempt to load the default
+    # configuration file: APP_ROOT/config/scout_apm.yml
+    def self.with_file(file_path=nil)
+      overlays = [
+        ConfigEnvironment.new,
+        ConfigFile.new(file_path),
+        ConfigDefaults.new,
+      ]
+      new(overlays)
     end
 
-    # Fetch a config value.
-    # It first attempts to fetch an ENV var prefixed with 'SCOUT_',
-    # then from the settings file.
-    #
-    # If you set env_only, then it will not attempt to read the config file at
-    # all, and only read off the ENV var this is useful to break a loop during
-    # boot, where we needed an option to set the application root.
-    def value(key, env_only=false)
-      value = if env_only
-                ENV['SCOUT_' + key.upcase]
-              else
-                ENV['SCOUT_' + key.upcase] || setting(key)
-              end
-
-      value.to_s.strip.length.zero? ? nil : value
+    def initialize(overlays)
+      @overlays = overlays
     end
 
-    private
+    def value(key)
+      @overlays.each do |overlay|
+        if result = overlay.value(key)
+          return result
+        end
+      end
 
-    def config_path
-      @config_path || File.join(ScoutApm::Environment.instance.root, "config", "scout_apm.yml")
+      nil
     end
 
-    def config_file
-      File.expand_path(config_path)
-    end
+    class ConfigDefaults
+      DEFAULTS = {
+        'host'                   => 'https://checkin.scoutapp.com',
+        'log_level'              => 'info',
+        'stackprof_interval'     => 20000, # microseconds, 1000 = 1 millisecond, so 20k == 20 milliseconds
+        'uri_reporting'          => 'full_path',
+        'report_format'          => 'json',
+        'disabled_instruments'   => [],
+        'enable_background_jobs' => true,
+      }.freeze
 
-    def setting(key)
-      settings[key] || settings(true)[key]
+      def value(key)
+        DEFAULTS[key]
+      end
     end
 
-    def settings(try_reload=false)
-      (@settings.nil? || try_reload) ? @settings = load_file : @settings
+    class ConfigEnvironment
+      def value(key)
+        val = ENV['SCOUT_' + key.upcase]
+        val.to_s.strip.length.zero? ? nil : val
+      end
     end
 
-    def config_environment
-      @config_environment ||= ScoutApm::Environment.instance.env
-    end
+    # Attempts to load a configuration file, and parse it as YAML. If the file
+    # is not found, inaccessbile, or unparsable, log a message to that effect,
+    # and move on.
+    class ConfigFile
+      def initialize(file_path=nil)
+        @resolved_file_path = file_path || determine_file_path
+        load_file(@resolved_file_path)
+      end
 
-    def load_file
-      settings_hash = {}
-      begin
-        if File.exist?(config_file)
-          settings_hash = YAML.load(ERB.new(File.read(config_file)).result(binding))[config_environment] || {}
+      def value(key)
+        if @file_loaded
+          val = @settings[key]
+          val.to_s.strip.length.zero? ? nil : val
         else
-          logger.warn "No config file found at [#{config_file}]."
+          nil
         end
-      rescue Exception => e
-        logger.warn "Unable to load the config file."
-        logger.warn e.message
-        logger.warn e.backtrace
       end
-      DEFAULTS.merge(settings_hash)
-    end
 
-    # if we error out early enough, we don't have access to ScoutApm's logger
-    # in that case, be silent unless ENV['SCOUT_DEBUG'] is set, then STDOUT it
-    def logger
-      if defined?(ScoutApm::Agent) && (apm_log = ScoutApm::Agent.instance.logger)
-        apm_log
-      else
-        require 'scout_apm/utils/null_logger'
-        ENV['SCOUT_DEBUG'] ? Logger.new(STDOUT) : ScoutApm::Utils::NullLogger.new
+      private
+
+      def load_file(file)
+        if !File.exist?(@resolved_file_path)
+          logger.info("Configuration file #{file} does not exist, skipping.")
+          @file_loaded = false
+          return
+        end
+
+        if !app_environment
+          logger.info("Could not determine application environment, aborting configuration file load")
+          @file_loaded = false
+          return
+        end
+
+        begin
+          raw_file = File.read(@resolved_file_path)
+          erb_file = ERB.new(raw_file).result(binding)
+          parsed_yaml = YAML.load(erb_file)
+          @settings = parsed_yaml[app_environment]
+
+          if !@settings.is_a? Hash
+            raise ("Missing environment key for: #{app_environment}. This can happen if the key is missing, or with a malformed configuration file," +
+                   " check that there is a top level #{app_environment} key.")
+          end
+
+          logger.info("Loaded Configuration: #{@resolved_file_path}. Using environment: #{app_environment}")
+          @file_loaded = true
+        rescue Exception => e # Explicit `Exception` handling to catch SyntaxError and anything else that ERB or YAML may throw
+          logger.info("Failed loading configuration file: #{e.message}. ScoutAPM will continue starting with configuration from ENV and defaults")
+          @file_loaded = false
+        end
+      end
+
+      def determine_file_path
+        File.join(ScoutApm::Environment.instance.root, "config", "scout_apm.yml")
+      end
+
+      def app_environment
+        ScoutApm::Environment.instance.env
+      end
+
+      # TODO: Make this better
+      def logger
+        ScoutApm::Agent.instance.logger || Logger.new(STDOUT)
       end
     end
   end

data/lib/scout_apm/environment.rb

@@ -70,10 +70,6 @@ module ScoutApm
       framework_integration.database_engine
     end
 
-    def raw_database_adapter
-      framework_integration.raw_database_adapter
-    end
-
     def processors
       @processors ||= begin
                         proc_file = '/proc/cpuinfo'
@@ -88,12 +84,11 @@ module ScoutApm
     end
 
     def root
-      return deploy_integration.root if deploy_integration
-      framework_root
+      @root ||= deploy_integration? ? deploy_integration.root : framework_root
     end
 
     def framework_root
-      if override_root = Agent.instance.config.value("application_root", true)
+      if override_root = Agent.instance.config.value("application_root")
         return override_root
       end
       if framework == :rails
@@ -108,8 +103,7 @@ module ScoutApm
     end
 
     def hostname
-      config_hostname = Agent.instance.config.value("hostname", !Agent.instance.config.config_file_exists?)
-      @hostname ||= config_hostname || platform_integration.hostname
+      @hostname ||= Agent.instance.config.value("hostname") || platform_integration.hostname
     end
 
     # Returns the whole integration object
@@ -134,7 +128,7 @@ module ScoutApm
     end
 
     def background_job_integration
-      if Agent.instance.config.value("enable_background_jobs", !Agent.instance.config.config_file_exists?)
+      if Agent.instance.config.value("enable_background_jobs")
         @background_job_integration ||= BACKGROUND_JOB_INTEGRATIONS.detect {|integration| integration.present?}
       else
         nil
@@ -175,6 +169,18 @@ module ScoutApm
       @ruby_2 ||= defined?(RUBY_VERSION) && RUBY_VERSION.match(/^2/)
     end
 
+    # Returns a string representation of the OS (ex: darwin, linux)
+    def os
+      return @os if @os
+      raw_os = RbConfig::CONFIG['target_os']
+      match = raw_os.match(/([a-z]+)/)
+      if match
+        @os = match[1]
+      else
+        @os = raw_os
+      end
+    end
+
     ### framework checks
 
     def sinatra?

data/lib/scout_apm/framework_integrations/rails_2.rb

@@ -37,14 +37,18 @@ module ScoutApm
         default = :mysql
 
         if defined?(ActiveRecord::Base)
-          case raw_database_adapter
-          when "postgres"   then :postgres
-          when "postgresql" then :postgres
-          when "postgis"    then :postgres
-          when "sqlite3"    then :sqlite
-          when "mysql"      then :mysql
-          when "mysql2"     then :mysql
-          else default
+          config = ActiveRecord::Base.configurations[env]
+          if config && config["adapter"]
+            case config["adapter"].to_s
+            when "postgres"   then :postgres
+            when "postgresql" then :postgres
+            when "postgis"    then :postgres
+            when "sqlite3"    then :sqlite
+            when "mysql"      then :mysql
+            else default
+            end
+          else
+            default
           end
         else
           default
@@ -52,12 +56,6 @@ module ScoutApm
       rescue
         default
       end
-
-      def raw_database_adapter
-        ActiveRecord::Base.configurations[env]["adapter"]
-      rescue
-        nil
-      end
     end
   end
 end

data/lib/scout_apm/framework_integrations/rails_3_or_4.rb

@@ -34,19 +34,17 @@ module ScoutApm
 
       def database_engine
         return @database_engine if @database_engine
-        default = :postgres
+        default = :mysql
 
         @database_engine = if defined?(ActiveRecord::Base)
-          adapter = raw_database_adapter # can be nil
+          adapter = get_database_adapter # can be nil
 
           case adapter.to_s
           when "postgres"   then :postgres
           when "postgresql" then :postgres
           when "postgis"    then :postgres
           when "sqlite3"    then :sqlite
-          when "sqlite"     then :sqlite
           when "mysql"      then :mysql
-          when "mysql2"     then :mysql
           else default
           end
         else
@@ -55,21 +53,11 @@ module ScoutApm
         end
       end
 
-      def raw_database_adapter
-        adapter = if ActiveRecord::Base.respond_to?(:connection_config)
-                    ActiveRecord::Base.connection_config[:adapter].to_s
-                  else
-                    nil
-                  end
-
-        if adapter.nil?
-          adapter = ActiveRecord::Base.configurations[env]["adapter"]
-        end
-
-        return adapter
+      def get_database_adapter
+        ActiveRecord::Base.configurations[env]["adapter"]
       rescue # this would throw an exception if ActiveRecord::Base is defined but no configuration exists.
         nil
       end
     end
   end
-end
+end

data/lib/scout_apm/framework_integrations/ruby.rb

@@ -30,10 +30,6 @@ module ScoutApm
       def database_engine
         :mysql
       end
-
-      def raw_database_adapter
-        :mysql
-      end
     end
   end
 end

data/lib/scout_apm/framework_integrations/sinatra.rb

@@ -37,10 +37,6 @@ module ScoutApm
       def database_engine
         :mysql
       end
-
-      def raw_database_adapter
-        :mysql
-      end
     end
   end
 end

data/lib/scout_apm/histogram.rb

@@ -56,26 +56,6 @@ module ScoutApm
       end
     end
 
-    # Given a value, where in this histogram does it fall?
-    # Returns a float between 0 and 1
-    def approximate_quantile_of_value(v)
-      mutex.synchronize do
-        return 100 if total == 0
-
-        count_examined = 0
-
-        bins.each_with_index do |bin, index|
-          if v <= bin.value
-            break
-          end
-
-          count_examined += bin.count
-        end
-
-        count_examined / total.to_f
-      end
-    end
-
     def mean
       mutex.synchronize do
         if total == 0

data/lib/scout_apm/instruments/action_controller_rails_3_rails4.rb

@@ -61,10 +61,7 @@ module ScoutApm
         req = ScoutApm::RequestManager.lookup
         path = ScoutApm::Agent.instance.config.value("uri_reporting") == 'path' ? request.path : request.fullpath
         req.annotate_request(:uri => path)
-
-        # IP Spoofing Protection can throw an exception, just move on w/o remote ip
-        req.context.add_user(:ip => request.remote_ip) rescue nil
-
+        req.context.add_user(:ip => request.remote_ip)
         req.set_headers(request.headers)
         req.web!
 

data/lib/scout_apm/instruments/active_record.rb

@@ -27,14 +27,44 @@ module ScoutApm
       end
 
       def add_instruments
-        if defined?(::ActiveRecord) && defined?(::ActiveRecord::Base)
+        # Setup Tracer on AR::Base
+        if Utils::KlassHelper.defined?("ActiveRecord::Base")
+          ::ActiveRecord::Base.class_eval do
+            include ::ScoutApm::Tracer
+          end
+        end
+
+        # Install #log tracing
+        if Utils::KlassHelper.defined?("ActiveRecord::ConnectionAdapters::AbstractAdapter")
           ::ActiveRecord::ConnectionAdapters::AbstractAdapter.module_eval do
             include ::ScoutApm::Instruments::ActiveRecordInstruments
             include ::ScoutApm::Tracer
           end
+        end
+
+        if Utils::KlassHelper.defined?("ActiveRecord::Querying")
+          ::ActiveRecord::Querying.module_eval do
+            include ::ScoutApm::Tracer
+            include ::ScoutApm::Instruments::ActiveRecordQueryingInstruments
+          end
+        end
 
-          ::ActiveRecord::Base.class_eval do
+        if Utils::KlassHelper.defined?("ActiveRecord::FinderMethods")
+          ::ActiveRecord::FinderMethods.module_eval do
             include ::ScoutApm::Tracer
+            include ::ScoutApm::Instruments::ActiveRecordFinderMethodsInstruments
+          end
+        end
+
+        if Utils::KlassHelper.defined?("ActiveSupport::Notifications")
+          ActiveSupport::Notifications.subscribe("instantiation.active_record") do |event_name, start, stop, uuid, payload|
+            req = ScoutApm::RequestManager.lookup
+            layer = req.current_layer
+            if layer && layer.type == "ActiveRecord"
+              layer.annotate_layer(payload)
+            else
+              ScoutApm::Agent.instance.logger.debug("Expected layer type: ActiveRecord, got #{layer && layer.type}")
+            end
           end
         end
       rescue
@@ -44,6 +74,13 @@ module ScoutApm
 
     # Contains ActiveRecord instrument, aliasing +ActiveRecord::ConnectionAdapters::AbstractAdapter#log+ calls
     # to trace calls to the database.
+    ################################################################################
+    # #log instrument.
+    #
+    # #log is very close to where AR calls out to the database itself.  We have access
+    # to the real SQL, and an AR generated "name" for the Query
+    #
+    ################################################################################
     module ActiveRecordInstruments
       def self.included(instrumented_class)
         ScoutApm::Agent.instance.logger.info "Instrumenting #{instrumented_class.inspect}"
@@ -51,19 +88,123 @@ module ScoutApm
           unless instrumented_class.method_defined?(:log_without_scout_instruments)
             alias_method :log_without_scout_instruments, :log
             alias_method :log, :log_with_scout_instruments
-            protected :log
           end
         end
-      end # self.included
+      end
 
       def log_with_scout_instruments(*args, &block)
+        # Extract data from the arguments
         sql, name = args
-        self.class.instrument("ActiveRecord",
-                              Utils::ActiveRecordMetricName.new(sql, name),
-                              :desc => Utils::SqlSanitizer.new(sql) ) do
+        metric_name = Utils::ActiveRecordMetricName.new(sql, name)
+        desc = Utils::SqlSanitizer.new(sql)
+
+        # Get current ScoutApm context
+        req = ScoutApm::RequestManager.lookup
+        current_layer = req.current_layer
+
+
+        # If we call #log, we have a real query to run, and we've already
+        # gotten through the cache gatekeeper. Since we want to only trace real
+        # queries, and not repeated identical queries that just hit cache, we
+        # mark layer as ignorable initially in #find_by_sql, then only when we
+        # know it's a real database call do we mark it back as usable.
+        #
+        # This flag is later used in SlowRequestConverter to skip adding ignorable layers
+        current_layer.annotate_layer(:ignorable => false) if current_layer
+
+        # Either: update the current layer and yield, don't start a new one.
+        if current_layer && current_layer.type == "ActiveRecord"
+          # TODO: Get rid of call .to_s, need to find this without forcing a previous run of the name logic
+          if current_layer.name.to_s == Utils::ActiveRecordMetricName::DEFAULT_METRIC
+            current_layer.name = metric_name
+            current_layer.desc = desc
+          end
+
           log_without_scout_instruments(sql, name, &block)
+
+        # OR: Start a new layer, we didn't pick up instrumentation earlier in the stack.
+        else
+          layer = ScoutApm::Layer.new("ActiveRecord", metric_name)
+          layer.desc = desc
+          req.start_layer(layer)
+          begin
+            log_without_scout_instruments(sql, name, &block)
+          ensure
+            req.stop_layer
+          end
+        end
+      end
+    end
+
+    ################################################################################
+    # Entry-point of instruments.
+    #
+    # We instrument both ActiveRecord::Querying#find_by_sql and
+    # ActiveRecord::FinderMethods#find_with_associations.  These are early in
+    # the chain of calls when you're using ActiveRecord.
+    #
+    # Later on, they will call into #log, which we also instrument, at which
+    # point, we can fill in additional data gathered at that point (name, sql)
+    #
+    # Caveats:
+    #   * We don't have a name for the query yet.
+    #   * The query hasn't hit the cache yet. In the case of a cache hit, we
+    #     won't hit #log, so won't get a name, leaving the misleading default.
+    #   * One call here can result in several calls to #log, especially in the
+    #     case where Rails needs to load the schema details for the table being
+    #     queried.
+    ################################################################################
+
+    module ActiveRecordQueryingInstruments
+      def self.included(instrumented_class)
+        ScoutApm::Agent.instance.logger.info "Instrumenting ActiveRecord::Querying - #{instrumented_class.inspect}"
+        instrumented_class.class_eval do
+          unless instrumented_class.method_defined?(:find_by_sql_without_scout_instruments)
+            alias_method :find_by_sql_without_scout_instruments, :find_by_sql
+            alias_method :find_by_sql, :find_by_sql_with_scout_instruments
+          end
+        end
+      end
+
+      def find_by_sql_with_scout_instruments(*args, &block)
+        req = ScoutApm::RequestManager.lookup
+        layer = ScoutApm::Layer.new("ActiveRecord", Utils::ActiveRecordMetricName::DEFAULT_METRIC)
+        layer.annotate_layer(:ignorable => true)
+        req.start_layer(layer)
+        req.ignore_children!
+        begin
+          find_by_sql_without_scout_instruments(*args)
+        ensure
+          req.acknowledge_children!
+          req.stop_layer
         end
       end
-    end # module ActiveRecordInstruments
+    end
+
+    module ActiveRecordFinderMethodsInstruments
+      def self.included(instrumented_class)
+        ScoutApm::Agent.instance.logger.info "Instrumenting ActiveRecord::FinderMethods - #{instrumented_class.inspect}"
+        instrumented_class.class_eval do
+          unless instrumented_class.method_defined?(:find_with_associations_without_scout_instruments)
+            alias_method :find_with_associations_without_scout_instruments, :find_with_associations
+            alias_method :find_with_associations, :find_with_associations_with_scout_instruments
+          end
+        end
+      end
+
+      def find_with_associations_with_scout_instruments(*args, &block)
+        req = ScoutApm::RequestManager.lookup
+        layer = ScoutApm::Layer.new("ActiveRecord", Utils::ActiveRecordMetricName::DEFAULT_METRIC)
+        layer.annotate_layer(:ignorable => true)
+        req.start_layer(layer)
+        req.ignore_children!
+        begin
+          find_with_associations_without_scout_instruments(*args)
+        ensure
+          req.acknowledge_children!
+          req.stop_layer
+        end
+      end
+    end
   end
 end