scout_apm 0.1

data/lib/scout_apm/agent/logging.rb

@@ -0,0 +1,44 @@
+# Contains methods specific to logging (initializing the log file, applying the log level, applying the log format, etc.)
+module ScoutApm
+  class Agent
+    module Logging
+      def log_path
+        "#{environment.root}/log"
+      end
+      
+      def init_logger
+        @log_file = "#{log_path}/scout_apm.log"
+        begin 
+          @logger = Logger.new(@log_file) 
+          @logger.level = log_level
+          apply_log_format
+        rescue Exception => e
+          @logger = Logger.new(STDOUT)
+          apply_log_format
+          @logger.error "Unable to access log file: #{e.message}"
+        end
+        @logger
+      end
+
+      def apply_log_format
+        def logger.format_message(severity, timestamp, progname, msg)
+          # since STDOUT isn't exclusive like the scout_apm.log file, apply a prefix.
+          prefix = @logdev.dev == STDOUT ? "scout_apm " : ''
+          prefix + "[#{timestamp.strftime("%m/%d/%y %H:%M:%S %z")} #{Socket.gethostname} (#{$$})] #{severity} : #{msg}\n"
+        end
+      end
+
+      def log_level
+        case config.settings['log_level'].downcase
+          when "debug" then Logger::DEBUG
+          when "info" then Logger::INFO
+          when "warn" then Logger::WARN
+          when "error" then Logger::ERROR
+          when "fatal" then Logger::FATAL
+          else Logger::INFO
+        end
+      end
+    end # module Logging
+    include Logging
+  end # class Agent
+end # moudle ScoutApm

data/lib/scout_apm/agent/reporting.rb

@@ -0,0 +1,110 @@
+# Methods related to sending metrics to scoutapp.com.
+module ScoutApm
+  class Agent
+    module Reporting
+      CA_FILE     = File.join( File.dirname(__FILE__), *%w[.. .. .. data cacert.pem] )
+      VERIFY_MODE = OpenSSL::SSL::VERIFY_PEER | OpenSSL::SSL::VERIFY_FAIL_IF_NO_PEER_CERT
+
+      # Called in the worker thread. Merges in-memory metrics w/those on disk and reports metrics
+      # to the server.
+      def process_metrics
+        logger.debug "Processing metrics"
+        run_samplers
+        payload = layaway.deposit_and_deliver
+        metrics = payload[:metrics]
+        slow_transactions = payload[:slow_transactions]
+        if payload.any?
+          add_metric_ids(metrics)  
+          logger.warn "Some data may be lost - metric size is at limit" if metrics.size == ScoutApm::Store::MAX_SIZE
+          # for debugging, count the total number of requests    
+          controller_count = 0
+          metrics.each do |meta,stats|
+            if meta.metric_name =~ /\AController/
+              controller_count += stats.call_count
+            end
+          end      
+          payload = Marshal.dump(:metrics => metrics, :slow_transactions => slow_transactions)
+          slow_transactions_kb = Marshal.dump(slow_transactions).size/1024 # just for performance debugging
+          logger.debug "#{config.settings['name']} Delivering total payload [#{payload.size/1024} KB] for #{controller_count} requests and slow transactions [#{slow_transactions_kb} KB] for #{slow_transactions.size} transactions of durations: #{slow_transactions.map(&:total_call_time).join(',')}."        
+          response =  post( checkin_uri,
+                             payload,
+                             "Content-Type"     => "application/octet-stream" )
+          if response and response.is_a?(Net::HTTPSuccess)
+            directives = Marshal.load(response.body)
+            self.metric_lookup.merge!(directives[:metric_lookup])
+            if directives[:reset]
+              logger.info "Resetting metric_lookup."
+              self.metric_lookup = Hash.new
+            end
+            logger.debug "Metric Cache Size: #{metric_lookup.size}"
+          elsif response
+            logger.warn "Error on checkin to #{checkin_uri.to_s}: #{response.inspect}"
+          end
+        end
+      rescue
+        logger.warn "Error on checkin to #{checkin_uri.to_s}"
+        logger.info $!.message
+        logger.debug $!.backtrace
+      end
+      
+      # Before reporting, lookup metric_id for each MetricMeta. This speeds up 
+      # reporting on the server-side.
+      def add_metric_ids(metrics)
+        metrics.each do |meta,stats|
+          if metric_id = metric_lookup[meta]
+            meta.metric_id = metric_id
+          end
+        end
+      end
+      
+      def checkin_uri
+        URI.parse("#{config.settings['host']}/apps/checkin.scout?key=#{config.settings['key']}&name=#{CGI.escape(config.settings['name'])}")
+      end
+
+      def post(uri, body, headers = Hash.new)
+        response = nil
+        request(uri) do |connection|
+          post = Net::HTTP::Post.new( uri.path +
+                                      (uri.query ? ('?' + uri.query) : ''),
+                                      HTTP_HEADERS.merge(headers) )
+          post.body = body
+          response=connection.request(post)
+        end
+        response
+      end
+
+      def request(uri, &connector)
+        response           = nil
+        response           = http(uri).start(&connector)
+        logger.debug "got response: #{response.inspect}"
+        case response
+        when Net::HTTPSuccess, Net::HTTPNotModified
+          logger.debug "/checkin OK"
+        when Net::HTTPBadRequest
+          logger.warn "/checkin FAILED: The Account Key [#{config.settings['key']}] is invalid."
+        else
+          logger.debug "/checkin FAILED: #{response.inspect}"
+        end
+      rescue Exception
+        logger.debug "Exception sending request to server: #{$!.message}\n#{$!.backtrace}"
+      ensure
+        response
+      end
+
+      # Take care of the http proxy, if specified in config.
+      # Given a blank string, the proxy_uri URI instance's host/port/user/pass will be nil.
+      # Net::HTTP::Proxy returns a regular Net::HTTP class if the first argument (host) is nil.
+      def http(url)
+        proxy_uri = URI.parse(config.settings['proxy'].to_s)
+        http = Net::HTTP::Proxy(proxy_uri.host,proxy_uri.port,proxy_uri.user,proxy_uri.password).new(url.host, url.port)
+        if url.is_a?(URI::HTTPS)
+          http.use_ssl = true
+          http.ca_file = CA_FILE
+          http.verify_mode = VERIFY_MODE
+        end
+        http
+      end
+    end # module Reporting
+    include Reporting
+  end # class Agent
+end # module ScoutApm

data/lib/scout_apm/agent.rb

@@ -0,0 +1,217 @@
+module ScoutApm
+  # The agent gathers performance data from a Ruby application. One Agent instance is created per-Ruby process. 
+  #
+  # Each Agent object creates a worker thread (unless monitoring is disabled or we're forking). 
+  # The worker thread wakes up every +Agent#period+, merges in-memory metrics w/those saved to disk, 
+  # saves tshe merged data to disk, and sends it to the Scout server.
+  class Agent
+    # Headers passed up with all API requests.
+    HTTP_HEADERS = { "Agent-Hostname" => Socket.gethostname }
+    # see self.instance
+    @@instance = nil 
+    
+    # Accessors below are for associated classes
+    attr_accessor :store
+    attr_accessor :layaway
+    attr_accessor :config
+    attr_accessor :environment
+    
+    attr_accessor :logger
+    attr_accessor :log_file # path to the log file
+    attr_accessor :options # options passed to the agent when +#start+ is called.
+    attr_accessor :metric_lookup # Hash used to lookup metric ids based on their name and scope
+    
+    # All access to the agent is thru this class method to ensure multiple Agent instances are not initialized per-Ruby process. 
+    def self.instance(options = {})
+      @@instance ||= self.new(options)
+    end
+    
+    # Note - this doesn't start instruments or the worker thread. This is handled via +#start+ as we don't 
+    # want to start the worker thread or install instrumentation if (1) disabled for this environment (2) a worker thread shouldn't
+    # be started (when forking).
+    def initialize(options = {})
+      @started = false
+      @options ||= options
+      @store = ScoutApm::Store.new
+      @layaway = ScoutApm::Layaway.new
+      @config = ScoutApm::Config.new(options[:config_path])
+      @metric_lookup = Hash.new
+      @process_cpu=ScoutApm::Instruments::Process::ProcessCpu.new(environment.processors)
+      @process_memory=ScoutApm::Instruments::Process::ProcessMemory.new
+    end
+    
+    def environment
+      @environment ||= ScoutApm::Environment.new
+    end
+    
+    # This is called via +ScoutApm::Agent.instance.start+ when ScoutApm is required in a Ruby application.
+    # It initializes the agent and starts the worker thread (if appropiate).
+    def start(options = {})
+      @options.merge!(options)
+      init_logger
+      logger.info "Attempting to start Scout Agent [#{ScoutApm::VERSION}] on [#{Socket.gethostname}]"
+      if !config.settings['monitor'] and !@options[:force]
+        logger.warn "Monitoring isn't enabled for the [#{environment.env}] environment."
+        return false
+      elsif !environment.app_server
+        logger.warn "Couldn't find a supported app server. Not starting agent."
+        return false
+      elsif started?
+        logger.warn "Already started agent."
+        return false
+      end
+      @started = true
+      logger.info "Starting monitoring. Framework [#{environment.framework}] App Server [#{environment.app_server}]."
+      start_instruments
+      if !start_background_worker?
+        logger.debug "Not starting worker thread"
+        install_passenger_events if environment.app_server == :passenger
+        install_unicorn_worker_loop if environment.app_server == :unicorn
+        install_rainbows_worker_loop if environment.app_server == :rainbows
+        return
+      end
+      start_background_worker
+      handle_exit
+      logger.info "Scout Agent [#{ScoutApm::VERSION}] Initialized"
+    end
+    
+    # at_exit, calls Agent#shutdown to wrapup metric reporting.
+    def handle_exit
+      if environment.sinatra? || environment.jruby? || environment.rubinius?
+        logger.debug "Exit handler not supported"
+      else
+        at_exit do 
+          logger.debug "Shutdown!"
+          # MRI 1.9 bug drops exit codes.
+          # http://bugs.ruby-lang.org/issues/5218
+          if environment.ruby_19?
+            status = $!.status if $!.is_a?(SystemExit)
+            shutdown
+            exit status if status
+          else
+            shutdown
+          end
+        end # at_exit
+      end
+    end
+    
+    # Called via an at_exit handler, it (1) stops the background worker and (2) runs it a final time. 
+    # The final run ensures metrics are stored locally to the layaway / reported to scoutapp.com. Otherwise,
+    # in-memory metrics would be lost and a gap would appear on restarts.
+    def shutdown
+      return if !started?
+      @background_worker.stop
+      @background_worker.run_once
+    end
+    
+    def started?
+      @started
+    end
+    
+    def gem_root
+      File.expand_path(File.join("..","..",".."), __FILE__)
+    end
+    
+    # The worker thread will automatically start UNLESS:
+    # * A supported application server isn't detected (example: running via Rails console)
+    # * A supported application server is detected, but it forks (Passenger). In this case, 
+    #   the agent is started in the forked process.
+    def start_background_worker?
+      !environment.forking? or environment.app_server == :thin
+    end
+    
+    def install_passenger_events
+      PhusionPassenger.on_event(:starting_worker_process) do |forked|
+        logger.debug "Passenger is starting a worker process. Starting worker thread."
+        self.class.instance.start_background_worker
+      end
+      # The agent's at_exit hook doesn't run when a Passenger process stops. 
+      # This does run when a process stops.
+      PhusionPassenger.on_event(:stopping_worker_process) do
+        logger.debug "Passenger is stopping a worker process, shutting down the agent."
+        ScoutApm::Agent.instance.shutdown
+      end
+    end
+    
+    def install_unicorn_worker_loop
+      logger.debug "Installing Unicorn worker loop."
+      Unicorn::HttpServer.class_eval do
+        old = instance_method(:worker_loop)
+        define_method(:worker_loop) do |worker|
+          ScoutApm::Agent.instance.start_background_worker
+          old.bind(self).call(worker)
+        end
+      end
+    end
+    
+    def install_rainbows_worker_loop
+      logger.debug "Installing Rainbows worker loop."
+      Rainbows::HttpServer.class_eval do
+        old = instance_method(:worker_loop)
+        define_method(:worker_loop) do |worker|
+          ScoutApm::Agent.instance.start_background_worker
+          old.bind(self).call(worker)
+        end
+      end
+    end    
+    
+    # Creates the worker thread. The worker thread is a loop that runs continuously. It sleeps for +Agent#period+ and when it wakes,
+    # processes data, either saving it to disk or reporting to Scout.
+    def start_background_worker
+      logger.debug "Creating worker thread."
+      @background_worker = ScoutApm::BackgroundWorker.new
+      @background_worker_thread = Thread.new do
+        @background_worker.start { process_metrics }
+      end # thread new
+      logger.debug "Done creating worker thread."
+    end
+    
+    # Called from #process_metrics, which is run via the background worker. 
+    def run_samplers
+      begin
+        cpu_util=@process_cpu.run # returns a hash
+        logger.debug "Process CPU: #{cpu_util.inspect} [#{environment.processors} CPU(s)]"
+        store.track!("CPU/Utilization",cpu_util,:scope => nil) if cpu_util
+      rescue => e
+        logger.info "Error reading ProcessCpu"
+        logger.debug e.message
+        logger.debug e.backtrace.join("\n")
+      end
+
+      begin
+        mem_usage=@process_memory.run # returns a single number, in MB
+        logger.debug "Process Memory: #{mem_usage}MB"
+        store.track!("Memory/Physical",mem_usage,:scope => nil) if mem_usage
+      rescue => e
+        logger.info "Error reading ProcessMemory"
+        logger.debug e.message
+        logger.debug e.backtrace.join("\n")
+      end
+    end
+    
+    # Loads the instrumention logic.
+    def load_instruments
+      case environment.framework
+      when :rails
+        require File.expand_path(File.join(File.dirname(__FILE__),'instruments/rails/action_controller_instruments.rb'))
+      when :rails3_or_4
+        require File.expand_path(File.join(File.dirname(__FILE__),'instruments/rails3_or_4/action_controller_instruments.rb'))
+      end
+      require File.expand_path(File.join(File.dirname(__FILE__),'instruments/active_record_instruments.rb'))
+      require File.expand_path(File.join(File.dirname(__FILE__),'instruments/net_http.rb'))
+      require File.expand_path(File.join(File.dirname(__FILE__),'instruments/moped_instruments.rb'))
+      require File.expand_path(File.join(File.dirname(__FILE__),'instruments/mongoid_instruments.rb'))
+    rescue
+      logger.warn "Exception loading instruments:"
+      logger.warn $!.message
+      logger.warn $!.backtrace
+    end
+    
+    # Injects instruments into the Ruby application.
+    def start_instruments
+      logger.debug "Installing instrumentation"
+      load_instruments
+    end
+    
+  end # class Agent
+end # module ScoutApm

data/lib/scout_apm/background_worker.rb

@@ -0,0 +1,43 @@
+# Used to run a given task every 60 seconds. 
+class ScoutApm::BackgroundWorker
+  # in seconds, time between when the worker thread wakes up and runs.
+  PERIOD = 60
+  
+  def initialize
+    @keep_running = true
+  end
+  
+  def stop
+    @keep_running = false
+  end
+  
+  # Runs the task passed to +start+ once.
+  def run_once
+    @task.call if @task
+  end
+  
+  # Starts running the passed block every 60 seconds (starting now).
+  def start(&block)
+    @task = block
+    begin
+      ScoutApm::Agent.instance.logger.debug "Starting Background Worker, running every #{PERIOD} seconds"
+      next_time = Time.now
+      while @keep_running do
+        now = Time.now
+        while now < next_time
+          sleep_time = next_time - now
+          sleep(sleep_time) if sleep_time > 0
+          now = Time.now
+        end
+        @task.call
+        while next_time <= now
+          next_time += PERIOD
+        end
+      end
+    rescue
+      ScoutApm::Agent.instance.logger.debug "Background Worker Exception!!!!!!!"
+      ScoutApm::Agent.instance.logger.debug $!.message
+      ScoutApm::Agent.instance.logger.debug $!.backtrace
+    end
+  end
+end

data/lib/scout_apm/config.rb

@@ -0,0 +1,42 @@
+module ScoutApm
+  class Config   
+    DEFAULTS =  {
+        'host' => 'https://apm.scoutapp.com',
+        'log_level' => 'info'
+    }
+
+    def initialize(config_path = nil)
+      @config_path = config_path
+    end
+    
+    def settings
+      return @settings if @settings
+      load_file
+    end
+    
+    def config_path
+      @config_path || File.join(ScoutApm::Agent.instance.environment.root,"config","scout_apm.yml")
+    end
+    
+    def config_file
+      File.expand_path(config_path)
+    end
+    
+    def load_file
+      begin
+        if !File.exist?(config_file)
+          ScoutApm::Agent.instance.logger.warn "No config file found at [#{config_file}]."
+          @settings = {}
+        else
+          @settings = YAML.load(ERB.new(File.read(config_file)).result(binding))[ScoutApm::Agent.instance.environment.env] || {} 
+        end  
+      rescue Exception => e
+        ScoutApm::Agent.instance.logger.warn "Unable to load the config file."
+        ScoutApm::Agent.instance.logger.warn e.message
+        ScoutApm::Agent.instance.logger.warn e.backtrace
+        @settings = {}
+      end
+      @settings = DEFAULTS.merge(@settings)
+    end
+  end # Config
+end # ScoutApm

data/lib/scout_apm/context.rb

@@ -0,0 +1,105 @@
+# Encapsulates adding context to requests. Context is stored via a simple Hash.
+#
+# There are 2 types of context: User and Extra.
+# For user-specific context, use @Context#add_user@.
+# For misc context, use @Context#add@.
+class ScoutApm::Context
+
+  def initialize
+    @extra = {}
+    @user = {}
+  end
+
+  # Generates a hash representation of the Context. 
+  # Example: {:monthly_spend => 100, :user => {:ip => '127.0.0.1'}}
+  def to_hash
+    @extra.merge({:user => @user})
+  end
+
+  def self.current
+    Thread.current[:scout_context] ||= new
+  end
+
+  def self.clear!
+    Thread.current[:scout_context] = nil
+  end
+
+  # Add context
+  # ScoutApm::Context.add(account: current_account.name)
+  def add(hash)
+    update_context(:extra,hash)
+  end
+
+  def add_user(hash)
+    update_context(:user,hash)
+  end
+
+  # Convenience accessor so you can just call @ScoutAPM::Context#add@
+  def self.add(hash)
+    self.current.add(hash)
+  end
+
+  # Convenience accessor so you can just call @ScoutAPM::Context#add_user@
+  def self.add_user(hash)
+    self.current.add_user(hash)
+  end
+
+  private
+
+  def update_context(attr,hash)
+    valid_hash = Hash.new
+    # iterate over the hash of new context, adding to the valid_hash if validation checks pass.
+    hash.each do |key,value|
+      # does both checks so we can get logging info on the value even if the key is invalid.
+      key_valid = key_valid?({key => value})
+      value_valid = value_valid?({key => value})
+      if key_valid and value_valid
+        valid_hash[key] = value
+      end
+    end
+
+    if valid_hash.any?
+      instance_variable_get("@#{attr.to_s}").merge!(valid_hash)
+    end
+  end
+
+  # Returns true if the obj is one of the provided valid classes.
+  def valid_type?(classes, obj)
+    valid_type = false
+    classes.each do |klass|
+      if obj.is_a?(klass)
+        valid_type = true
+        break
+      end
+    end
+    valid_type
+  end
+
+  # take the entire Hash vs. just the value so the logger output is more helpful on error.
+  def value_valid?(key_value)
+    # ensure one of our accepted types. 
+    value = key_value.values.last
+    if !valid_type?([String, Symbol, Numeric, Time, Date, TrueClass, FalseClass],value)
+      ScoutApm::Agent.instance.logger.warn "The value for [#{key_value.keys.first}] is not a valid type [#{value.class}]."
+     false
+    else
+      true
+    end
+  end
+
+  # for consistently with #value_valid?, takes a hash eventhough the value isn't yet used.
+  def key_valid?(key_value)
+    key = key_value.keys.first
+    # ensure a string or a symbol
+    if !valid_type?([String, Symbol],key)
+      ScoutApm::Agent.instance.logger.warn "The key [#{key}] is not a valid type [#{key.class}]."
+      return false
+    end
+    # only alphanumeric, dash, and underscore allowed.
+    if key.to_s.match(/[^\w-]/)
+      ScoutApm::Agent.instance.logger.warn "They key name [#{key}] is not valid."
+      return false
+    end
+    true
+  end
+end

data/lib/scout_apm/environment.rb

@@ -0,0 +1,135 @@
+# Used to retrieve environment information for this application.
+module ScoutApm
+  class Environment
+    def env
+      @env ||= case framework
+               when :rails then RAILS_ENV.dup
+               when :rails3_or_4 then Rails.env
+               when :sinatra
+                 ENV['RACK_ENV'] || ENV['RAILS_ENV'] || 'development'
+               end
+    end
+    
+    def framework
+      @framework ||= case
+                      when defined?(::Rails) && defined?(ActionController)
+                        if Rails::VERSION::MAJOR < 3
+                          :rails
+                        else
+                          :rails3_or_4
+                        end
+                      when defined?(::Sinatra) && defined?(::Sinatra::Base) then :sinatra
+                      else :ruby
+                      end
+    end
+    
+    def processors
+      return @processors if @processors
+      unless @processors
+        proc_file = '/proc/cpuinfo'
+        if !File.exist?(proc_file)
+          @processors = 1
+        elsif `cat #{proc_file} | grep 'model name' | wc -l` =~ /(\d+)/
+          @processors = $1.to_i
+        end
+        if @processors < 1
+          @processors = 1
+        end 
+      end
+      @processors
+    end
+    
+    def root
+      if framework == :rails
+        RAILS_ROOT.to_s
+      elsif framework == :rails3_or_4
+        Rails.root
+      elsif framework == :sinatra
+        Sinatra::Application.root
+      else
+        '.'
+      end
+    end
+    
+    # This needs to be improved. Frequently, multiple app servers gem are present and which
+    # ever is checked first becomes the designated app server. 
+    # 
+    # I've put Thin and Webrick last as they are often used in development and included in Gemfiles 
+    # but less likely used in production. 
+    #
+    # Next step: (1) list out all detected app servers (2) install hooks for those that need it (passenger, rainbows, unicorn). 
+    #
+    # Believe the biggest downside is the master process for forking app servers will get a background worker. Not sure how this will
+    # impact metrics (it shouldn't process requests). 
+    def app_server
+      @app_server ||= if passenger? then :passenger
+                    elsif rainbows? then :rainbows
+                    elsif unicorn? then :unicorn
+                    elsif thin? then :thin
+                    elsif webrick? then :webrick
+                    else nil
+                    end
+    end
+    
+    ### app server related-checks
+    
+    def thin?
+      if defined?(::Thin) && defined?(::Thin::Server)
+        # Ensure Thin is actually initialized. It could just be required and not running.
+        ObjectSpace.each_object(Thin::Server) { |x| return true }
+        false
+      end
+    end
+    
+    # Called via +#forking?+ since Passenger forks. Adds an event listener to start the worker thread
+    # inside the passenger worker process.
+    # Background: http://www.modrails.com/documentation/Users%20guide%20Nginx.html#spawning%5Fmethods%5Fexplained
+    def passenger?
+      (defined?(::Passenger) && defined?(::Passenger::AbstractServer)) || defined?(::PhusionPassenger)
+    end
+    
+    def webrick?
+      defined?(::WEBrick) && defined?(::WEBrick::VERSION)
+    end
+
+    def rainbows?
+      if defined?(::Rainbows) && defined?(::Rainbows::HttpServer)
+        ObjectSpace.each_object(::Rainbows::HttpServer) { |x| return true }
+      end
+    end
+    
+    def unicorn?
+      if defined?(::Unicorn) && defined?(::Unicorn::HttpServer)
+        # Ensure Unicorn is actually initialized. It could just be required and not running.
+        ObjectSpace.each_object(::Unicorn::HttpServer) { |x| return true }
+      end
+    end
+    
+    # If forking, don't start worker thread in the master process. Since it's started as a Thread, it won't survive
+    # the fork. 
+    def forking?
+      passenger? or unicorn? or rainbows?
+    end
+    
+    ### ruby checks
+    
+    def rubinius?
+      RUBY_VERSION =~ /rubinius/i
+    end
+
+    def jruby?
+      defined?(JRuby)
+    end
+    
+    def ruby_19?
+      defined?(RUBY_ENGINE) && RUBY_ENGINE == "ruby" && RUBY_VERSION.match(/^1\.9/)
+    end
+    
+    ### framework checks
+
+    def sinatra?
+      defined?(Sinatra::Application)
+    end
+
+  end # class Environemnt
+end