scout 1.1.0

data/Rakefile

@@ -0,0 +1,120 @@
+require "rake/rdoctask"
+require "rake/testtask"
+require "rake/gempackagetask"
+require "rake/contrib/rubyforgepublisher"
+require "net/ssh"
+
+require "rubygems"
+require "rubyforge"
+
+dir     = File.dirname(__FILE__)
+lib     = File.join(dir, "lib", "scout.rb")
+version = File.read(lib)[/^\s*VERSION\s*=\s*(['"])(\d\.\d\.\d)\1/, 2]
+history = File.read("CHANGELOG").split(/^(===.*)/)
+changes ||= history[0..2].join.strip
+
+need_tar = true
+need_zip = true
+
+task :default => [:test]
+
+Rake::TestTask.new do |test|
+  test.libs       << "test"
+  test.test_files = [ "test/scout_test.rb" ]
+  test.verbose    = true
+end
+
+Rake::RDocTask.new do |rdoc|
+	rdoc.main     = "README"
+	rdoc.rdoc_dir = "doc/html"
+	rdoc.title    = "Scout Client Documentation"
+	rdoc.rdoc_files.include( "README",  "INSTALL",
+	                         "TODO",    "CHANGELOG",
+	                         "AUTHORS", "COPYING",
+	                         "LICENSE", "lib/" )
+end
+
+desc "Upload current documentation to Scout Gem Server"
+task :upload_docs => [:rdoc] do
+	sh "scp -r doc/html/* " +
+	   "deploy@gems.scoutapp.com:/var/www/gems/docs"
+end
+
+spec = Gem::Specification.new do |spec|
+	spec.name    = "scout"
+	spec.version = version
+
+	spec.platform = Gem::Platform::RUBY
+	spec.summary  = "Scout makes monitoring and reporting on your web applications as flexible and simple as possible."
+
+  # TODO: test suite
+	# spec.test_suite_file = "test/ts_all.rb"
+	spec.files           = Dir.glob("{lib,test,examples}/**/*.rb").
+	                           reject { |item| item.include?(".svn") } +
+	                       Dir.glob("{test,examples}/**/*.csv").
+	                           reject { |item| item.include?(".svn") } +
+	                           ["Rakefile", "setup.rb"]
+  spec.executables     = ["scout"]
+
+	spec.has_rdoc         = true
+	spec.extra_rdoc_files = %w[ AUTHORS COPYING README INSTALL TODO CHANGELOG
+	                            LICENSE ]
+	spec.rdoc_options     << "--title" << "Scout Client Documentation" <<
+	                         "--main"  << "README"
+
+	spec.require_path = "lib"
+	
+  spec.add_dependency "elif"
+  # spec.add_dependency "hpricot", "=0.6"
+
+	spec.author            = "Highgroove Studios"
+	spec.email             = "scout@highgroove.com"
+	spec.rubyforge_project = "scout"
+	spec.homepage          = "http://scoutapp.com"
+	spec.description       = <<END_DESC
+Scout makes monitoring and reporting on your web applications as flexible and simple as possible.
+
+Scout is a product of Highgroove Studios.
+END_DESC
+end
+
+Rake::GemPackageTask.new(spec) do |pkg|
+	pkg.need_zip = need_tar
+	pkg.need_tar = need_zip
+end
+
+desc "Publish Gem to Scout Gem Server"
+task :publish => [:package] do
+  pkg = "pkg/#{spec.name}-#{spec.version}"
+
+  if $DEBUG then
+    puts "release_id = rf.add_release #{spec.rubyforge_project.inspect}, #{spec.name.inspect}, #{spec.version.inspect}, \"#{pkg}.tgz\""
+    puts "rf.add_file #{spec.rubyforge_project.inspect}, #{spec.name.inspect}, release_id, \"#{pkg}.gem\""
+  end
+
+  puts "Publishing on RubyForge"
+  rf = RubyForge.new
+  puts "Logging in"
+  rf.login
+
+  c = rf.userconfig
+  puts rf.inspect
+  c["release_notes"] = spec.description if spec.description
+  c["release_changes"] = changes if changes
+  c["preformatted"] = true
+
+  files = [(need_tar ? "#{pkg}.tgz" : nil),
+           (need_zip ? "#{pkg}.zip" : nil),
+           "#{pkg}.gem"].compact
+
+  puts "Releasing #{spec.name} v. #{spec.version}"
+  rf.add_release spec.rubyforge_project, spec.name, spec.version, *files
+
+  puts "Publishing on Scout Server"
+	sh "scp -r pkg/*.gem " +
+	   "deploy@gems.scoutapp.com:/var/www/gems/gems"
+	ssh = Net::SSH.start('gems.scoutapp.com','deploy')
+	ssh_shell = ssh.shell.sync
+	ssh_out = ssh_shell.send_command "/usr/bin/index_gem_repository.rb -d /var/www/gems"
+  puts "Published, and updated gem server." if ssh_out.stdout.empty? && !ssh_out.stderr
+end

data/TODO

@@ -0,0 +1,6 @@
+= To Do List
+
+The following is a list of planned expansions for the Scout Client:
+
+ * optionally find the "nice" command if available and use that in the 
+   crontab output

data/bin/scout

@@ -0,0 +1,200 @@
+#!/usr/bin/env ruby
+
+$VERBOSE = true # -w
+$KCODE   = "u"  # -Ku
+
+$LOAD_PATH << File.join(File.dirname(__FILE__), *%w[.. lib])
+require "scout"
+require "optparse"
+require "logger"
+require "fileutils"
+require "pp"
+
+CONFIG_DIR = File.join((File.expand_path("~") rescue "/"), ".scout")
+USER       = ENV["USER"] || ENV["USERNAME"] || "root"
+
+options = { :server  => "https://scoutapp.com/",
+            :history => File.join(CONFIG_DIR , "client_history.yaml"),
+            :verbose => false,
+            :level   => "info" }
+
+ARGV.options do |opts|
+  opts.banner = "Usage:  #{File.basename($PROGRAM_NAME)} [OPTIONS] CLIENT_KEY"
+  
+  opts.separator ""
+  opts.separator "CLIENT_KEY is the indentification key assigned to this " +
+                 "client by the server."
+  opts.separator ""
+  opts.separator "Note: This client is meant to be installed and invoked " +
+                 "through cron or any other scheduler."
+  opts.separator ""
+  opts.separator "Specific Options:"
+  
+  opts.on( "-s", "--server SERVER", String,
+           "The URL for the server this client reports to." ) do |url|
+    options[:server] = url
+  end
+  opts.on( "-p", "--plugin PLUGIN", String,
+           "The path to a plugin to run locally. " +
+           "Useful for Testing." ) do |plugin|
+    options[:plugin] = plugin
+  end  
+
+  opts.separator ""
+  
+  opts.on( "-d", "--data DATA", String,
+           "The data file used to track the history of executions." ) do |file|
+    options[:history] = file
+  end
+  opts.on( "-l", "--level LEVEL", Logger::SEV_LABEL.map { |l| l.downcase },
+           "The level of logging to report." ) do |level|
+    options[:level] = level
+  end
+  opts.on( "-o", "--plugin-options PLUGIN_OPTIONS", String,
+           "The options YAML file to pass to the plugin." ) do |plugin_options|
+    options[:plugin_options] = plugin_options
+  end  
+  
+  opts.separator "Common Options:"
+  
+  opts.on( "-h", "--help",
+           "Show this message." ) do
+    puts opts
+    exit
+  end
+  opts.on( "-v", "--[no-]verbose",
+           "Turn on logging to STDOUT" ) do |bool|
+    options[:verbose] = bool
+  end
+  
+  begin
+    opts.parse!
+    options[:client_key] = ARGV.shift if ARGV.size == 1
+  rescue
+    puts opts
+    exit
+  end
+end
+
+log                 = Logger.new($stdout)
+log.datetime_format = "%Y-%m-%d %H:%M "
+log.level           = Logger.const_get(options[:level].upcase) \
+                        rescue Logger::INFO
+
+real_config_dir = File.dirname(options[:history])
+FileUtils.mkdir_p(real_config_dir) # ensure dir exists
+
+# make sure only one copy is ever running at a time
+pid_file = File.join(real_config_dir, "scout_client_pid.txt")
+begin
+  File.open(pid_file, File::CREAT|File::EXCL|File::WRONLY) { |pid| pid.puts $$ }
+  at_exit do
+    begin
+      File.unlink(pid_file)
+    rescue
+      log.error "Unable to unlink pid file:  #{$!.message}"
+    end
+  end
+rescue
+  pid     = File.read(pid_file).strip.to_i rescue "unknown"
+  running = true
+  begin
+    Process.kill(0, pid)
+  rescue Errno::ESRCH
+    running = false
+  rescue
+    # do nothing, we didn't have permission to the running process
+  end
+  if running
+    log.warn "Process #{pid} was already running"
+    exit
+  else
+    log.info "Stale PID file found.  Clearing it and reloading..."
+    File.unlink(pid_file)
+    retry
+  end
+end
+
+if [:client_key, :plugin].all? { |o| options[o].nil? } and
+   $stdin.tty?  # install wizard
+  puts <<-END_INTRO.gsub(/^ {2}/, "")
+  == Scout Installation Wizard ==
+  
+  You need the Client Key displayed in the Client Settings
+  tab. It looks like:
+  
+    6ecad322-0d17-4cb8-9b2c-a12c4541853f
+  
+  Enter the Client Key:
+  END_INTRO
+  options[:client_key] = gets.to_s.chomp!
+  
+  # puts "Attempting to contact the server..."
+  begin
+    Scout::Server.new( options[:server],
+                       options[:client_key],
+                       options[:history],
+                       options[:verbose] ? log : nil ) { |server| server.test }
+  
+    puts <<-END_SUCCESS.gsub(/^ {4}/, "")
+    
+    Success!
+  
+    ******* NOW, INSTALL IN CRONTAB *******
+  
+    */10 * * * *  #{USER} #{File.expand_path($PROGRAM_NAME)} #{options[:client_key]}
+    
+    ******* END CRONTAB SAMPLE *******
+    
+    For help setting up Scout with crontab, please visit:
+
+      http://scoutapp.com/help#cron
+    
+    END_SUCCESS
+  rescue SystemExit
+    puts <<-END_ERROR.gsub(/^ {4}/, "")
+    
+    Could not contact server. The client key may be incorrect. For more help,
+    please visit:
+    
+    http://scoutapp.com/help
+    
+    END_ERROR
+  end
+elsif options[:plugin]                        # local plugin
+  # read the plugin_code from the file specified
+  plugin_code = File.read(options[:plugin])
+
+  plugin_options = if options[:plugin_options].to_s[0..0] == "{"
+    eval(options[:plugin_options])  # options from command-line
+  elsif options[:plugin_options]
+    # 
+    # read the plugin_options from the YAML file specified,
+    # parse each option and use the default value specified
+    # in the options as the value to be passed to the test plugin
+    # 
+    Hash[ *File.open(options[:plugin_options]) { |f| YAML.load(f) }["options"].
+           map { |name, details| [name, details["default"]] }.flatten ]
+  else
+    Hash.new
+  end
+
+  Scout::Server.new( nil,
+                     options[:client_key],
+                     options[:history],
+                     options[:verbose] ? log : nil ) do |server|
+    pp server.process_plugin( { :interval  => 0,
+                                :plugin_id => 1,
+                                :name      => "Local Plugin",
+                                :code      => plugin_code,
+                                :options   => plugin_options,
+                                :path      => options[:plugin] } )
+  end  
+else                                          # normal run
+  Scout::Server.new( options[:server],
+                     options[:client_key],
+                     options[:history],
+                     options[:verbose] ? log : nil ) do |server|
+    server.run_plugins_by_plan
+  end
+end

data/lib/scout.rb

@@ -0,0 +1,8 @@
+#!/usr/bin/env ruby -wKU
+
+require "scout/plugin"
+require "scout/server"
+
+module Scout
+  VERSION = "1.1.0".freeze
+end

data/lib/scout/plugin.rb

@@ -0,0 +1,25 @@
+#!/usr/bin/env ruby -wKU
+
+module Scout
+  class Plugin
+    class << self
+      attr_reader :last_defined
+
+      def inherited(new_plugin)
+        @last_defined = new_plugin
+      end
+
+      def load(last_run, memory, options)
+        new(last_run, memory, options)
+      end
+    end
+
+    # Creates a new Scout Plugin to run.
+    # 
+    def initialize(last_run, memory, options)
+      @last_run = last_run
+      @memory   = memory
+      @options  = options
+    end
+  end
+end

data/lib/scout/server.rb

@@ -0,0 +1,285 @@
+#!/usr/bin/env ruby -wKU
+
+require "net/https"
+require "uri"
+require "yaml"
+require "timeout"
+
+module Scout
+  class Server
+    # A new class for plugin Timeout errors.
+    class PluginTimeoutError < RuntimeError; end
+    
+    # The default URLS are used to communicate with the Scout Server.
+    URLS = { :plan   => "/clients/CLIENT_KEY/plugins.scout?version=CLIENT_VERSION",
+             :report => "/clients/CLIENT_KEY/plugins/PLUGIN_ID/reports.scout?version=CLIENT_VERSION",
+             :error  => "/clients/CLIENT_KEY/plugins/PLUGIN_ID/errors.scout?version=CLIENT_VERSION",
+             :alert  => "/clients/CLIENT_KEY/plugins/PLUGIN_ID/alerts.scout?version=CLIENT_VERSION" }
+
+    # 
+    # A plugin cannot take more than PLUGIN_TIMEOUT seconds to execute, 
+    # otherwise, a timeout error is generated.
+    # 
+    PLUGIN_TIMEOUT = 60
+    
+    # Creates a new Scout Server connection.
+    def initialize(server, client_key, history_file, logger = nil)
+      @server       = server
+      @client_key   = client_key
+      @history_file = history_file
+      @history      = Hash.new
+      @logger       = logger
+      
+      if block_given?
+        load_history
+        yield self
+        save_history
+      end
+    end
+    
+    # 
+    # Loads the history file from disk. If the file does not exist, 
+    # it creates one.
+    #
+    def load_history
+      unless File.exist? @history_file
+        debug "Creating empty history file..."
+        File.open(@history_file, "w") do |file|
+          YAML.dump({"last_runs" => Hash.new, "memory" => Hash.new}, file)
+        end
+        info "History file created."
+      end
+      debug "Loading history file..."
+      @history = File.open(@history_file) { |file| YAML.load(file) }
+      info "History file loaded."
+    end
+    
+    # Saves the history file to disk.
+    def save_history
+      debug "Saving history file..."
+      File.open(@history_file, "w") { |file| YAML.dump(@history, file) }
+      info "History file saved."
+    end
+    
+    # Runs all plugins from a given plan. Calls process_plugin on each plugin.
+    def run_plugins_by_plan
+      plan do |plugin|
+        process_plugin(plugin)
+      end
+    end
+    
+    # 
+    # This is the heart of Scout.  
+    # 
+    # First, it determines if a plugin is past interval and needs to be run.
+    # If it is, it simply evals the code, compiling it.
+    # It then loads the plugin and runs it with a PLUGIN_TIMEOUT time limit.
+    # The plugin generates data, alerts, and errors. In addition, it will
+    # set memory and last_run information in the history file.
+    # 
+    def process_plugin(plugin)
+      info "Processing the #{plugin[:name]} plugin:"
+      last_run = @history["last_runs"][plugin[:name]]
+      memory   = @history["memory"][plugin[:name]]
+      run_time = Time.now
+      if last_run.nil? or run_time > last_run + plugin[:interval]
+        debug "Plugin is past interval and needs to be run.  " +
+              "(last run:  #{last_run || 'nil'})"
+        debug "Compiling plugin..."
+        begin
+          eval(plugin[:code], TOPLEVEL_BINDING, plugin[:path] || plugin[:name])
+          info "Plugin compiled."
+        rescue Exception
+          error "Plugin would not compile: #{$!.message}"
+          return
+        end
+        debug "Loading plugin..."
+        if job = Plugin.last_defined.load( last_run, (memory || Hash.new),
+                                           plugin[:options] || Hash.new )
+          info "Plugin loaded."
+          debug "Running plugin..."
+          begin
+            data = {}
+            Timeout.timeout(PLUGIN_TIMEOUT, PluginTimeoutError) do
+              data = job.run
+            end
+          rescue Timeout::Error
+            error "Plugin took too long to run."
+            return
+          rescue Exception
+            error "Plugin failed to run: #{$!.backtrace}"
+          end
+          info "Plugin completed its run."
+          
+          # handle single report or array of reports
+          send_report(data[:report], plugin[:plugin_id])   if data[:report]
+          if data[:reports] and not data[:reports].empty?
+            data[:reports].each { |r| send_report(r, plugin[:plugin_id]) }
+          end          
+          # handle single alert or array of alerts
+          send_alert(data[:alert], plugin[:plugin_id])   if data[:alert]
+          if data[:alerts] and not data[:alerts].empty?
+            data[:alerts].each { |a| send_alert(a, plugin[:plugin_id]) }
+          end
+          # handle single error or array of errors
+          send_error(data[:error], plugin[:plugin_id]) if data[:error]
+          if data[:errors] and not data[:errors].empty?
+            data[:errors].each { |e| send_error(e, plugin[:plugin_id]) }
+          end
+          
+          @history["last_runs"][plugin[:name]] = run_time
+          @history["memory"][plugin[:name]]    = data[:memory]
+        else
+          scout_error({:subject => "Plugin would not load."}, plugin[:plugin_id])
+        end
+      else
+        debug "Plugin does not need to be run at this time.  " +
+              "(last run:  #{last_run || 'nil'})"
+      end
+      info "Plugin #{plugin[:name]} processing complete."
+      data
+    end
+    
+    # 
+    # Retrieves the Plugin Plan from the server. This is the list of plugins 
+    # to execute, along with all options.
+    # 
+    def plan
+      url = urlify(:plan)
+      info "Loading plan from #{url}..."
+      get(url, "Could not retrieve plan from server.") do |res|
+        begin
+          plugin_execution_plan = Marshal.load(res.body)
+          info "Plan loaded.  (#{plugin_execution_plan.size} plugins:  " +
+               "#{plugin_execution_plan.map { |p| p[:name] }.join(', ')})"
+        rescue TypeError
+          fatal "Plan from server was malformed."
+          exit
+        end
+        plugin_execution_plan.each do |plugin|
+          begin
+            yield plugin if block_given?
+          rescue RuntimeError
+            scout_error( { :subject => "Exception:  #{$!.message}.",
+                     :body    => $!.backtrace },
+                   plugin[:plugin_id] )
+          end
+        end
+      end
+    end
+    alias_method :test, :plan
+
+    # Sends report data to the Scout Server.
+    def send_report(data, plugin_id)
+      url = urlify(:report, :plugin_id => plugin_id)
+      report_hash = {:data => data, :plugin_id => plugin_id}
+      
+      # add in any special fields
+      if time = ( data.delete(:scout_time) || data.delete("scout_time") )
+        report_hash[:time] = time
+      end
+      
+      debug "Sending report to #{url} (#{data.inspect})..."
+      post url,
+           "Unable to send report to server.",
+           :report => report_hash
+      info "Report sent."
+    end
+
+    # Sends an alert to the Scout Server.
+    def send_alert(data, plugin_id)
+      url = urlify(:alert, :plugin_id => plugin_id)
+      debug "Sending alert to #{url} (subject: #{data[:subject]})..."
+      post url,
+           "Unable to send alert to server.",
+           :alert => data.merge(:plugin_id => plugin_id)
+      info "Alert sent."
+    end
+
+    # Sends an error to the Scout Server.
+    def send_error(data, plugin_id)
+      url = urlify(:error, :plugin_id => plugin_id)
+      debug "Sending error to #{url} (subject: #{data[:subject]})..."
+      post url,
+           "Unable to log error on server.",
+            :error => data.merge(:plugin_id => plugin_id)
+      info "Error sent."
+    end
+    
+    private
+
+    def urlify(url_name, options = Hash.new)
+      return unless @server
+      options.merge!(:client_version => Scout::VERSION)
+      URI.join( @server,
+                URLS[url_name].
+                  gsub(/\bCLIENT_KEY\b/, @client_key).
+                  gsub(/\b[A-Z_]+\b/) { |k| options[k.downcase.to_sym] || k } )
+    end
+
+    def paramify(params, prefix = nil)
+      params.inject(Hash.new) do |all, (key, value)|
+        parent = prefix ? "#{prefix}[#{key}]" : String(key)
+        if value.is_a? Hash
+          all.merge(paramify(value, parent))
+        else
+          all.merge(parent => String(value))
+        end
+      end
+    end
+    
+    def post(url, error, params = {}, &response_handler)
+      return unless url
+      request(url, response_handler, error) do |connection|
+        post = Net::HTTP::Post.new(url.path + (url.query ? ('?' + url.query) : ''))
+        post.set_form_data(paramify(params))
+        connection.request(post)
+      end
+    end
+
+    def get(url, error, params = {}, &response_handler)
+      return unless url
+      request(url, response_handler, error) do |connection|
+        connection.get(url.path + (url.query ? ('?' + url.query) : ''))
+      end
+    end
+    
+    def request(url, response_handler, error, &connector)
+      http             = Net::HTTP.new(url.host, url.port)
+      if url.is_a? URI::HTTPS
+        http.use_ssl     = true
+        http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+      end
+      case response    = no_warnings { http.start(&connector) }
+      when Net::HTTPSuccess
+        response_handler[response] unless response_handler.nil?
+      else
+        fatal error
+        exit
+      end
+    rescue Timeout::Error
+      fatal "Request timed out."
+      exit
+    rescue Exception
+      fatal "An HTTP error occurred:  #{$!.message}"
+      exit
+    end
+    
+    def no_warnings
+      old_verbose = $VERBOSE
+      $VERBOSE    = false
+      yield
+    ensure
+      $VERBOSE = old_verbose
+    end
+    
+    # Forward Logger methods to an active instance, when there is one.
+    def method_missing(meth, *args, &block)
+      if (Logger::SEV_LABEL - %w[ANY]).include? meth.to_s.upcase
+        @logger.send(meth, *args, &block) unless @logger.nil?
+      else
+        super
+      end
+    end
+  end
+end