averell23-watchdogger 0.1.0

data/README.rdoc

@@ -0,0 +1,86 @@
+= Watchdogger
+
+Watchdogger is a simple ruby program that will monitor your log file or web 
+page (and potentially other stuff, too) and take action if something is amiss.
+It is intentionally simple, so there is less chance of having things go wrong
+with the watchdog itself.
+
+If you want to build more complicated reporting or such, you could do so 
+separately and act on notifications from the watchdog.
+
+= Installation
+
+Should be a matter of a simple
+
+  gem install averell23-watchdogger
+
+= Quick start
+
+You will have to make a configuration file like this:
+
+  # Actions you want to execute
+  actions:
+    log_event:
+      type: log_action
+      severity: info
+    restart_server:
+      type: kill_process
+      pidfile: /var/pid/server.pid
+  # Watchers: Things you want to monitor
+  watchers:
+    check_my_site:
+      type: http_watcher
+      url: http://www.mysite.some/
+      content_match: 'some.*other?string'
+      actions: 
+        - log_event
+        - restart_server
+  # Run each minute
+  interval: 60
+  logfile: /mystuff/dogger.log
+  log_level: info
+
+If your log file is called watcher.yml, you could call the watchdog script with
+
+  watchdogger -c watcher.yml
+
+(If you omit the log file from the configuration, you'll also see the log 
+output on your console). 
+
+The script above will check each minute if the web site responds and if the body
+matches the given regular expression. If not, it will log a message and restart
+the server.
+
+There are other options too, just check the API docs for more.
+
+To find out the options of the program, call
+
+  watchdogger --help
+
+= Why Watchdogger?
+
+I wrote this to monitor our instance of Tomcat. While there are other scripts
+around, they seem to be either "quick" shell scripts that require a standard
+linux layout, or potentially "Enterprise" solutions with a lot of overhead.
+
+This script is just plain Ruby and does not depend on external libraries. So
+you should be able to install the gem, set up your configuration and be 
+good to go. It doesn't even assume a particular install location.
+
+(In the future there may be some actions or things that require jruby, to 
+monitor java servers)
+
+= What if Watchdogger dies?
+
+Watchdogger has been designed to run as a daemon, instead of being a cron job
+that needs to be called every few minutes. This has several advantages: 
+We can have state information throughout the session, we can spawn helper 
+processes if needed, etc.
+
+However, the Watchdogger process may die for some reason, which would not be
+a good thing. 
+
+There is an easy solution, though: Just *do* install Watchdogger
+as a cron job. It will check if the old daemon process is still running and
+exit with code 1 if that's the case. If the old process is stale, it will 
+restart itself.

data/bin/watchdogger

@@ -0,0 +1,54 @@
+#!/usr/bin/env ruby
+$: << File.expand_path(File.join(File.dirname(__FILE__), '..', 'lib'))
+require 'watchdogger'
+
+module DoggerFlags extend OptiFlagSet
+  optional_flag 'config' do
+    alternate_forms 'c'
+    long_form 'configuration'
+    description 'Configuration file from which to read the servers to guard. Defaults to .watchdogger.yml in the users config dir.'
+  end
+  
+  optional_switch_flag 'daemon' do
+    alternate_forms 'd'
+    long_form 'daemonize'
+    description 'Run as a daemon, using the configured pidfile or .watchdogger.pid in your home directory'
+  end
+  
+  optional_switch_flag 'shutdown' do
+    alternate_forms 's'
+    long_form 'shutdown'
+    description 'Shutdown the daemon. You must given the -daemon flag in the same way as on startup'
+  end
+  
+  optional_switch_flag 'status' do
+    description 'Get the daemon status'
+  end
+  
+  and_process!
+end
+
+flags = DoggerFlags.flags
+config_file_name = flags.config || File.join(Etc.getpwuid.dir, '.watchdogger.yml')
+
+begin
+  config = YAML.load_file(config_file_name)
+  WatchDogger.init_system(config)
+  if(flags.status)
+    puts WatchDogger.check_daemon ? "The daemon is running." : "The daemon seems to be dead."
+  elsif(flags.shutdown)
+    puts "Shutting down the old daemon"
+    WatchDogger.shutdown_daemon
+  elsif(flags.daemon)
+    puts "Starting daemon."
+    WatchDogger.daemon
+  else
+    puts "Starting in foreground."
+    WatchDogger.watch_loop
+  end
+rescue SystemExit
+  # nothing
+rescue Exception => e
+  puts "Problem with this command: #{e.message}. Exiting."
+  exit(1)
+end

data/lib/dog_log.rb

@@ -0,0 +1,52 @@
+require 'logger'
+
+# Logging facility for the watchdog
+class DogLog # :nodoc:
+  
+  class << self
+    
+    # Set the log file and severity. This will reset the current logger,
+    # but should not usually be called on an active log.
+    def setup(logfile, severity)
+      @logfile = logfile
+      @severity = severity
+      if(@logger)
+        assit_fail('Resetting logfile')
+        @logger.close if(@logger.respond_to?(:close))
+        @logger = nil
+      end
+    end
+    
+    # If nothing is configured, we log to STDERR by default
+    def logger
+      @logger ||= begin
+        @logfile ||= STDERR
+        severity = @severity || Logger::DEBUG
+        logger = Logger.new(get_log_io, 3)
+        logger.level = severity
+        logger
+      end
+    end
+    
+    def get_log_io
+      return @logfile if(@logfile.kind_of?(IO))
+      if(Module.constants.member?(@logfile.upcase))
+        const = Module.const_get(@logfile.upcase)
+        return const if(const.kind_of?(IO))
+      end
+      File.open(@logfile, 'a')
+    end
+    
+    
+  end
+end
+
+class Object # :nodoc:
+  def self.dog_log
+    DogLog.logger
+  end
+  
+  def dog_log
+    DogLog.logger
+  end
+end

data/lib/watchdogger.rb

@@ -0,0 +1,177 @@
+require 'rubygems'
+require 'optiflag'
+require 'etc'
+require 'yaml'
+require 'assit'
+
+# require our own stuff
+lib_dir = File.expand_path(File.dirname(__FILE__)) 
+$: << lib_dir
+require 'dog_log'
+require 'watcher'
+require 'watcher_action'
+require 'watcher_event'
+
+# Require the Watcher
+Dir[File.join(lib_dir, 'watcher', '*.rb')].each { |f| require 'watcher/' + File.basename(f, '.rb') }
+# Require the actions
+Dir[File.join(lib_dir, 'watcher_action', '*.rb')].each { |f| require 'watcher_action/' + File.basename(f, '.rb') }
+
+module WatchDogger # :nodoc:
+  
+  class << self
+    
+    # Initializes the watchdog system, sets up the log. In addition to the configured
+    # Watchers and WatcherActions, the system can take the following arguments:
+    #
+    #   log_level     - The log level for the system log. This will apply to all log messages
+    #   logfile       - The log file for the system. Defaults to STDOUT
+    #   interval      - The watch interval in seconds. Defaults to 60
+    def init_system(options)
+      # First setup the logging options
+      @log_level = options.get_value(:log_level)
+      @logfile = options.get_value(:logfile)
+      DogLog.setup(@logfile, @log_level)
+      
+      # Now setup the actions
+      actions = options.get_value(:actions, false)
+      raise(ArgumentError, "Actions not configured correctly.") unless(actions.is_a?(Hash))
+      actions.each do |act_name, act_options|
+        WatcherAction.register(act_name, act_options)
+      end
+      
+      # Setupup the watchers
+      watchers = options.get_value(:watchers, false)
+      raise(ArgumentError, "Watchers not configured correctly.") unless(watchers.is_a?(Hash))
+      watchers.each do |watch_name, watch_options|
+        Watcher.register(watch_name, watch_options)
+      end
+      
+      dog_log.info('Watchdogger') { 'System Initialized' }
+      @watch_interval = options.get_value(:interval, 60).to_i
+      @pidfile = options.get_value(:pidfile) || File.join(Etc.getpwuid.dir, '.watchdogger.pid')
+      @pidfile = File.expand_path(@pidfile)
+    end
+    
+    
+    # This is the main loop of the watcher
+    def watch_loop
+      signal_traps
+      dog_log.info('Watchdogger') { "Starting watch loop with interval #{@watch_interval}"}
+      loop do
+        Watcher.watch_all!
+        sleep(@watch_interval)
+      end
+    end
+    
+    # Run as a daemon
+    def daemon 
+      raise(RuntimeError, "Daemon still running") if(check_daemon)
+      raise(ArgumentError, "Not daemonizing without a logfile") unless(@logfile && @logfile.upcase != 'STDOUT' && @logfile.upcase != 'STDERR') 
+      # Test the file opening before going daemon, so we know that
+      # it should usually work
+      File.open(@pidfile, 'w') { |io| io << 'starting' }
+      daemonize
+      # Now write the pid for real
+      File.open(@pidfile, 'w') { |io| io << Process.pid.to_s }
+      dog_log.info('Watchdogger') { "Running as daemon with pid #{Process.pid}" }
+      watch_loop
+    end
+    
+    # By default, camelize converts strings to UpperCamelCase. If the argument to camelize
+    # is set to ":lower" then camelize produces lowerCamelCase.
+    #
+    # camelize will also convert '/' to '::' which is useful for converting paths to namespaces
+    #
+    # Examples
+    #   "active_record".camelize #=> "ActiveRecord"
+    #   "active_record".camelize(:lower) #=> "activeRecord"
+    #   "active_record/errors".camelize #=> "ActiveRecord::Errors"
+    #   "active_record/errors".camelize(:lower) #=> "activeRecord::Errors"
+    def camelize(lower_case_and_underscored_word, first_letter_in_uppercase = true) # :nodoc:
+      if first_letter_in_uppercase
+        lower_case_and_underscored_word.to_s.gsub(/\/(.?)/) { "::" + $1.upcase }.gsub(/(^|_)(.)/) { $2.upcase }
+      else
+        lower_case_and_underscored_word.first + camelize(lower_case_and_underscored_word)[1..-1]
+      end
+    end
+    
+    # Shutdown the given daemon
+    def shutdown_daemon
+      Process.kill('TERM', get_pid)
+    end
+    
+    # Check for running daemon. Returns true if the system thinks that the 
+    # daemon is still running.
+    def check_daemon
+      return false unless(File.exists?(@pidfile))
+      pid = get_pid
+      begin
+        Process.kill(0, pid)
+        true
+      rescue Errno::EPERM
+        true
+      rescue Errno::ESRCH
+        dog_log.info('Watchdogger') { "Old process #{pid} is stale, good to go." }
+        false
+      rescue Exception => e
+        dog_log.error('Watchdogger') { "Could not find out if process #{pid} still runs (#{e.message}). Hoping for the best..." }
+        false
+      end
+    end
+    
+    private
+    
+    def get_pid
+      pid = File.open(@pidfile, 'r') { |io| io.read }
+      pid.to_i
+    end
+    
+    # Clean shutdown
+    def shutdown
+      dog_log.info('Watchdogger') { "Cleaning watchers..." }
+      Watcher.cleanup_watchers
+      if(@my_pidfile)
+        dog_log.info('Watchdogger') { "Removing pidfile at #{@my_pidfile}" }
+        FileUtils.remove(@my_pidfile) if(File.exists?(@my_pidfile))
+      end
+      dog_log.info('Watchdogger') { "Shutting down."}
+      exit(0)
+    end
+    
+    # Setup handler for the signals that should be handled by the skript
+    def signal_traps
+      Signal.trap('INT') { shutdown }
+      Signal.trap('TERM') { shutdown }
+      Signal.trap('HUP', 'IGNORE')
+    end
+    
+    # File active_support/core_ext/kernel/daemonizing.rb, line 4
+    def daemonize
+      exit if fork                   # Parent exits, child continues.
+      Process.setsid                 # Become session leader.
+      exit if fork                   # Zap session leader. See [1].
+      Dir.chdir "/"                  # Release old working directory.
+      File.umask 0000                # Ensure sensible umask. Adjust as needed.
+      STDIN.reopen "/dev/null"       # Free file descriptors and
+      STDOUT.reopen "/dev/null", "a" # point them somewhere sensible.
+      STDERR.reopen STDOUT           # STDOUT/ERR should better go to a logfile.
+    end
+    
+  end
+  
+end
+
+class Hash # :nodoc:
+  
+  # Gets the value from the Hash, regardless if it's stored with a symbol
+  # or a key as a string. You may supply a default value - if the default
+  # is set to false, the method will raise an argument error. By default, it
+  # will return nil if the element isn't found.
+  def get_value(sym_or_string, default = nil)
+    value = self[sym_or_string.to_s] || self[sym_or_string.to_sym] || default
+    raise(ArgumentError, "No value set for #{sym_or_string}") if((default == false) && !value)
+    value
+  end
+  
+end

data/lib/watcher/base.rb

@@ -0,0 +1,96 @@
+module Watcher
+  
+  # Base class for all Watcher.
+  #
+  # A watcher checks for a condition (e.g., if a web site responds, if a log file shows signs of
+  # trouble). Each watcher will have one or more actions attached that will be called if the
+  # watched condition is triggered.
+  #
+  # Each watcher will accept the following options, which are handled by the superclass:
+  #
+  #   severity  - Severity of the event. Each time the event is triggered, the watcher will
+  #               add this value to the internal "severity". If the internal severity reaches
+  #               100, the action is triggered. This means that with a severity of 100 the
+  #               action is run each time the watcher triggers. With a severity of 1, it is
+  #               only executed every 100th time. The global mechanism will reset the
+  #               severity once the action is triggered. The watcher class may decide
+  #               to reset the severity also on other occasions. Default: 100
+  #   
+  #   actions   - The actions that should be executed when the watcher triggers. These 
+  #               are names of actions that have been set up previously. (Required)
+  #  
+  #   warn_actions - Additional actions that are executed if the watcher triggers, but the 
+  #                  severity for a real action is not yet reached.
+  #
+  # Each watcher object must respond to the #watch_it! method. It must check the watched condition
+  # and return nil or false if the condition is not met. If the condition is met, it may return
+  # true or an error message.
+  #
+  # The Watcher _may_ also respond to the #cleanup method - which will be used to clean
+  # up all existing Watcher on a clean shutdown.
+  class Base
+    attr_accessor :severity
+    attr_accessor :name
+
+     # Sets up all actions for this watcher
+     def setup_actions(configuration)
+       action_config = configuration.get_value(:actions)
+       raise(ArgumentError, "No actions passed to watcher.") unless(action_config)
+       if(action_config.is_a?(Array))
+         action_config.each { |ac| actions << ac }
+       else
+         assit(action_config.is_a?(String) || action_config.is_a?(Symbol))
+         actions << action_config
+       end
+       warn_config = configuration.get_value(:warn_actions)
+       if(warn_config.is_a?(Array))
+         warn_config.each { |ac| warn_actions << ac }
+       elsif(warn_config)
+         assit_kind_of(String, warn_config)
+         warn_actions << warn_config
+       end
+     end
+
+     private
+
+     # Checks the trigger and does everything to call the actions connected to this watcher
+     def do_watch!
+       @current_severity ||= 0
+       result = watch_it!
+       return unless(result)
+
+       event = WatcherEvent.new
+       event.watcher = self
+       event.message = result unless(result.is_a?(TrueClass))
+       event.timestamp = Time.now
+
+       @current_severity = @current_severity + severity
+       if(@current_severity >= 100)
+         run_actions(event)
+         @current_severity = 0
+       else
+         warn_actions(event)
+       end
+
+       @last_run = Time.now
+     end
+
+     # Executes all actions for this watcher
+     def run_actions(event)
+       actions.each { |ac| WatcherAction.run_action(ac, event) }
+     end
+
+     def warn_actions(event)
+       warn_actions.each { |ac| WatcherAction.run_action(ac, event) }
+     end
+
+     def actions
+       @actions ||= []
+     end
+
+     def warn_actions
+       @warn_actions ||= []
+     end
+  end
+  
+end

data/lib/watcher/http_watcher.rb

@@ -0,0 +1,52 @@
+require 'net/http'
+
+module Watcher
+  
+  # Checks an http connection if it is active and returns the expected results.
+  # Options for this watcher:
+  # 
+  #   url           - The URL to query (required)
+  #   response      - The response code that is expected from the operation
+  #   content_match - A regular expression that is matched against the result.
+  #                   The watcher fails if the expression doesn't match
+  #   timeout       - The timeout for the connection attempt. Defaults to 10 sec
+  # 
+  # If neither response nor content_match are given, the watcher will expect a 
+  # 200 OK response from the server.
+  #
+  # This watcher resets the current severity on each successful connect, so that
+  # only continuous failures count against the trigger condition.
+  class HttpWatcher < Watcher::Base
+    
+    def initialize(config)
+      @url = config.get_value(:url, false)
+      match = config.get_value(:content_match)
+      @content_match = Regexp.new(match) if(match)
+      response = config.get_value(:response)
+      @response = ((!response && !match) ? "200" : response)
+      @timeout = config.get_value(:timeout, 10).to_i
+    end
+    
+    def watch_it!
+      url = URI.parse(@url)
+      res = Net::HTTP.start(url.host, url.port) do |http| 
+        http.read_timeout = @timeout
+        http.get(url.path) 
+      end
+      test_failed = false
+      if(@response && (@response != res.code))
+        test_failed = "Unexpected HTTP response: #{res.code} for #{@url} - expected #{@response}"
+      elsif(@content_match && !@content_match.match(res.body))
+        test_failed = "Did not find #{@content_match.to_s} at #{@url}"
+      end
+      @current_severity = 0 unless(test_failed)
+      dog_log.debug('HttpWatcher') { "Watch of #{@url} resulted in #{test_failed}" }
+      test_failed
+    rescue Exception => e
+      dog_log.debug('HttpWatcher') { "Watch of #{@url} had exception #{e.message}"}
+      "Exception on connect to #{@url} - #{e.message}"
+    end
+    
+  end
+  
+end

data/lib/watcher/log_watcher.rb

@@ -0,0 +1,97 @@
+require 'file/tail'
+
+module Watcher
+  
+  # Watches a log file for a given regular expression. This watcher will start a background thread that 
+  # tails the log and matches against the regexp.
+  #
+  # The current implementation is not tested against logs with very high load.
+  # 
+  # = Options
+  # 
+  #   logfile       - The log file to watch (required)
+  #   match         - A regular expression against which the log file will be matched (required)
+  #   reopen_after  - A number indicating after how many "unchanged" checks the file will
+  #                   be reopened. Defaults to 10.
+  #   interval_first, interval_max are the start and the max value for waiting on an unchanged
+  #   log file. They default to 60 (1 minute) and 300 (5 minutes). The log file will
+  #   be considered stale and reopened after max_value * 3.
+  #
+  # = Warning
+  # 
+  # Depending on your Ruby implementation and platform, the background thread may not be
+  # taken down if Watchdogger explodes during runtime.
+  #
+  # On a clean exit, all threads will be cleanly shut down, but if you kill -9 it,
+  # you may want to check for any rogue processes.
+  class LogWatcher < Watcher::Base
+    
+    def initialize(options)
+       @file_name = options.get_value(:logfile, false)
+       @match_str = options.get_value(:match, false)
+      
+       @interval_first = options.get_value(:interval_first, 60).to_i
+       @interval_max = options.get_value(:interval_max, 300).to_i
+      
+       watch_log # Start the watcher
+    end
+    
+    def watch_it!
+      unless(@log_watcher.status) # Restart the watcher if killed for some reason
+        dog_log.warn { "Log watcher on #{@file_name} died? Restarting..." }
+        watch_log
+      end
+      is_triggered = false
+      if(triggered?)
+        is_triggered = "Found #{@match_str} in #{@file_name}"
+      end
+      is_triggered
+    rescue Exception => e
+      "Exception running the log watcher: #{e}"
+    end
+    
+    def cleanup
+      @log_watcher.kill if(@log_watcher && !@log_watcher.stop?)
+    end
+    
+    private
+    
+    def triggered?
+      @triggered
+    end
+    
+    def triggered!
+      @my_mutex.synchronize { @triggered = true }
+    end
+    
+    def reset_trigger
+      @my_mutex.synchronize { @triggered = false }
+    end
+    
+    def watch_log
+      @my_mutex = Mutex.new
+      if(@log_watcher)
+        dog_log.warn { "Existing log watcher on #{@file_name}, killing."}
+        @log_watcher.terminate! 
+      end
+      @log_watcher = Thread.new(@match_str, @file_name) do |match_str, file_name|
+        matcher = Regexp.new(match_str) 
+        logfile = File::Tail::Logfile.open(file_name, 
+          :backward => 1,
+          :reopen_deleted => true,
+          :interval => @interval_first,
+          :max_interval => @interval_max,
+          :reopen_suspicious => true,
+          :suspicious_interval => (@interval_max * 3)
+        )
+        logfile.tail do |line|
+          if(matcher.match(line))
+            triggered!
+          end
+        end
+      end
+      dog_log.debug { "Log watcher thread started for #{@file_name}" }
+    end
+    
+  end
+end

data/lib/watcher.rb

@@ -0,0 +1,51 @@
+# Handling for the Watcher objects in the system. The Watcher are not access directly,
+# but handled through the static methods of this module. The module will also keep track
+# of the watcher state and wrap the invocation procedure.
+module Watcher
+  
+  # Number of times the Watcher were called
+  @@watch_runs = 0
+  
+  class << self
+    
+    # Create a new watcher with the given configuration. The type identifies the watcher class
+    # that should be used.
+    # 
+    # This will *not* return the watcher object, as it is not to be used externally. The watcher
+    # will be registered internally, and called when the #watch_all! method is called
+    def register(name, config_options)
+      assit_kind_of(String, name)
+      raise(ArgumentError, "Illegal options") unless(config_options.is_a?(Hash))
+      type = config_options.get_value(:type, false)
+      type = WatchDogger.camelize(type)
+      watcher =  Watcher.const_get(type).new(config_options)
+      watcher.setup_actions(config_options)
+      severity = config_options.get_value(:severity, 100).to_i
+      watcher.severity = severity
+      watcher.name = name
+      registered_watchers << watcher
+      dog_log.debug('Watcher') { "Registered Watcher of type #{type}" }
+    end
+  
+    # This will execute all registered Watcher, which, in turn, will execute their
+    # actions if necessary. Normally, this will run all Watcher each time this method is
+    # called. However, the watcher may implement conditions on which the check is skipped.
+    def watch_all!
+      @last_check = Time.now
+      registered_watchers.each { |w| w.send(:do_watch!) }
+    end
+    
+    # Cleans up all Watcher
+    def cleanup_watchers
+      registered_watchers.each { |w| w.cleanup if(w.respond_to?(:cleanup)) }
+    end
+    
+    private
+  
+    def registered_watchers
+      @registered_watchers ||= []
+    end
+    
+  end # end class methods
+  
+end