alert_machine 0.0.1

data/README

@@ -0,0 +1,65 @@
+= AlertMachine
+
+Get notifications if bad things happen to your server. You can easily make sure
+all processes are running, ports are open. You can also add checks for bad events
+that get run once every few minutes and get error reports to your email.
+
+== Usage
+
+1. Defining a Watcher class:
+
+class MyWatcher < AlertMachine::Watcher
+
+    # Example 1: Make sure port 80 is running on server1 and server2.
+    watch_process(["server1.example.com", "server2.com"], :port => 80)
+
+    # Example 2: Make sure the two thin servers are running in server1.
+    # Check if the two ports are open, and check if the two pid files are present
+    # and pointing to valid processes.
+    watch_process("server1.example.com", :port => [3000, 3001], :pid_file =>
+        ["/tmp/thin.3000.pid", "/tmp/thin.3001.pid"])
+
+    # Example 3: We can also make sure there are no new crashes.
+    watch(:retries => 0) do
+        new_crashes = Crash.where(unread: false).all
+        assert new_crashes.empty?, <<MAIL
+        #{new_crashes.length} new crashes found.
+        #{ new_crashes.collect {|c| c.print }.join("\n") }
+MAIL
+    # The above code asserts that new_crashes is empty. If it's not empty
+    # an alert is triggerred with the message contents being the string
+    # that follows the assert.
+    end
+end
+
+2. Running the watcher class:
+
+File: my_watcher_runner.rb
+require 'alert_machine'
+
+# The below line safely loads the rails environment sending out alerts
+# incase things are broken.
+AlertMachine::RailsEnvironment.bootup
+
+# Require your alert files.
+require "offline/alerts/my_watcher1.rb"
+require "offline/alerts/my_watcher2.rb"
+
+# Run the machine.
+AlertMachine.run
+
+== Configuration
+
+If you want to change the default settings, you can call:
+
+AlertMachine.config("config_file_path")
+
+before the `AlertMachine.run`
+
+You can also easily pass diff config files for development and production, if
+you are using rails.
+
+A list of all config options are available at AlertMachine::Watcher#watch and
+AlertMachine::Watcher#watch_process
+
+

data/alert_machine.gemspec

@@ -0,0 +1,23 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+
+Gem::Specification.new do |s|
+  s.name        = "alert_machine"
+  s.version     = "0.0.1"
+  s.authors     = ["prasanna"]
+  s.email       = ["myprasanna@gmail.com"]
+  s.homepage    = "http://github.com/likealittle/alert_machine"
+  s.summary     = "Ruby way of alerting server events."
+  s.description = "Make sure you get mailed when bad things happen to your server."
+
+  s.rubyforge_project = "alert_machine"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  s.add_runtime_dependency "rye"
+  s.add_runtime_dependency "actionmailer"
+  s.add_runtime_dependency "eventmachine"
+end

data/lib/alert_machine.rb

@@ -0,0 +1,173 @@
+require 'eventmachine'
+require 'action_mailer'
+
+class AlertMachine
+  
+  class Watcher
+      # == Options:
+      #     The below options can also be overridden via config/alert_machine.yml
+      #
+      # * interval:
+      #     Seconds between each run, during the steady state. 5 min default.
+      #
+      # * interval_error:
+      #     How soon to check again, in-case an error occurred. (interval)/5 default.
+      #
+      # * from, to:
+      #     Comma seperated list of emails, to bother when there are alerts. defaults
+      #     to whatever was specified in the config file.
+      #
+      # * retries:
+      #     Number of times to try before alerting on error. Defaults to 1.
+      #     
+      # * dont_check_long_processes:
+      #     Don't assert if my watch took too long to run. [false defaults]
+      #
+      def self.watch(opts = {}, caller = caller, &block)
+        AlertMachine.tasks << RunTask.new(opts, block, caller)
+      end
+
+      def self.assert(conditions, msg = nil, caller = caller)
+        AlertMachine.current_task.assert(conditions, msg, caller)
+      end
+
+      # Make sure the process keeps running. machines can be one or many.
+      # 
+      # == Options:
+      # One or more of the below constraints. Any of the below can either
+      # be a single element or an array. (eg. multiple ports)
+      #
+      # * port: 
+      #     Ensure the port is open.
+      #     
+      # * pid_file:
+      #     Make sure the pid file exists and the process corresponding to it,
+      #     is alive.
+      #
+      # * grep:
+      #     Executes `ps aux | grep <string>` to ensure process is running.
+      #
+      # Other usual options of watcher, mentioned above.
+      #
+      def self.watch_process(machines, opts = {})
+        machines = [machines].flatten
+        Process.watch(machines, opts, caller)
+      end
+
+      # Run a command on a set of machines.
+      def self.run_command(machines, cmd)
+        machines = [machines].flatten
+        require 'rye'
+        set = Rye::Set.new(machines.join(","), :parallel => true)
+        machines.each { |m| set.add_box(Rye::Box.new(m, AlertMachine.ssh_config.merge(:safe => false))) }
+        puts "executing on #{machines}: #{cmd}"
+        res = set.execute(cmd).group_by {|ry| ry.box.hostname }.sort_by {|name, op| machines.index(name) }
+        res.each { |machine, op|
+          puts "[#{machine}]\n#{op.join("\n")}\n"
+        }
+      end
+
+      private
+      # To suppress logging in test mode.
+      def puts(*args)
+        super unless AlertMachine.test_mode?
+      end
+  end
+
+  # Configure your machine before running it.
+  CONFIG_FILE = 'config/alert_machine.yml'
+  @@config = nil
+  def self.config(config_file = CONFIG_FILE)
+    @@config ||= YAML::load(File.open(config_file))
+  rescue
+    {}
+  end
+
+  # Invoke this whenever you are ready to enter the AlertMachine loop.
+  def self.run
+    unless @@em_invoked
+      @@em_invoked = true
+      EM::run do
+        @@tasks.each do |t|
+          t.schedule
+        end
+        yield if block_given?
+      end
+    end
+  end
+
+  def self.ssh_config
+    res = {}
+    config['ssh'].each_pair do |k, v|
+      res[k.to_sym] = v
+    end
+    return res
+  end
+
+  def self.disable(disabled = true)
+    @@em_invoked = disabled
+  end
+
+  # Figures out how to parse the call stack and pretty print it.
+  class Caller
+    attr_reader :caller, :file, :line
+    
+    def initialize(caller, &block)
+      @block = block if block_given?
+      @caller = caller
+      /^(?<fname>[^:]+)\:(?<line>\d+)\:/ =~ caller[0] and
+        @file = fname and @line = line
+    end
+
+    def file_line
+      "#{file}:#{line}"
+    end
+
+    def log
+      "#{caller[0]}\n" +
+        log_source_file.to_s
+    end
+    
+    def log_source_file
+      File.open(file) {|fh|
+        fh.readlines[line.to_i - 1..line.to_i + 3].collect {|l|
+          ">> #{l}"
+        }.join + "\n---\n"
+      } if file && File.exists?(file)
+    end
+  end
+
+  @@tasks = []
+  @@em_invoked = false
+  @@current_task = nil
+
+  def self.tasks
+    @@tasks
+  end
+
+  def self.current_task
+    @@current_task
+  end
+
+  def self.current_task=(task)
+    @@current_task = task
+  end
+
+  def self.reset
+    @@tasks = []
+  end
+
+  private
+  def puts(*args)
+    super unless AlertMachine.test_mode?
+  end
+  
+  def self.test_mode?
+    false
+  end
+end
+
+dname = File.dirname(__FILE__)
+require "#{dname}/process.rb"
+require "#{dname}/run_task.rb"
+require "#{dname}/rails_environment.rb"

data/lib/process.rb

@@ -0,0 +1,71 @@
+class AlertMachine
+
+  # Checks if processes are living, and have their ports open.
+  class Process < Watcher
+    class << self
+
+      def watch(machines, opts, caller)
+        raise ArgumentError, "Must mention atleast one of (port, pid_file, grep)" unless
+          opts[:port] || opts[:pid_file] || opts[:grep]
+          raise ArgumentError, "Must not be passed a block" if block_given?
+
+        super(opts, caller) do
+          check(:port, machines, opts, caller)
+          check(:pid_file, machines, opts, caller)
+          check(:grep, machines, opts, caller)
+        end
+      end
+
+      def check_port(machines, port, caller)
+        check_command(machines, 
+          "netstat -na | grep 'LISTEN' | grep '\\(\\:\\|\\.\\)#{port} ' | grep -v grep",
+          "Checking if port #{port} is open on %s",
+          "Port #{port} seems down on %s", caller)
+      end
+      
+      def check_pid_file(machines, file, caller)
+        check_command(machines, "ps -p `cat #{file}`",
+          "Checking if valid pidfile #{file} exists in %s",
+          "Pidfile #{file} doesnt seem valid at %s", caller)
+      end
+
+      def check_grep(machines, grep, caller)
+        check_command(machines, "ps aux | grep '#{grep}' | grep -v grep",
+          "Grepping the process list for '#{grep}' in %s",
+          "Grepping the process list for '#{grep}' failed at %s", caller)
+      end
+
+      def check_command(machines, cmd, check_msg, error_msg, caller)
+        puts check_msg % machines.join(", ")
+        bad_machines = []
+        run_command(machines,
+          "#{cmd} || echo BAD"
+        ).each { |machine, output|
+          bad_machines << machine if output.join(" ").match(/BAD/)
+        }
+        check_command_failed(bad_machines, error_msg, caller) unless
+          bad_machines.empty?
+      rescue Exception => e
+        puts "Exception: #{e.to_s}"
+        puts "#{e.backtrace.join("\n")}"
+        check_command_failed(machines, error_msg, caller)
+      end
+
+      def check_command_failed(machines, error_msg, caller)
+        assert false, error_msg % machines.join(", "), caller
+      end
+
+      def check(entity, machines, opts, caller)
+        [opts[entity]].flatten.each { |val|
+          Process.send("check_#{entity}".to_sym, machines, val, caller)
+        } if opts[entity]
+      end
+
+      private
+      def puts(*args)
+        super unless AlertMachine.test_mode?
+      end
+    end
+  end
+  
+end

data/lib/rails_environment.rb

@@ -0,0 +1,27 @@
+class AlertMachine
+  class RailsEnvironment
+
+    ENVIRONMENT_PATH = "./config/environment.rb"
+    def self.bootup(path = ENVIRONMENT_PATH)
+      require path
+    rescue Exception => e
+      puts "Exception: #{e.to_s}"
+      puts e.backtrace.join("\n")
+      config = AlertMachine.config
+      ActionMailer::Base.mail(
+        :from => config['from'],
+        :to => config['to'],
+        :subject => "AlertMachine Failed: Environment could not load."
+      ) do |format|
+        format.text {
+          render :text => <<TXT
+          machine: #{`hostname`}
+          exception: #{e.to_s}
+          #{e.caller.join('\n')}
+TXT
+        }
+      end.deliver
+    end
+    
+  end
+end

data/lib/run_task.rb

@@ -0,0 +1,122 @@
+# A single watch and it's life cycle.
+class AlertMachine
+  class RunTask
+    def initialize(opts, block, caller)
+      @opts, @block, @caller = opts, block, caller
+      @errors = []
+      @alert_state = false
+    end
+
+    def schedule
+      @timer = EM::PeriodicTimer.new(interval) do
+        with_task do
+          start = Time.now
+          begin
+            # The main call to the user-defined watcher function.
+            @block.call(*@opts[:args])
+
+            assert(Time.now - start < interval / 5.0,
+              "Task ran for too long. Invoked every #{
+              interval}s. Ran for #{Time.now - start}s.", @caller) unless
+              dont_check_long_processes?
+
+            # Things finished successfully.
+            @timer.interval = interval if !@errors.empty?
+            @errors = []
+
+            alert_state(false)
+
+          rescue Exception => af
+            unless af.is_a?(AssertionFailure)
+              puts "Task Exception: #{af.to_s}"
+              puts "#{af.backtrace.join("\n")}"
+              af = AssertionFailure.new(af.to_s, af.backtrace)
+            end
+
+            @timer.interval = interval_error if @errors.empty?
+            @errors << af
+
+            alert_state(true) if @errors.length > retries
+          end
+        end
+      end
+    end
+
+    def with_task
+      AlertMachine.current_task = self
+      yield
+    ensure
+      AlertMachine.current_task = nil
+    end
+
+    def assert(condition, msg, caller)
+      return if condition
+      assert_failed(msg, caller)
+    end
+
+    def assert_failed(msg, caller)
+      fail = AssertionFailure.new(msg, caller)
+      puts fail.log
+      raise fail
+    end
+
+    # Is the alert firing?
+    def alert_state(firing)
+      if firing != @alert_state
+        mail unless @last_mailed && @last_mailed > Time.now - 60*10 && firing
+        @last_mailed = Time.now
+      end
+      @alert_state = firing
+    end
+
+    def mail
+      last = @errors[-1]
+      ActionMailer::Base.mail(
+        :from => opts(:from),
+        :to => opts(:to),
+        :subject => "AlertMachine Failed: #{last.msg || last.parsed_caller.file_line}",
+        :body => @errors.collect {|e| e.log}.join("\n=============\n")
+      ).deliver
+    end
+
+    def opts(key, defaults = nil)
+      @opts[key] || config[key.to_s] || defaults || block_given? && yield
+    end
+
+    def interval; opts(:interval, 5 * 60).to_f; end
+
+    def interval_error; opts(:interval_error) { interval / 5.0 }.to_f; end
+
+    def retries; opts(:retries, 1).to_i; end
+
+    def dont_check_long_processes?; opts(:dont_check_long_processes, false).to_s == "true"; end
+
+    def config; AlertMachine.config; end
+
+    # When an assertion fails, this exception is thrown so that
+    # we can unwind the stack frame. It's also deliberately throwing
+    # something that's not derived from Exception.
+    class AssertionFailure < Exception
+      attr_reader :msg, :caller, :time
+      def initialize(msg, caller)
+        @msg, @caller, @time = msg, caller, Time.now
+        super(@msg)
+      end
+
+      def log
+        "[#{Time.now}] #{msg ? msg + "\n" : ""}" +
+          "#{Caller.new(caller).log}"
+      end
+
+      def parsed_caller
+        Caller.new(caller)
+      end
+    end
+
+
+    private
+    def puts(*args)
+      super unless AlertMachine.test_mode?
+    end
+  end
+end

data/test/all.rb

@@ -0,0 +1,2 @@
+require File.expand_path(File.dirname(__FILE__) + '/process.rb')
+require File.expand_path(File.dirname(__FILE__) + '/watcher.rb')

data/test/helper.rb

@@ -0,0 +1,46 @@
+require 'rubygems'
+require 'test/unit'
+require 'mocha'
+require File.dirname(__FILE__) + '/../lib/alert_machine.rb'
+
+AlertMachine.disable
+
+class AlertMachineTestHelper < Test::Unit::TestCase
+  def setup(runs_long = false)
+    AlertMachine.reset
+    AlertMachine.expects(:config).returns(
+      {
+        'dont_check_long_processes' => runs_long ? "true" : "false",
+        'ssh' => {
+        }
+      }
+    ).at_least(0)
+    AlertMachine.expects(:test_mode?).returns(true).at_least(0)
+  end
+
+  def watcher(opts = {})
+    Class.new(AlertMachine::Watcher) do
+      watch opts.merge(:interval => 0.05) do
+        yield
+      end
+    end
+  end
+
+  def process_watcher(opts = {})
+    Class.new(AlertMachine::Watcher) do
+      watch_process "localhost", {interval: 0.05}.merge(opts)
+    end
+  end
+
+  def run_machine
+    AlertMachine.disable(false)
+    AlertMachine.run {
+      EM::Timer.new(0.1) do
+        EM::stop_event_loop
+      end
+      yield if block_given?
+    }
+  ensure
+    AlertMachine.disable
+  end
+end

data/test/process.rb

@@ -0,0 +1,52 @@
+require File.expand_path(File.dirname(__FILE__) + '/helper.rb')
+
+class ProcessTest < AlertMachineTestHelper
+
+  def setup
+    super(true)
+  end
+  
+  def test_port_open_failuire
+    process_watcher(:port => 3343)
+    AlertMachine::RunTask.any_instance.expects(:assert_failed).at_least_once
+    run_machine
+  end
+
+  def test_port_open_success
+    process_watcher(:port => 3343)
+    AlertMachine::RunTask.any_instance.expects(:assert_failed).never
+    run_machine {
+      EM::start_server "localhost", 3343 do
+      end
+    }
+  end
+
+  def test_pid_file_failuire
+    `rm -f /tmp/pid_x; touch /tmp/pid_x`
+    process_watcher(:pid_file => "/tmp/pid_x")
+    AlertMachine::RunTask.any_instance.expects(:assert_failed).at_least_once
+    run_machine
+  end
+
+  def test_pid_file_success
+    `rm -f /tmp/pid_x; touch /tmp/pid_x`
+    process_watcher(:pid_file => "/tmp/pid_x")
+    AlertMachine::RunTask.any_instance.expects(:assert_failed).never
+    File.open("/tmp/pid_x", "w") {|fh| fh.write "#{Process.pid}" }
+    run_machine
+  end
+
+  def test_grep_failuire
+    process_watcher(:grep => "test/stupid.rb")
+    AlertMachine::RunTask.any_instance.expects(:assert_failed).at_least_once
+    run_machine
+  end
+
+  def test_grep_success
+    process_watcher(:grep => "test/process.rb\\|test/all.rb")
+    AlertMachine::RunTask.any_instance.expects(:assert_failed).never
+    File.open("/tmp/pid_x", "w") {|fh| fh.write "#{Process.pid}" }
+    run_machine
+  end
+
+end

data/test/watcher.rb

@@ -0,0 +1,31 @@
+require File.expand_path(File.dirname(__FILE__) + '/helper.rb')
+
+class WatcherTest < AlertMachineTestHelper
+
+  def test_no_alerts_triggerred
+    watcher {}
+    AlertMachine::RunTask.any_instance.expects(:assert_failed).never
+    run_machine
+  end
+
+  def test_alerts_for_long_running_processes
+    watcher { sleep 0.05 }
+    AlertMachine::RunTask.any_instance.expects(:assert_failed).at_least_once
+    run_machine
+  end
+
+  def test_no_alerts_before_retries
+    cnt = 0
+    watcher(:retries => 1) { AlertMachine::Watcher.assert false if (cnt += 1) <= 1 }
+    AlertMachine::RunTask.any_instance.expects(:mail).never
+    run_machine
+  end
+
+  def test_alert_fires_after_retries
+    cnt = 0
+    watcher(:retries => 1) { AlertMachine::Watcher.assert false if (cnt += 1) <= 2 }
+    AlertMachine::RunTask.any_instance.expects(:mail).at_least_once
+    run_machine
+  end
+  
+end

metadata

@@ -0,0 +1,92 @@
+--- !ruby/object:Gem::Specification
+name: alert_machine
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- prasanna
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-01-15 00:00:00.000000000Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rye
+  requirement: &70208510413660 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70208510413660
+- !ruby/object:Gem::Dependency
+  name: actionmailer
+  requirement: &70208510412880 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70208510412880
+- !ruby/object:Gem::Dependency
+  name: eventmachine
+  requirement: &70208510412160 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70208510412160
+description: Make sure you get mailed when bad things happen to your server.
+email:
+- myprasanna@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- README
+- alert_machine.gemspec
+- lib/alert_machine.rb
+- lib/process.rb
+- lib/rails_environment.rb
+- lib/run_task.rb
+- test/all.rb
+- test/helper.rb
+- test/process.rb
+- test/watcher.rb
+homepage: http://github.com/likealittle/alert_machine
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: alert_machine
+rubygems_version: 1.8.10
+signing_key: 
+specification_version: 3
+summary: Ruby way of alerting server events.
+test_files:
+- test/all.rb
+- test/helper.rb
+- test/process.rb
+- test/watcher.rb