flapjack 0.4.12 → 0.5.1

data/features/persistence/steps/sqlite3_steps.rb

@@ -0,0 +1,13 @@
+Given /^I set up the Sqlite3 backend with the following options:$/ do |table|
+  @backend_options = table.hashes.first
+  @backend = Flapjack::Persistence::Sqlite3.new(@backend_options)
+end
+
+Then /^the check with id "([^\"]*)" on the Sqlite3 backend should have an event created$/ do |id|
+  @backend.all_events_for(id).size.should > 0
+end
+
+Then /^the check with id "([^\"]*)" on the Sqlite3 backend should have a status of "([^\"]*)"$/ do |id, status|
+  @backend.get_check(id)["status"].should == status
+end
+

data/features/steps/flapjack-notifier-manager_steps.rb

@@ -0,0 +1,24 @@
+__DIR__ = File.expand_path(File.join(File.dirname(__FILE__), '..', '..'))
+bin_path = File.join(__DIR__, 'bin')
+
+Given /^there are no instances of flapjack\-notifier running$/ do
+  command = "#{bin_path}/flapjack-notifier-manager stop"
+  silent_system(command)
+
+  sleep 0.5 # wait for the notifier manager
+
+  output = `ps -eo cmd |grep ^flapjack-notifier`
+  output.split.size.should == 0
+end
+
+Given /^there is an instance of the flapjack\-notifier running$/ do
+  command = "#{bin_path}/flapjack-notifier-manager start"
+  command += " --recipients spec/fixtures/recipients.yaml --config spec/fixtures/flapjack-notifier.yaml"
+  silent_system(command).should be_true
+  
+  sleep 0.5 # again, waiting for the notifier manager
+
+  output = `ps -eo cmd |grep ^flapjack-notifier`
+  output.split.size.should == 1
+end
+

data/features/steps/flapjack-worker-manager_steps.rb

@@ -0,0 +1,50 @@
+__DIR__ = File.expand_path(File.join(File.dirname(__FILE__), '..', '..'))
+
+Given /^the (.+) is on my path$/ do |command|
+  @bin_path = File.join(__DIR__, 'bin')
+  # (and is executable)
+  silent_system("test -x #{@bin_path}/#{command}").should be_true
+end
+
+Given /^the "([^\"]*)" directory exists and is writable$/ do |directory|
+  File.exists?(directory).should be_true
+  File.writable?(directory).should be_true
+end
+
+When /^I run "([^\"]*)"$/ do |cmd|
+  parts = [cmd]
+  parts.unshift(@bin_path) if @bin_path
+  command = parts.join('/')
+  # this might be dodgy
+  @output = `#{command}`
+  #@output.size.should > 0
+  #p command
+  #p @output
+end
+
+Then /^(\d+) instances of "([^\"]*)" should be running$/ do |number, command|
+  sleep 0.5 # this truly is a dodgy hack. 
+            # sometimes the the worker manager can take a while to fork
+  output = `ps -eo cmd |grep ^#{command} |grep -v grep`
+  output.split.size.should >= number.to_i
+end
+
+Given /^there are (\d+) instances of the flapjack\-worker running$/ do |number|
+  command = "#{@bin_path}/flapjack-worker-manager start --workers=5"
+  silent_system(command).should be_true
+end
+
+Given /^there are no instances of flapjack\-worker running$/ do
+  command = "#{@bin_path}/flapjack-worker-manager stop"
+  silent_system(command)
+
+  sleep 0.5 # again, waiting for the worker manager
+
+  output = `ps -eo cmd |grep ^flapjack-worker |grep -v grep`
+  output.split("\n").size.should == 0
+end
+
+Given /^beanstalkd is running on localhost$/ do
+  output = `ps -eo cmd |grep beanstalkd |grep -v grep`
+  output.split("\n").size.should == 1
+end

data/features/steps/packaging-lintian_steps.rb

@@ -0,0 +1,13 @@
+Given /^I am at the project root$/ do
+  Dir.pwd.split('/').last.should == "flapjack"
+end
+
+Then /^I should see (\d+) lines of output$/ do |number|
+  @output.split.size.should == number.to_i
+end
+
+Then /^every file in the output should start with "([^\"]*)"$/ do |string|
+  @output.split.each do |file|
+    `head -n 1 #{file}`.should =~ /^#{string}\s*$/
+  end
+end

data/features/support/env.rb

@@ -0,0 +1,22 @@
+#!/usr/bin/env ruby 
+
+$: << File.expand_path(File.join(File.dirname(__FILE__), '..', '..', 'lib'))
+
+require 'flapjack/inifile'
+require 'flapjack/filters/ok'
+require 'flapjack/filters/any_parents_failed'
+require 'flapjack/notifier_engine'
+require 'flapjack/transports/result'
+require 'flapjack/transports/beanstalkd'
+require 'flapjack/patches'
+require 'flapjack/inifile'
+require 'flapjack/cli/worker_manager'
+require 'flapjack/cli/notifier_manager'
+require 'flapjack/cli/notifier'
+require 'flapjack/cli/worker'
+require 'flapjack/persistence/couch'
+require 'flapjack/persistence/sqlite3'
+require 'flapjack/applications/notifier'
+require 'flapjack/applications/worker'
+require 'flapjack/notifiers/mailer/init'
+require 'flapjack/notifiers/xmpp/init'

data/features/support/silent_system.rb

@@ -0,0 +1,4 @@
+def silent_system(cmd)
+  #silent_cmd = cmd + " 2>&1 > /dev/null"
+  @output = `#{cmd}`
+end

data/flapjack.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
   s.name = 'flapjack'
-  s.version = '0.4.12'
-  s.date = '2009-11-22'
+  s.version = '0.5.1'
+  s.date = '2010-02-28'
   
   s.summary = "a scalable and distributed monitoring system"
   s.description = "Flapjack is highly scalable and distributed monitoring system. It understands the Nagios plugin format, and can easily be scaled from 1 server to 1000."
@@ -13,19 +13,15 @@ Gem::Specification.new do |s|
 
   s.add_dependency('daemons', '= 1.0.10')
   s.add_dependency('beanstalk-client', '= 1.0.2')
-  s.add_dependency('log4r', '= 1.0.5')
+  s.add_dependency('log4r', '= 1.1.5')
   s.add_dependency('xmpp4r', '= 0.5')
   s.add_dependency('tmail', '= 1.2.3.1')
-  s.add_dependency('dm-core', '= 0.9.11')
-  s.add_dependency('dm-timestamps', '= 0.9.11')
-  s.add_dependency('dm-types', '= 0.9.11')
-  s.add_dependency('dm-validations', '= 0.9.11')
-  s.add_dependency('data_objects', '= 0.9.12')
-  s.add_dependency('do_sqlite3', '= 0.9.12')
+  s.add_dependency('yajl-ruby', '= 0.6.4')
+  s.add_dependency('sqlite3-ruby', '= 1.2.5')
  
   s.bindir = "bin"
-  s.executables = ["flapjack-notifier", "flapjack-notifier-manager", "flapjack-stats", "flapjack-worker", "flapjack-worker-manager", "install-flapjack-systemwide"]
-  s.files = ["LICENCE", "README.md", "Rakefile", "TODO.md", "bin/flapjack-notifier", "bin/flapjack-notifier-manager", "bin/flapjack-stats", "bin/flapjack-worker", "bin/flapjack-worker-manager", "bin/install-flapjack-systemwide", "doc/CONFIGURING.md", "doc/DEVELOPING.md", "doc/INSTALL.md", "etc/default/flapjack-notifier", "etc/default/flapjack-workers", "etc/flapjack/flapjack-notifier.yaml.example", "etc/flapjack/recipients.yaml.example", "etc/init.d/flapjack-notifier", "etc/init.d/flapjack-workers", "flapjack.gemspec", "lib/flapjack/checks/http_content", "lib/flapjack/cli/notifier.rb", "lib/flapjack/cli/notifier_manager.rb", "lib/flapjack/cli/worker.rb", "lib/flapjack/cli/worker_manager.rb", "lib/flapjack/database.rb", "lib/flapjack/models/check.rb", "lib/flapjack/models/check_template.rb", "lib/flapjack/models/node.rb", "lib/flapjack/models/related_check.rb", "lib/flapjack/notifier.rb", "lib/flapjack/notifiers/mailer/init.rb", "lib/flapjack/notifiers/mailer/mailer.rb", "lib/flapjack/notifiers/xmpp/init.rb", "lib/flapjack/notifiers/xmpp/xmpp.rb", "lib/flapjack/patches.rb", "lib/flapjack/result.rb"]
+  s.executables = ["flapjack-benchmark", "flapjack-notifier", "flapjack-notifier-manager", "flapjack-stats", "flapjack-worker", "flapjack-worker-manager", "install-flapjack-systemwide" ]
+  s.files = ["LICENCE", "README.md", "Rakefile", "TODO.md", "bin/flapjack-benchmark", "bin/flapjack-notifier", "bin/flapjack-notifier-manager", "bin/flapjack-stats", "bin/flapjack-worker", "bin/flapjack-worker-manager", "bin/install-flapjack-systemwide", "doc/CONFIGURING.md", "doc/DEVELOPING.md", "doc/INSTALL.md", "doc/PACKAGING.md", "etc/default/flapjack-notifier", "etc/default/flapjack-workers", "etc/flapjack/flapjack-notifier.conf.example", "etc/flapjack/recipients.conf.example", "etc/init.d/flapjack-notifier", "etc/init.d/flapjack-workers", "features/flapjack-notifier-manager.feature", "features/flapjack-worker-manager.feature", "features/packaging-lintian.feature", "features/persistence/couch.feature", "features/persistence/sqlite3.feature", "features/persistence/steps/couch_steps.rb", "features/persistence/steps/generic_steps.rb", "features/persistence/steps/sqlite3_steps.rb", "features/steps/flapjack-notifier-manager_steps.rb", "features/steps/flapjack-worker-manager_steps.rb", "features/steps/packaging-lintian_steps.rb", "features/support/env.rb", "features/support/silent_system.rb", "flapjack.gemspec", "lib/flapjack/applications/notifier.rb", "lib/flapjack/applications/worker.rb", "lib/flapjack/checks/http_content", "lib/flapjack/checks/ping", "lib/flapjack/cli/notifier.rb", "lib/flapjack/cli/notifier_manager.rb", "lib/flapjack/cli/worker.rb", "lib/flapjack/cli/worker_manager.rb", "lib/flapjack/filters/any_parents_failed.rb", "lib/flapjack/filters/ok.rb", "lib/flapjack/inifile.rb", "lib/flapjack/notifier_engine.rb", "lib/flapjack/notifiers/mailer/init.rb", "lib/flapjack/notifiers/mailer/mailer.rb", "lib/flapjack/notifiers/xmpp/init.rb", "lib/flapjack/notifiers/xmpp/xmpp.rb", "lib/flapjack/patches.rb", "lib/flapjack/persistence/couch.rb", "lib/flapjack/persistence/couch/connection.rb", "lib/flapjack/persistence/couch/couch.rb", "lib/flapjack/persistence/data_mapper.rb", "lib/flapjack/persistence/data_mapper/data_mapper.rb", "lib/flapjack/persistence/data_mapper/models/check.rb", "lib/flapjack/persistence/data_mapper/models/check_template.rb", "lib/flapjack/persistence/data_mapper/models/event.rb", "lib/flapjack/persistence/data_mapper/models/node.rb", "lib/flapjack/persistence/data_mapper/models/related_check.rb", "lib/flapjack/persistence/sqlite3.rb", "lib/flapjack/persistence/sqlite3/sqlite3.rb", "lib/flapjack/transports/beanstalkd.rb", "lib/flapjack/transports/result.rb"]
 end
 
 

data/lib/flapjack/applications/notifier.rb

@@ -0,0 +1,222 @@
+#!/usr/bin/env ruby 
+
+$: << File.expand_path(File.join(File.dirname(__FILE__), '..', '..'))
+require 'log4r'
+require 'log4r/outputter/syslogoutputter'
+require 'flapjack/patches'
+require 'flapjack/notifier_engine'
+
+module Flapjack
+  module Notifier
+    class Application
+     
+      # boots the notifier
+      def self.run(options={})
+        app = self.new(options)
+        app.setup_config
+        app.setup_loggers
+        app.setup_notifiers
+        app.setup_notifier_engine
+        app.setup_recipients
+        app.setup_persistence
+        app.setup_queues
+        app.setup_filters
+
+        app
+      end
+
+      attr_accessor :log, :recipients
+
+      def initialize(options={})
+        @log = options[:log]
+        @notifier_directories = options[:notifier_directories]
+        @filter_directories = options[:filter_directories]
+        @options = options
+      end
+
+      def setup_loggers
+        unless @log
+	        @log = Log4r::Logger.new("notifier")
+	        @log.add(Log4r::StdoutOutputter.new("notifier"))
+	        @log.add(Log4r::SyslogOutputter.new("notifier"))
+        end
+      end
+
+      def setup_config
+        @config = OpenStruct.new(@options)
+      end
+
+      def setup_notifiers
+        @notifier_directories ||= []
+
+        default_directory = File.expand_path(File.join(File.dirname(__FILE__), '..', 'notifiers'))
+        # the default directory should be the last in the list
+        if @notifier_directories.include?(default_directory)
+          @notifier_directories << @notifier_directories.delete(default_directory)
+        else
+          @notifier_directories << default_directory
+        end
+     
+        # filter to the directories that actually exist
+        @notifier_directories = @notifier_directories.find_all do |dir|
+          if File.exists?(dir)
+            true
+          else
+            @log.warning("Notifiers directory #{dir} doesn't exist. Skipping.")
+            false
+          end
+        end
+
+        @notifiers = []
+      
+        # load up the notifiers and pass a config
+        @config.notifiers.each_pair do |notifier, config|
+          filenames = @notifier_directories.map {|dir| File.join(dir, notifier.to_s, 'init' + '.rb')}
+          filename = filenames.find {|filename| File.exists?(filename)}
+
+          if filename
+            @log.info("Loading the #{notifier.to_s.capitalize} notifier (from #{filename})")
+            require filename
+            config.merge!(:log => @log)
+            notifier = Flapjack::Notifiers.const_get("#{notifier.to_s.capitalize}").new(config)
+            @notifiers << notifier
+          else
+            @log.warning("Flapjack::Notifiers::#{notifier.to_s.capitalize} doesn't exist!")
+          end
+        end
+
+      end
+
+      def setup_notifier_engine
+        options = { :log => @log, :notifiers => @notifiers }
+        @notifier_engine = Flapjack::NotifierEngine.new(options)
+      end
+
+      def setup_recipients
+        @recipients ||= []
+     
+        @recipients += (@config.recipients || [])
+        # so poking at a recipient within notifiers is easier
+        @recipients.map! do |recipient|
+          OpenStruct.new(recipient)
+        end
+      end
+
+      def setup_persistence
+        defaults = { :backend => :data_mapper,
+                     :log => @log }
+        config = defaults.merge(@config.persistence || {})
+        basedir = config.delete(:basedir) || File.join(File.dirname(__FILE__), '..', 'persistence')
+       
+        filename = File.join(basedir, "#{config[:backend]}.rb")
+        class_name = config[:backend].to_s.camel_case
+
+        @log.info("Loading the #{class_name} persistence backend")
+        
+        begin 
+          require filename
+          @persistence = Flapjack::Persistence.const_get(class_name).new(config)
+        rescue LoadError => e
+          @log.warning("Attempted to load #{class_name} persistence backend, but it doesn't exist!")
+          @log.warning("Exiting.")
+          raise # preserves original exception
+        end
+
+      end
+
+      def setup_queues
+        defaults = { :backend => :beanstalkd, 
+                     :host => 'localhost', 
+                     :port => '11300',
+                     :queue_name => 'results',
+                     :log => @log }
+        config = defaults.merge(@config.transport || {})
+        basedir = config.delete(:basedir) || File.join(File.dirname(__FILE__), '..', 'transports')
+
+        class_name = config[:backend].to_s.camel_case
+        filename = File.join(basedir, "#{config[:backend]}.rb")
+
+        @log.info("Loading the #{class_name} transport")
+      
+        begin 
+          require filename
+          @results_queue = Flapjack::Transport.const_get(class_name).new(config)
+        rescue LoadError => e
+          @log.warning("Attempted to load #{class_name} transport, but it doesn't exist!")
+          @log.warning("Exiting.")
+          raise # preserves original exception
+        end
+      end
+
+      def setup_filters 
+        @filter_directories ||= []
+
+        default_directory = File.expand_path(File.join(File.dirname(__FILE__), '..', 'filters'))
+        # the default directory should be the last in the list
+        if @filter_directories.include?(default_directory)
+          @filter_directories << @filter_directories.delete(default_directory)
+        else
+          @filter_directories << default_directory
+        end
+     
+        # filter to the directories that actually exist
+        @filter_directories = @filter_directories.find_all do |dir|
+          if File.exists?(dir)
+            true
+          else
+            @log.warning("Filters directory #{dir} doesn't exist. Skipping.")
+            false
+          end
+        end
+
+        @filters = []
+
+        @config.filters.each do |filter|
+          filenames = @filter_directories.map {|dir| File.join(dir, filter.to_s + '.rb')}
+          filename = filenames.find {|filename| File.exists?(filename)}
+
+          if filename 
+            @log.info("Loading the #{filter.camel_case} filter (from #{filename})")
+            require filename
+            filter = Flapjack::Filters.const_get(filter.camel_case).new(:log => @log, :persistence => @persistence)
+            @filters << filter
+          else
+            @log.warning("Flapjack::Filters::#{filter.camel_case} doesn't exist!")
+          end
+        end
+        
+      end
+
+      def process_result
+        @log.debug("Waiting for new result...")
+        result = @results_queue.next # this blocks until a result is popped
+        
+        @log.info("Processing result for check #{result.check_id}.")
+        event = @persistence.create_event(result)
+       
+
+        block = @filters.find {|filter| filter.block?(result) }
+        unless block
+          # do munging
+          @notifier_engine.notify!(:result => result, 
+                                   :event => event, 
+                                   :recipients => recipients)
+        end
+
+        @log.info("Storing status of check.")
+        @persistence.save(result)
+
+        @log.info("Deleting result for check #{result.check_id}.")
+        @results_queue.delete(result)
+      end
+
+      def main
+        @log.info("Booting main loop.")
+        loop do 
+          process_result
+        end
+      end
+
+    end
+  end
+end

data/lib/flapjack/applications/worker.rb

@@ -0,0 +1,99 @@
+#!/usr/bin/env ruby 
+
+require 'log4r'
+require 'log4r/outputter/syslogoutputter'
+require File.expand_path(File.join(File.dirname(__FILE__), '..', 'patches'))
+
+module Flapjack
+  module Worker
+    class Application
+     
+      # boots the notifier
+      def self.run(options={})
+        app = self.new(options)
+        app.setup_loggers
+        app.setup_config
+        app.setup_queues
+
+        app
+      end
+
+      attr_accessor :log
+
+      def initialize(options={})
+        @log = options[:log]
+        @notifier_directories = options[:notifier_directories]
+        @options = options
+      end
+
+      def setup_loggers
+        unless @log
+	        @log = Log4r::Logger.new("notifier")
+	        @log.add(Log4r::StdoutOutputter.new("notifier"))
+	        @log.add(Log4r::SyslogOutputter.new("notifier"))
+        end
+      end
+
+      def setup_config
+        @config = OpenStruct.new(@options)
+      end
+
+      def setup_queues
+        defaults = { :backend => :beanstalkd, 
+                     :host => 'localhost', 
+                     :port => '11300',
+                     :log => @log }
+        config = defaults.merge(@config.transport || {})
+        basedir = config.delete(:basedir) || File.join(File.dirname(__FILE__), '..', 'transports')
+
+        %w(results checks).each do |queue_name|
+          
+          queue_config = config.merge(:queue_name => queue_name)
+       
+          class_name = config[:backend].to_s.camel_case
+          filename = File.join(basedir, "#{config[:backend]}.rb")
+          
+          @log.info("Loading the #{class_name} transport for queue: #{queue_name}.")
+
+          begin 
+            require filename
+            queue = Flapjack::Transport.const_get("#{class_name}").new(queue_config)
+            instance_variable_set("@#{queue_name}_queue", queue)
+          rescue LoadError => e
+            @log.warning("Attempted to load #{class_name} transport, but it doesn't exist!")
+            @log.warning("Exiting.")
+            raise # preserves original exception
+          end
+        end
+      end
+
+      def process_check
+        @log.info("Waiting for check...")
+        check = @checks_queue.next
+        @log.info("Processing check with id #{check.check_id}")
+
+        command = "sh -c '#{check.command}'"
+        @log.info("Executing check: #{command}")
+
+        output = `#{command}`
+        return_value = $?.exitstatus
+
+        @log.info("Sending result.")
+        @results_queue.put({:check_id => check.check_id, :output => output, :retval => return_value})
+        @log.info("Returning check to transport.")
+        @checks_queue.put({:check_id => check.check_id, :command => check.command, :frequency => check.frequency})
+
+        @log.info("Cleaning up check.")
+        @checks_queue.delete(check)
+      end
+
+      def main
+        @log.info("Booting main loop.")
+        loop do 
+          process_check
+        end
+      end
+
+    end
+  end
+end