flapjack 0.5.5 → 0.6.23

data/Rakefile

@@ -1,176 +1,79 @@
 #!/usr/bin/env ruby
 
-require 'rubygems'
+unless $:.include?(File.dirname(__FILE__) + '/lib/')
+  $: << File.dirname(__FILE__) + '/lib'
+end
+
 require 'fileutils'
-require 'spec/rake/spectask'
 require 'rake'
 
-#
-# Release management
-#
-begin
-  require 'jeweler'
-  Jeweler::Tasks.new do |gem|
-    gem.name = "flapjack"
-    gem.summary = %Q{a scalable and distributed monitoring system}
-    gem.description = %Q{lapjack is highly scalable and distributed monitoring system. It understands the Nagios plugin format, and can easily be scaled from 1 server to 1000.}
-    gem.email = "lindsay@holmwood.id.au"
-    gem.homepage = "http://flapjack-project.com/"
-    gem.authors = ["Lindsay Holmwood"]
-    gem.has_rdoc = false
-
-    gem.add_dependency('daemons', '= 1.0.10')
-    gem.add_dependency('beanstalk-client', '= 1.0.2')
-    gem.add_dependency('log4r', '= 1.1.5')
-    gem.add_dependency('xmpp4r', '= 0.5')
-    gem.add_dependency('tmail', '= 1.2.3.1')
-    gem.add_dependency('yajl-ruby', '= 0.6.4')
-    gem.add_dependency('sqlite3-ruby', '= 1.2.5')
-
-    # Don't release unsanitised NetSaint config into gem.
-    gem.files.exclude('features/support/data/etc/netsaint')
-  end
-  Jeweler::GemcutterTasks.new
-rescue LoadError
-  puts "Jeweler (or a dependency) not available. Install it with: gem install jeweler"
-end
+require 'resque/tasks'
 
-#
-# Testing
-#
-require 'rake/testtask'
-Rake::TestTask.new(:test) do |test|
-  test.libs << 'lib' << 'test'
-  test.pattern = 'test/**/test_*.rb'
-  test.verbose = true
-end
+Dir['tasks/**/*.rake'].each { |t| load t }
 
-begin
-  require 'rcov/rcovtask'
-  Rcov::RcovTask.new do |test|
-    test.libs << 'test'
-    test.pattern = 'test/**/test_*.rb'
-    test.verbose = true
-  end
-rescue LoadError
-  task :rcov do
-    abort "RCov is not available. In order to run rcov, you must: sudo gem install spicycode-rcov"
-  end
+require 'cucumber'
+require 'cucumber/rake/task'
+require 'colorize'
+Cucumber::Rake::Task.new(:features) do |t|
+  t.cucumber_opts = "features --format pretty"
 end
 
-begin
-  require 'cucumber/rake/task'
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new(:spec)
 
-  Cucumber::Rake::Task.new do |task|
-    task.cucumber_opts = "--require features/"
-  end
-rescue LoadError
-end
+task :default => :spec
 
-Spec::Rake::SpecTask.new do |t|
-  t.spec_opts = ["--options", "spec/spec.opts"]
-end
-task :test => :check_dependencies
 
-task :default => :test
+desc "build gem"
+task :build => :verify do
+  build_output = `gem build flapjack.gemspec`
+  puts build_output
 
-require 'rake/rdoctask'
-Rake::RDocTask.new do |rdoc|
-  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+  gem_filename = build_output[/File: (.*)/,1]
+  pkg_path = "pkg"
+  FileUtils.mkdir_p(pkg_path)
+  FileUtils.mv(gem_filename, pkg_path)
 
-  rdoc.rdoc_dir = 'rdoc'
-  rdoc.title = "flapjack #{version}"
-  rdoc.rdoc_files.include('README*')
-  rdoc.rdoc_files.include('lib/**/*.rb')
+  puts "Gem built in #{pkg_path}/#{gem_filename}".green
 end
 
-#
-# misc
-#
-desc "display FIXMEs in the codebase"
-task :fixmes do
-  output = `grep -nR FIXME lib/* spec/* bin/`
-  output.split("\n").each do |line|
-    parts = line.split(':')
-    puts "#{parts[0].strip} +#{parts[1].strip}"
-    puts "  - #{parts[3].strip}"
+desc "push gem"
+task :push do
+  filenames = Dir.glob("pkg/*.gem")
+  filenames_with_times = filenames.map do |filename|
+    [filename, File.mtime(filename)]
   end
+
+  newest = filenames_with_times.sort_by { |tuple| tuple.last }.last
+  newest_filename = newest.first
+
+  command = "gem push #{newest_filename}"
+  system(command)
 end
 
-desc "build a tarball suitable for building packages from"
-task :tarball do
-  require 'open-uri'
-  require 'tmpdir'
-  tmpdir = Dir.mktmpdir
-
-  current_commit  = `git log`.first.split[1][0..5]
-  release_tarball = "/tmp/flapjack-#{current_commit}.tar.gz"
-  flapjack_path   = File.join(tmpdir, "flapjack-#{current_commit}")
-  repo_path       = File.expand_path(File.dirname(__FILE__))
-  ignores         = %w(.gitignore .git)
-
-  # create new staging directory
-  command = "git clone #{repo_path} #{flapjack_path}"
-  `#{command}`
-
-  # remove files we don't want to publish
-  ignores.each do |filename|
-    path = File.join(flapjack_path, filename)
-    FileUtils.rm_rf(path)
+desc "clean up various generated files"
+task :clean do
+  [ "pkg/"].each do |filename|
+    puts "Removing #{filename}"
+    FileUtils.rm_rf(filename)
   end
+end
 
-  deps = { :daemons => {
-             :source => "http://rubyforge.org/frs/download.php/34223/daemons-1.0.10.tgz",
-             :filename => "daemons-1.0.10.tar.gz"
-           },
-           :beanstalkd_client => {
-             :source => "http://github.com/kr/beanstalk-client-ruby/tarball/v1.0.2",
-             :filename => "beanstalkd-client-1.0.2.tar.gz"
-           },
-           :yajl_ruby => {
-             :source => "http://github.com/brianmario/yajl-ruby/tarball/0.6.4",
-             :filename => "yajl-ruby-0.6.4.tar.gz"
-           },
-           :tmail => {
-             :source => "http://rubyforge.org/frs/download.php/35297/tmail-1.2.3.1.tgz",
-             :filename => "tmail-1.2.3.1.tar.gz"
-           },
-           :xmpp4r => {
-             :source => "http://download.gna.org/xmpp4r/xmpp4r-0.4.tgz",
-             :filename => "xmpp4r-0.4.tar.gz"
-           },
-           :log4r => {
-             :source => "http://qa.debian.org/watch/sf.php/log4r/log4r-1.0.5.tgz",
-             :filename => "log4r-1.0.5.tar.gz"
-           }
-  }
-  deps.each_pair do |name, details|
-    puts "Pulling in #{name} dependency..."
-    # save file
-    saved_file = "/tmp/#{details[:filename]}"
-    File.open(saved_file, 'w') do |f|
-      f << open(details[:source]).read
+namespace :verify do
+  task :uncommitted do
+    uncommitted = `git ls-files -m`.split("\n")
+    if uncommitted.size > 0
+      puts "The following files are uncommitted:".red
+      uncommitted.each do |filename|
+        puts " - #{filename}".red
+      end
+      exit 1
     end
-
-    `tar zxf #{saved_file} -C #{tmpdir}`
   end
 
-  # build tarball
-  under = File.dirname(tmpdir)
-  staging = File.join("/tmp", "flapjack-#{current_commit}")
-  FileUtils.mv(tmpdir, staging)
-  actual = File.basename(staging)
-  command = "cd #{under} ; tar czf #{release_tarball} #{actual}"
-  system(command)
-
-  puts "Release tarball with deps at #{release_tarball}"
+  task :all => [ :uncommitted ]
 end
 
-desc "dump out statements to create sqlite3 schema"
-task :dm_debug do
-  require 'lib/flapjack/persistence/data_mapper'
-
-  DataMapper.logger.set_log(STDOUT, :debug)
-  DataMapper.setup(:default, "sqlite3:///tmp/sqlite3.db")
-  DataMapper.auto_migrate!
-end
+# FIXME: getting that intermittent gherken lexing error so removing :features from verify list
+#task :verify => [ 'verify:all', :spec, :features]
+task :verify => [ 'verify:all', :spec]

data/bin/flapjack

@@ -0,0 +1,72 @@
+#!/usr/bin/env ruby
+
+require 'yaml'
+require 'optparse'
+require 'ostruct'
+
+options = OpenStruct.new
+options.config    = File.join('etc', 'flapjack_config.yaml')
+options.daemonize = nil
+
+OptionParser.new do |opts|
+  opts.banner = "Usage: flapjack [options]"
+
+  opts.on("-c", "--config [PATH]", String, "PATH to the config file to use") do |c|
+    options.config = c
+  end
+
+  opts.on("-d", "--[no-]daemonize", "Daemonize?") do |d|
+    options.daemonize = d
+  end
+
+  opts.on("-p", "--pidfile [PATH]", String, "PATH to the pidfile to write to") do |p|
+    options.pidfile = p
+  end
+
+end.parse!(ARGV)
+
+FLAPJACK_ENV = ENV['FLAPJACK_ENV'] || 'development'
+
+# load the config hash for the current environment
+
+if File.file?(options.config)
+  config = YAML::load_file(options.config)
+else
+  puts "Could not find config file at '#{options.config}', you may want to specify one with the --config option"
+  exit(false)
+end
+
+config_env = config[FLAPJACK_ENV]
+
+if config_env.nil? || config_env.empty?
+  puts "No config data for environment '#{FLAPJACK_ENV}' found in '#{options.config}'"
+  exit(false)
+end
+
+if options.pidfile.nil?
+  pid_file = (config_env['pid_file'] || 'tmp/pids/flapjack.pid')
+else
+  pid_file = options.pidfile
+end
+
+# add lib to the default include path
+unless $:.include?(File.dirname(__FILE__) + '/../lib/')
+  $: << File.dirname(__FILE__) + '/../lib'
+end
+
+# TODO Flapjack falls over when Redis restarted -- trap errors and attempt reconnect
+
+require 'flapjack/coordinator'
+
+coordinator = Flapjack::Coordinator.new(config_env)
+coordinator.log_file = (config_env['log_file'] || 'log/flapjack.log')
+coordinator.pid_file = pid_file
+
+if options.daemonize.nil?
+  daemonize = !!config_env['daemonize']
+else
+  daemonize = options.daemonize
+end
+
+puts "Daemonising ... " if daemonize
+coordinator.start(:daemonize => daemonize)

data/bin/flapjack-nagios-receiver

@@ -0,0 +1,111 @@
+#!/usr/bin/env ruby
+
+require 'redis'
+require 'json'
+require 'yaml'
+require 'optparse'
+require 'ostruct'
+require 'flapjack/data/entity_check'
+
+options = OpenStruct.new
+options.config    = File.join('etc', 'flapjack_config.yaml')
+options.daemonize = nil
+
+OptionParser.new do |opts|
+  opts.banner = "Usage: flapjack-nagios-receiver [options]"
+
+  opts.on("-c", "--config [PATH]", String, "PATH to the config file to use") do |c|
+    options.config = c
+  end
+
+  opts.on("-f", "--fifo [PATH]", String, "PATH to the nagios perfdata named pipe") do |f|
+    options.fifo_path = f
+  end
+
+end.parse!(ARGV)
+
+FLAPJACK_ENV = ENV['FLAPJACK_ENV'] || 'development'
+
+# load the config hash for the current environment
+
+if File.file?(options.config)
+  config = YAML::load(File.open(options.config))
+else
+  puts "Could not find config file at '#{options.config}', you may want to specify one with the --config option"
+  exit(false)
+end
+
+config_env = config[FLAPJACK_ENV]
+
+if config_env.nil? || config_env.empty?
+  puts "No config data for environment '#{FLAPJACK_ENV}' found in '#{options.config}'"
+  exit(false)
+end
+
+# add lib to the default include path
+#unless $:.include?(File.dirname(__FILE__) + '/../lib/')
+#  $: << File.dirname(__FILE__) + '/../lib'
+#end
+
+# nagios.cfg contains the following templates for host and service data (modified from the default
+# to include hoststate / servicestate, and a fake service 'HOST' for hostperfdata, so that the
+# fields match up
+# host_perfdata_file_template=[HOSTPERFDATA]\t$TIMET$\t$HOSTNAME$\tHOST\t$HOSTSTATE$\t$HOSTEXECUTIONTIME$\t$HOSTLATENCY$\t$HOSTOUTPUT$\t$HOSTPERFDATA$
+# service_perfdata_file_template=[SERVICEPERFDATA]\t$TIMET$\t$HOSTNAME$\t$SERVICEDESC$\t$SERVICESTATE$\t$SERVICEEXECUTIONTIME$\t$SERVICELATENCY$\t$SERVICEOUTPUT$\t$SERVICEPERFDATA$
+
+@redis_host     = config_env['redis']['host'] || 'localhost'
+@redis_port     = config_env['redis']['port'] || '6379'
+@redis_path     = config_env['redis']['path'] || nil
+@redis_db       = config_env['redis']['db']   || 0
+
+if config_env.nil? || config_env.empty?
+  puts "No config data for environment '#{FLAPJACK_ENV}'"
+  exit(false)
+end
+
+def process_input
+
+  if @redis_path
+    redis = Redis.new(:db => @redis_db, :path => @redis_path)
+  else
+    redis = Redis.new(:db => @redis_db, :host => @redis_host, :port => @redis_port)
+  end
+
+  puts "ARGV after option processing looks like this: [#{ARGV.inspect}]"
+
+  begin
+    while line = ARGF.readline.split("\t")
+      object_type, timestamp, entity, check, state, check_time, check_latency, check_output, check_perfdata = line
+      puts "#{object_type}, #{timestamp}, #{entity}, #{check}, #{state}, #{check_output}"
+      next unless line.length == 9
+      next unless timestamp =~ /^\d+$/
+      next unless (object_type == '[HOSTPERFDATA]') or (object_type == '[SERVICEPERFDATA]')
+      state = 'ok'       if state.downcase == 'up'
+      state = 'critical' if state.downcase == 'down'
+      event = {
+        'entity'    => entity,
+        'check'     => check,
+        'type'      => 'service',
+        'state'     => state,
+        'summary'   => check_output,
+        'timestamp' => timestamp,
+      }.to_json
+      redis.rpush 'events', event
+    end
+  rescue Redis::CannotConnectError
+    puts "Error, unable to to connect to the redis server (#{$!})"
+  rescue EOFError
+    puts "Error - end of file reached when reading from the named pipe. Nagios down?"
+    sleep 10
+  end
+end
+
+def main
+  while true
+    process_input
+    sleep 10
+  end
+end
+
+main
+

data/bin/flapjack-nagios-receiver-control

@@ -0,0 +1,15 @@
+#!/usr/bin/env ruby
+
+require 'daemons'
+opts = {
+  :app_name   => 'flapjack-nagios-receiver',
+  :dir_mode   => :normal,
+  :dir        => '/var/run/flapjack',
+  :multiple   => false,
+  :backtrace  => true,
+  :monitor    => false,
+  :log_dir    => '/var/log/flapjack',
+  :log_output => true,
+}
+Daemons.run(File.join(File.dirname(__FILE__) ,'flapjack-nagios-receiver'), opts)
+

data/bin/flapjack-netsaint-parser

@@ -1,7 +1,5 @@
 #!/usr/bin/env ruby
 
-require 'pathname'
-require 'rubygems'
 require 'yajl/json_gem'
 
 module Netsaint

data/bin/flapjack-populator

@@ -1,29 +1,146 @@
 #!/usr/bin/env ruby
 
-require 'rubygems'
 require 'yajl'
-require 'beanstalk-client'
+require 'redis'
+require 'yaml'
+require 'optparse'
+require 'ostruct'
+require 'flapjack/data/entity_check'
 
-command = ARGV[0]
+options = OpenStruct.new
+options.config    = File.join('etc', 'flapjack_config.yaml')
+options.daemonize = nil
 
-case command
-when "deploy"
-  options = ARGV[1..-1]
-  options = Hash[options.map {|o| o.scan(/--(.+)=(.+)/).flatten }]
+command = ARGV.shift
+
+banner  = "Usage: flapjack-populator command [options]\n"
+banner += "\n"
+banner += "  commands:\n"
+banner += "    - import-entities\n"
+banner += "    - import-contacts\n"
+banner += "    - purge-events (purge queued monitoring events)\n"
+
+OptionParser.new do |opts|
+  opts.banner = "Usage: flapjack-populator command [options]"
+
+  opts.on("-c", "--config [PATH]", String, "PATH to the config file to use") do |c|
+    options.config = c
+  end
+
+  opts.on("-f", "--from [FILE]", String, "path to the FILE to import") do |f|
+    options.from = f
+  end
+
+end.parse!(ARGV)
+
+FLAPJACK_ENV = ENV['FLAPJACK_ENV'] || 'development'
+
+# load the config hash for the current environment
+
+if File.file?(options.config)
+  config = YAML::load(File.open(options.config))
+else
+  puts "Could not find config file at '#{options.config}', you may want to specify one with the --config option"
+  exit(false)
+end
+
+config_env = config[FLAPJACK_ENV]
 
-  @queue = Beanstalk::Connection.new('localhost:11300', 'checks')
+if config_env.nil? || config_env.empty?
+  puts "No config data for environment '#{FLAPJACK_ENV}' found in '#{options.config}'"
+  exit(false)
+end
 
-  filename = options["from"]
+if options.from
+  filename = options.from
   file     = File.new(filename)
-  parser   = Yajl::Parser.new
-  batch    = parser.parse(file)
-  batch_id = batch["batch"]["id"]
+end
+
+parser   = Yajl::Parser.new
+
+@redis_host = config_env['redis']['host'] || 'localhost'
+@redis_port = config_env['redis']['port'] || '6379'
+@redis_path = config_env['redis']['path'] || nil
+@redis_db   = config_env['redis']['db']   || 0
 
-  batch["checks"].each do |check|
-    job = check.merge({"batch_id" => batch_id})
-    @queue.yput(job)
+# add lib to the default include path
+unless $:.include?(File.dirname(__FILE__) + '/../lib/')
+  $: << File.dirname(__FILE__) + '/../lib'
+end
+
+def get_redis_connection
+  if @redis_path
+    redis = Redis.new(:db => @redis_db, :path => @redis_path)
+  else
+    redis = Redis.new(:db => @redis_db, :host => @redis_host, :port => @redis_port)
   end
+  redis
+end
+
+case command
+when "import-contacts"
+  @persistence = get_redis_connection
+  contacts = parser.parse(file)
+  contacts.each {|contact|
+    if not contact['id']
+      puts "Contact not imported as it has no id: " + contact.inspect
+      next
+    end
+    @persistence.multi
+    @persistence.del("contact:#{contact['id']}")
+    @persistence.del("contact_media:#{contact['id']}")
+    @persistence.del("contact_pagerduty:#{contact['id']}")
+    @persistence.hset("contact:#{contact['id']}", 'first_name', contact['first_name'])
+    @persistence.hset("contact:#{contact['id']}", 'last_name',  contact['last_name'])
+    @persistence.hset("contact:#{contact['id']}", 'email',      contact['email'])
+    contact['media'].each_pair {|medium, address|
+      case medium
+      when 'pagerduty'
+        @persistence.hset("contact_media:#{contact['id']}", medium, address['service_key'])
+        @persistence.hset("contact_pagerduty:#{contact['id']}", 'subdomain', address['subdomain'])
+        @persistence.hset("contact_pagerduty:#{contact['id']}", 'username',  address['username'])
+        @persistence.hset("contact_pagerduty:#{contact['id']}", 'password',  address['password'])
+      else
+        @persistence.hset("contact_media:#{contact['id']}", medium, address)
+      end
+    }
+    @persistence.exec
+  }
+
+when "import-entities"
+  @persistence = get_redis_connection
+  entities = parser.parse(file)
+  entities.each {|entity|
+    if not entity['id']
+      puts "Entity not imported as it has no id: " + entity.inspect
+      next
+    end
+    @persistence.multi
+    existing_name = @persistence.hget("entity:#{entity['id']}", 'name')
+    @persistence.del("entity_id:#{existing_name}") unless existing_name == entity['name']
+    @persistence.set("entity_id:#{entity['name']}", entity['id'])
+    @persistence.hset("entity:#{entity['id']}", 'name', entity['name'])
+
+    @persistence.del("contacts_for:#{entity['id']}")
+    entity['contacts'].each {|contact|
+      @persistence.sadd("contacts_for:#{entity['id']}", contact)
+    }
+    @persistence.exec
+
+  }
+
+when "purge-events"
+  @persistence = get_redis_connection
+  events_size = @persistence.llen('events')
+  puts "purging #{events_size} events..."
+  timestamp = Time.now.to_i
+  puts "renaming events to events.#{timestamp}"
+  @persistence.rename('events', "events.#{timestamp}")
+  puts "setting expiry of events.#{timestamp} to 8 hours"
+  @persistence.expire("events.#{timestamp}", (60 * 60 * 8))
 
-  puts "Deployed batch #{batch_id}"
+else
+  puts "you need to give me something to do, eg a command like 'import-entities' or 'import-clients' etc"
+  raise ArgumentError
 end