ey-big-brother 0.0.9

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2009 YOUR NAME
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,3 @@
+== ey-big-brother
+
+A gem that provides...

data/Rakefile

@@ -0,0 +1,60 @@
+require 'rubygems'
+require 'rake/gempackagetask'
+require 'rubygems/specification'
+require 'date'
+require 'spec/rake/spectask'
+
+GEM = "ey-big-brother"
+GEM_VERSION = "0.0.9"
+AUTHOR = "Ezra Zygmuntowicz"
+EMAIL = "ez@engineyard.com"
+HOMEPAGE = "http://engineyard.com"
+SUMMARY = "A gem that provides oversight of the ey-cloud"
+
+spec = Gem::Specification.new do |s|
+  s.name = GEM
+  s.version = GEM_VERSION
+  s.platform = Gem::Platform::RUBY
+  s.has_rdoc = true
+  s.extra_rdoc_files = ["README.rdoc", "LICENSE", 'TODO']
+  s.summary = SUMMARY
+  s.description = s.summary
+  s.author = AUTHOR
+  s.email = EMAIL
+  s.homepage = HOMEPAGE
+  
+  # Uncomment this to add a dependency
+  # s.add_dependency "foo"
+  s.bindir       = "bin"
+  s.executables  = %w(ey-monitor-agent ey-alert-agent ey-drain-queues)
+  
+  
+  s.require_path = 'lib'
+  s.autorequire = GEM
+  s.files = %w(LICENSE README.rdoc Rakefile TODO) + Dir.glob("{lib,spec}/**/*")
+end
+
+task :default => :spec
+
+desc "Run specs"
+Spec::Rake::SpecTask.new do |t|
+  t.spec_files = FileList['spec/**/*_spec.rb']
+  t.spec_opts = %w(-fs --color)
+end
+
+
+Rake::GemPackageTask.new(spec) do |pkg|
+  pkg.gem_spec = spec
+end
+
+desc "install the gem locally"
+task :install => [:package] do
+  sh %{sudo gem install pkg/#{GEM}-#{GEM_VERSION}}
+end
+
+desc "create a gemspec file"
+task :make_spec do
+  File.open("#{GEM}.gemspec", "w") do |file|
+    file.puts spec.to_ruby
+  end
+end

data/TODO

@@ -0,0 +1,4 @@
+TODO:
+Fix LICENSE with your name
+Fix Rakefile with your name and contact info
+Add your code to lib/<%= name %>.rb

data/bin/ey-alert-agent

@@ -0,0 +1,26 @@
+require 'alert_agent'
+require 'optparse'
+require 'ey_config'
+include EY::Config
+
+options = {:host => 'ec2-67-202-29-203.compute-1.amazonaws.com', :user => 'guest', :pass => 'guest', :vhost => '/production'}
+
+opts = OptionParser.new do |opts|
+  opts.banner = "Usage: ey-monitor-agent [-flag] [argument]"
+  opts.define_head "Ey Monitor Agent: watching over your cloud since way back when"
+  opts.separator '*'*80
+
+  setup_common_options(opts, options)
+end
+
+opts.parse!
+
+Signal.trap('INT') { AMQP.stop{ EM.stop } }
+Signal.trap('TERM'){ AMQP.stop{ EM.stop } }
+
+AMQP.start(:host => options[:host], :user => options[:user], 
+           :pass => options[:pass], :vhost => options[:vhost]) do
+  mq = MQ.new
+  mq.prefetch(10)
+  AlertAgent.new(mq)
+end

data/bin/ey-drain-queues

@@ -0,0 +1,43 @@
+require 'optparse'
+require 'ey_config'
+require 'alert_agent'
+require 'monitor_agent'
+
+include EY::Config
+
+options = {:host => 'ec2-67-202-29-203.compute-1.amazonaws.com', :user => 'guest', :pass => 'guest', :vhost => '/production'}
+
+opts = OptionParser.new do |opts|
+  opts.banner = "Usage: ey-drain-queues [-flag] [argument]"
+  opts.define_head "Draining ur q's since way back when"
+  opts.separator '*'*80
+
+  setup_common_options(opts, options)
+end
+
+opts.parse!
+
+Signal.trap('INT') { AMQP.stop{ EM.stop } }
+Signal.trap('TERM'){ AMQP.stop{ EM.stop } }
+
+AMQP.start(:host => options[:host], :user => options[:user], 
+           :pass => options[:pass], :vhost => options[:vhost]) do
+  mq = MQ.new
+  mq.prefetch(100)
+  mq.queue(MonitorAgent::QUEUE_NAME).subscribe(:ack => true) do |h,m|
+    if AMQP.closing?
+      Utils::LOG.info "(ignoring message, will be redelivered later)"
+    else
+      puts "drained message"
+      h.ack
+    end
+  end
+  mq.queue(AlertAgent::QUEUE_NAME).subscribe(:ack => true) do |h,m|
+    if AMQP.closing?
+      Utils::LOG.info "(ignoring message, will be redelivered later)"
+    else
+      puts "drained alert"
+      h.ack
+    end
+  end
+end

data/bin/ey-monitor-agent

@@ -0,0 +1,26 @@
+require 'monitor_agent'
+require 'optparse'
+require 'ey_config'
+include EY::Config
+
+options = {:host => 'ec2-67-202-29-203.compute-1.amazonaws.com', :user => 'guest', :pass => 'guest', :vhost => '/production'}
+
+opts = OptionParser.new do |opts|
+  opts.banner = "Usage: ey-monitor-agent [-flag] [argument]"
+  opts.define_head "Ey Monitor Agent: watching over your cloud since way back when"
+  opts.separator '*'*80
+
+  setup_common_options(opts, options)
+end
+
+opts.parse!
+
+Signal.trap('INT') { AMQP.stop{ EM.stop } }
+Signal.trap('TERM'){ AMQP.stop{ EM.stop } }
+
+AMQP.start(:host => options[:host], :user => options[:user], 
+           :pass => options[:pass], :vhost => options[:vhost]) do
+  mq = MQ.new
+  mq.prefetch(10)
+  MonitorAgent.new(mq)
+end

data/lib/alert_agent.rb

@@ -0,0 +1,126 @@
+require 'rubygems'
+gem 'amqp'
+require 'socket'
+require 'mq'
+require "#{File.dirname(__FILE__)}/packet"
+require "#{File.dirname(__FILE__)}/utils"
+gem 'ezmobius-redis'
+require 'redis'
+
+require 'rest_client'
+
+Thread.abort_on_exception = true
+
+class AlertAgent
+  QUEUE_NAME = 'failed-cloud-monitors'
+  attr_accessor :mq, :redis
+  FAIL = 'fail'.freeze
+  OK = 'ok'.freeze
+  
+  def initialize(mq)
+    @redis = Redis.new
+    @mq = mq
+    mq.queue(QUEUE_NAME).subscribe(:ack => true) do |headers, message|
+      if AMQP.closing?
+        Utils::LOG.info "(ignoring message, will be redelivered later)"
+      else
+        process_alerts(message, headers)
+      end
+    end
+  end
+    
+  def process_alerts(message, headers)
+   alert = JSON.parse(message)
+   alert.servers.each do |server|
+     process_server(server.merge('env_id' => alert.env_id, 'api' => alert.api))
+   end
+   headers.ack
+  end
+
+  # {"host" => host, "result" => result, 
+  #  "timestamp" => Time.now.to_i,
+  #  'token' => token}
+  def process_server(server)
+   errors = []
+   server['result'].each do |service, status|
+     errors << service if status == 'down'
+   end
+   state = get_state(server)
+   if errors.empty?
+     if state != OK
+       set_state(server, OK)
+       process_alert(server)
+     end
+   else
+     if state != FAIL
+       set_state(server, FAIL)
+       process_alert(server)
+     end
+   end
+  end
+
+  def set_state(server, state)
+   tries = 0
+   @redis[server['host']] = state
+  rescue RuntimeError
+   @redis = Redis.new
+   tries += 1
+   retry if tries < 10
+  end
+
+  def get_state(server)
+   tries = 0
+   @redis[server['host']]
+  rescue RuntimeError
+   @redis = Redis.new
+   tries += 1
+   retry if tries < 10
+  end
+
+  def format_alert(server)
+   result = {}
+   result['message'] = 'alert'
+   result['data'] = {}
+   result['data']['Time'] = server['timestamp'].to_s
+   result['data']['Severity'] = server['result'].any? {|_, status| status == 'down' } ? 'FAILURE' : 'OKAY'
+   result['data']['Type'] = result['data']['Plugin'] = "services"
+   result['data']['Host'] = server['host']
+   result['data']['alert_message'] = server['result'].sort.map do |service, status|
+     "#{service}: #{status}"
+   end.join(", ")
+   result
+  end
+
+  def report_to_irc(server, env)
+   s = ::TCPSocket.new('localhost', 5678)
+   s.write "[#{env}] #{server['host']} [#{server['role']}:#{server['env_id']}]\n#{server['result'].inspect}\n"
+   s.close
+  rescue => e
+   Utils::LOG.error e.message
+  end
+  
+  def report_to_awsm(server)
+    rest = RestClient::Resource.new(server['api'])
+    rest["/reporting/#{server['token']}"].post(format_alert(server), {"Accept" => "application/json"})
+    Utils::LOG.info "http report to awsm successful"
+  rescue RestClient::RequestFailed => e
+    Utils::LOG.error "http report failed!: #{msg}:#{err}"
+  end
+
+  def process_alert(server)
+   env = nil
+   if server['api'] == 'https://cloud.engineyard.com'
+     env = 'production'
+   else
+     env = 'staging'
+   end
+   
+   report_to_irc(server, env)
+   report_to_awsm(server)
+   Utils::LOG.info '*' * 50
+   Utils::LOG.info server['timestamp']
+   Utils::LOG.info server['host']
+   Utils::LOG.info server['result']
+   Utils::LOG.info server['token']
+  end
+end

data/lib/cron_feeder.rb

@@ -0,0 +1,68 @@
+require 'rubygems'
+require 'json'
+gem 'famoseagle-carrot'
+require 'carrot'
+
+class Packet
+  def initialize
+    raise NotImplementedError.new("#{self.class.name} is an abstract class.")
+  end
+  def to_json(*a)
+    {
+      'json_class'   => self.class.name,
+      'data'         => instance_variables.inject({}) {|m,ivar| m[ivar.sub(/@/,'')] = instance_variable_get(ivar); m }
+    }.to_json(*a)
+  end
+end
+
+class MonitorGroup < Packet
+  attr_accessor :api, :token, :servers
+  def initialize(env_id, api, servers)
+    @env_id = env_id
+    @api = api
+    @servers = servers
+  end
+
+  def self.json_create(o)
+    i = o['data']
+    new(i['env_id'],i['api'],i['servers'])
+  end
+end
+
+def next_envs(page, q)
+  envs = Environment.paginate(:page => page).each do |e|
+    arr = []
+    e.instances(:status => :running).each do |i|
+      host = i.public_hostname
+      next unless host
+      arr << {:host => host, :role => i.role, :token => i.token}
+    end
+    
+    next if arr.empty?
+    
+    mg = MonitorGroup.new e.id, HOSTNAME, arr
+    
+    puts "publishing:  #{mg}"
+    
+    q.publish(mg.to_json)
+  end
+end
+
+start = Time.now
+
+carrot = Carrot.new(:host => 'ec2-67-202-29-203.compute-1.amazonaws.com',
+                    :user => 'production', :pass => '3kslices!',
+                    :vhost => '/production')
+                  
+q = carrot.queue('cloud-monitors')
+
+    
+count = Environment.count
+
+pages = (count / Array.new.paginate.per_page) + 1
+
+1.upto(pages) do |page|
+  next_envs(page, q)
+end
+  
+puts "time:  #{Time.now - start}"

data/lib/ey_config.rb

@@ -0,0 +1,42 @@
+
+module EY
+  module Config
+    def setup_common_options(opts, options)
+      opts.on("-u", "--user USER", "Specify the rabbitmq username.") do |user|
+        options[:user] = user
+      end
+      
+      opts.on("-h", "--host HOST", "Specify the rabbitmq hostname.") do |host|
+        options[:host] = host
+      end
+      
+      opts.on("-P", "--port PORT", "Specify the rabbitmq PORT, default 5672.") do |port|
+        options[:port] = port
+      end
+      
+      opts.on("-p", "--pass PASSWORD", "Specify the rabbitmq password") do |pass|
+        options[:pass] = pass
+      end
+      
+      opts.on("-v", "--vhost VHOST", "Specify the rabbitmq vhost") do |vhost|
+        options[:vhost] = vhost
+      end
+      
+      opts.on("-d", "--daemonize", "Run #{type} as a daemon") do |d|
+        options[:daemonize] = true
+      end
+      
+      opts.on("--pid-dir PATH", "Specify the pid path, only used with daemonize") do |dir|
+        options[:pid_dir] = dir
+      end
+      
+      opts.on("-l", "--log-level LEVEL", "Specify the log level (fatal, error, warn, info, debug). Default is info") do |level|
+        options[:log_level] = level
+      end
+      
+      opts.on("--log-dir PATH", "Specify the log path") do |dir|
+        options[:log_dir] = dir
+      end
+    end
+  end
+end

data/lib/feeder.rb

@@ -0,0 +1,18 @@
+require 'rubygems'
+gem 'amqp'
+require 'mq'
+require 'json'
+require "#{File.dirname(__FILE__)}/packet"
+
+mg1 = MonitorGroup.new '42',
+                       'https://cloud.engineyard.com', [
+ {:host=>"75.101.150.230", :role=>:app_master, :token => 'deadbeef'}, 
+ {:host=>"ec2-75-101-183-93.compute-1.amazonaws.com", :role=>:app, :token => 'deadbeefdsfs'}, 
+ {:host=>"ec2-174-129-88-76.compute-1.amazonaws.com", :role=>:app, :token => 'deadbeeffdsf'},
+ {:host=>"ec2-174-129-83-10.compute-1.amazonaws.com", :role=>:db_master, :token => 'deadbeefweqwef'}
+]
+
+AMQP.start(:host => '174.129.11.230') do
+  MQ.queue('cloud-monitors1').publish(mg1.to_json)
+  AMQP.stop{ EM.stop } 
+end

data/lib/monitor_agent.rb

@@ -0,0 +1,137 @@
+require 'rubygems'
+gem 'amqp'
+require 'mq'
+require 'dataflow'
+require 'extlib'
+require 'net/ssh'
+require "#{File.dirname(__FILE__)}/packet"
+require "#{File.dirname(__FILE__)}/alert_agent"
+require "#{File.dirname(__FILE__)}/utils"
+Thread.abort_on_exception = true
+
+class MonitorAgent
+  include Dataflow
+  QUEUE_NAME = 'cloud-monitors'
+  attr_reader :mq
+  
+  def initialize(mq)
+    @mq = mq
+    mq.queue(QUEUE_NAME).subscribe(:ack => true) do |h,m|
+      if AMQP.closing?
+        Utils::LOG.info "(ignoring message, will be redelivered later)"
+      else
+        process_message(m,h)
+      end
+    end
+  end
+  
+  def process_message(msg, header)
+    msg = JSON.parse(msg)
+    if MonitorGroup === msg
+      Utils::LOG.info "got MonitorGroup: #{msg}"
+      process_monitor(msg, header)
+    end
+  end
+  
+  def process_monitor(monitor_group, header)
+    EM.defer(lambda {
+        monitor_group.servers.map do |srv|
+          check_server(srv)
+        end.map{|m| m.to_hash }
+      },
+      
+      lambda {|monitors|
+        alert = MonitorResult.new(monitor_group.env_id, monitor_group.api, monitors)
+        Utils::LOG.info alert.inspect
+        mq.queue(AlertAgent::QUEUE_NAME).publish(alert.to_json)
+        header.ack
+      })
+  end
+  
+  def check_server(srv)
+    need_later do
+      Utils::LOG.info "checking server: #{srv.inspect}"
+      MonitorFactory.build(srv['role'], srv['host'], srv['token']).check
+    end
+  end  
+end
+
+class MonitorFactory  
+  def self.build(role, host, token)
+    Object.const_get("#{role.camel_case}Monitor").new(host, token)
+  end
+end
+
+class AbstractMonitor
+  
+  attr_reader :host, :token, :result
+  
+  def initialize(host, token)
+    @host = host
+    @token = token
+    @result = {}
+  end
+  
+  def role
+    self.class.name.snake_case.chomp("_monitor")
+  end
+  
+  def check
+    @result = check_ssh
+    self
+  end
+  
+  def to_hash
+    {"host" => host, "result" => result, 
+     "timestamp" => Utils.epoch_time,
+     'token' => token, 'role' => role}
+  end
+  
+  def check_ssh
+    Utils::LOG.info "checking ssh: #{host}"
+    result = {}
+    begin
+      Net::SSH.start(host, "root", :keys => ["/root/.ssh/awsm"], :paranoid => false, :timeout => 10) do |ssh|
+        begin
+          # ignore any STDOUT and only grab the final json from the scripts output
+          script_output = ssh.exec!("bash -lc \"ey-agent\"").split("\n").last
+          result = JSON.parse(script_output)
+          result['ssh'] = 'up'
+        rescue JSON::ParserError
+          Utils::LOG.error '*' * 80
+          Utils::LOG.error "#{host}>>>>>>>>>>>\n#{script_output}"
+          result['ey-agent'] = 'down'
+        end
+      end
+    rescue Net::SSH::HostKeyMismatch => e
+      e.remember_host!
+      retry
+    end
+    result
+  rescue SocketError, Errno::ECONNREFUSED, Errno::EHOSTUNREACH, Errno::ETIMEDOUT, Timeout::Error, Net::SSH::AuthenticationFailed
+    result['ssh'] = 'down'
+    result
+  end
+end
+
+class AppMonitor < AbstractMonitor  
+end
+
+class AppMasterMonitor < AbstractMonitor
+  # def check
+  #   check_url
+  #   super
+  # end
+end
+
+class DbMasterMonitor < AbstractMonitor
+end
+
+class DbSlaveMonitor < AbstractMonitor  
+end
+
+class SoloMonitor < AbstractMonitor  
+end
+
+class UtilMonitor < AbstractMonitor  
+end

data/lib/packet.rb

@@ -0,0 +1,61 @@
+require 'json'
+
+# this is an abstract packet class that knows how to 
+# marshal itself into and out of JSON format
+class Packet
+  def initialize
+    raise NotImplementedError.new("#{self.class.name} is an abstract class.")
+  end
+  def to_json(*a)
+    {
+      'json_class'   => self.class.name,
+      'data'         => instance_variables.inject({}) {|m,ivar| m[ivar.sub(/@/,'')] = instance_variable_get(ivar); m }
+    }.to_json(*a)
+  end
+end
+
+class MonitorGroup < Packet
+  attr_accessor :api, :env_id, :servers
+  def initialize(env_id, api, servers)
+    @env_id = env_id
+    @api = api
+    @servers = servers
+  end
+
+  def self.json_create(o)
+    i = o['data']
+    new(i['env_id'],i['api'],i['servers'])
+  end
+end
+
+class MonitorResult < Packet
+  attr_accessor :api, :env_id, :servers
+  def initialize(env_id, api, servers)
+    @env_id = env_id
+    @api = api
+    @servers = servers
+  end
+
+  def self.json_create(o)
+    i = o['data']
+    new(i['env_id'],i['api'],i['servers'])
+  end
+end
+
+class MonitorGroupTimeStamp < Packet
+  attr_accessor :time
+  def initialize(time)
+    @time = time
+  end
+  def self.json_create(o)
+    i = o['data']
+    new(i['time'])
+  end
+end
+
+class Quit < Packet
+  attr_accessor :time
+  def initialize(time)
+    @time = time
+  end
+end

data/lib/utils.rb

@@ -0,0 +1,8 @@
+require 'logger'
+class Utils
+  LOG = Logger.new(STDOUT)
+  
+  def self.epoch_time
+    Time.now.to_i
+  end
+end

data/spec/integration_spec.rb

@@ -0,0 +1,185 @@
+require File.dirname(__FILE__) + '/spec_helper'
+
+class MockAppMasterMonitor < AbstractMonitor
+  def check
+    @result = {'haproxy' => 'up', 'nginx' => 'up', 'ssh' => 'up'}
+    self
+  end
+end
+
+class MockAppMonitor < AbstractMonitor
+  def check
+    @result = {'haproxy' => 'down', 'nginx' => 'down', 'ssh' => 'up'}
+    self
+  end
+end
+
+class MockDbMasterMonitor < AbstractMonitor
+  def check
+    @result = {'mysqld' => 'down', 'ssh' => 'up'}
+    self
+  end
+end
+
+class MockSoloMonitor < AbstractMonitor
+  def check
+    @result = {'mysqld' => 'down', 'nginx' => 'down', 'ssh' => 'up'}
+    self
+  end
+end
+
+describe "ey-big-brother" do
+  it "should monitor and process alerts on flex clusters and solos" do    
+    AMQP.start(:host => 'localhost') do
+      flex = MonitorGroup.new '42',
+                               'https://cloud.engineyard.com', [
+        {:host=>"75.101.150.230", :role=>:mock_app_master, :token => 'deadbeef'}, 
+        {:host=>"ec2-75-101-183-93.compute-1.amazonaws.com", :role=>:mock_app, :token => 'jhghjg'}, 
+        {:host=>"ec2-174-129-83-10.compute-1.amazonaws.com", :role=>:mock_db_master, :token => 'dfdsf'}
+      ]
+      solo = MonitorGroup.new '41',
+                               'https://cloud.engineyard.com', [
+        {:host=>"75.101.150.230", :role=>:mock_solo,:token => 'dsdfsdf'}, 
+      ]
+      @monitor_agent = MonitorAgent.new(MQ.new)
+      @alert_agent = AlertAgent.new(MQ.new)
+
+      @alert_agent.should_receive(:process_server).exactly(4).times
+
+      @monitor_agent.mq.queue(MonitorAgent::QUEUE_NAME).publish(flex.to_json)
+      @monitor_agent.mq.queue(MonitorAgent::QUEUE_NAME).publish(solo.to_json)
+
+      EM.add_timer(0.0001) { AMQP.stop{ EM.stop } }
+    end
+
+    @monitor_agent.mq.queue(MonitorAgent::QUEUE_NAME).should have(2).acked_messages
+    alerts = @alert_agent.mq.queue(AlertAgent::QUEUE_NAME).acked_messages
+    alerts.size.should == 2
+    alerts.map{|x| JSON.parse(x).servers }.flatten.each do |server|
+      case server['role']
+      when 'mock_solo'
+        server['result'].should == {'mysqld' => 'down', 'nginx' => 'down', 'ssh' => 'up'}
+      when 'mock_app_master'
+        server['result'].should ==  {'haproxy' => 'up', 'nginx' => 'up', 'ssh' => 'up'}
+      when 'mock_app'
+        server['result'].should == {'haproxy' => 'down', 'nginx' => 'down', 'ssh' => 'up'}
+      when 'mock_db_master'
+        server['result'].should ==  {'mysqld' => 'down', 'ssh' => 'up'}
+      end
+    end
+  end
+  
+  it "should POST alerts as collectd'formatted json objects to the awsm reporting api" do    
+    Utils.stub!(:epoch_time).and_return(1249595111)
+    AMQP.start(:host => 'localhost') do
+      flex = MonitorGroup.new '42',
+                               'https://cloud.engineyard.com', [
+        {:host=>"75.101.150.230", :role=>:mock_app_master, :token => 'deadbeef'}, 
+        {:host=>"ec2-174-129-83-10.compute-1.amazonaws.com", :role=>:mock_db_master, :token => 'dfdsf'}
+      ]
+      @monitor_agent = MonitorAgent.new(MQ.new)
+      @alert_agent = AlertAgent.new(MQ.new)
+  
+      @alert_agent.should_receive(:process_alert).exactly(2).times
+  
+      @monitor_agent.mq.queue(MonitorAgent::QUEUE_NAME).publish(flex.to_json)
+  
+      EM.add_timer(0.0001) { AMQP.stop{ EM.stop } }
+    end
+  
+    @monitor_agent.mq.queue(MonitorAgent::QUEUE_NAME).should have(1).acked_messages
+    alerts = @alert_agent.mq.queue(AlertAgent::QUEUE_NAME).acked_messages
+    alerts.size.should == 1
+    alerts.map{|x| JSON.parse(x).servers }.flatten.each do |server|
+      case server['role']
+      when 'mock_app_master'
+        @alert_agent.format_alert(server).should == {
+          "message" => "alert", 
+          "data" => {
+            "Time"=>"1249595111", 
+            "alert_message"=>"haproxy: up, nginx: up, ssh: up", 
+            "Severity"=>"OKAY",
+            "Type"=>"services", 
+            "Plugin"=>"services", 
+            "Host"=>"75.101.150.230"
+          }
+        }
+      when 'mock_db_master'
+        @alert_agent.format_alert(server).should == {
+          "message" => "alert", 
+          "data" => {
+            "Time"=>"1249595111", 
+            "alert_message"=>"mysqld: down, ssh: up", 
+            "Severity"=>"FAILURE",
+            "Type"=>"services", 
+            "Plugin"=>"services", 
+            "Host"=>"ec2-174-129-83-10.compute-1.amazonaws.com"
+          }
+        }
+      end
+    end
+  end
+
+  it "should send alert if state was nil" do    
+    AMQP.start(:host => 'localhost') do
+      solo = MonitorGroup.new '41',
+                               'https://cloud.engineyard.com', [
+        {:host=>"75.101.150.230", :role=>:mock_solo,:token => 'dsdfsdf'}, 
+      ]
+      @monitor_agent = MonitorAgent.new(MQ.new)
+      @alert_agent = AlertAgent.new(MQ.new)
+
+      @alert_agent.should_receive(:process_alert).exactly(1).times
+      @monitor_agent.mq.queue(MonitorAgent::QUEUE_NAME).publish(solo.to_json)
+
+      EM.add_timer(0.0001) { AMQP.stop{ EM.stop } }
+    end
+
+    @monitor_agent.mq.queue(MonitorAgent::QUEUE_NAME).should have(1).acked_messages
+    alerts = @alert_agent.mq.queue(AlertAgent::QUEUE_NAME).acked_messages
+    alerts.size.should == 1
+  end
+  
+  it "should send alert if state was changed" do    
+    AMQP.start(:host => 'localhost') do
+      solo = MonitorGroup.new '41',
+                               'https://cloud.engineyard.com', [
+        {:host=>"75.101.150.230", :role=>:mock_solo,:token => 'dsdfsdf'}, 
+      ]
+      @monitor_agent = MonitorAgent.new(MQ.new)
+      @alert_agent = AlertAgent.new(MQ.new)
+      
+      @alert_agent.redis = {"75.101.150.230" => 'ok'}
+      @alert_agent.should_receive(:process_alert).exactly(1).times
+      @monitor_agent.mq.queue(MonitorAgent::QUEUE_NAME).publish(solo.to_json)
+
+      EM.add_timer(0.0001) { AMQP.stop{ EM.stop } }
+    end
+
+    @monitor_agent.mq.queue(MonitorAgent::QUEUE_NAME).should have(1).acked_messages
+    alerts = @alert_agent.mq.queue(AlertAgent::QUEUE_NAME).acked_messages
+    alerts.size.should == 1
+  end
+  
+  it "should not send alert if state has not changed" do
+    AMQP.start(:host => 'localhost') do
+      solo = MonitorGroup.new '41',
+                               'https://cloud.engineyard.com', [
+        {:host=>"75.101.150.230", :role=>:mock_solo,:token => 'dsdfsdf'}, 
+      ]
+      @monitor_agent = MonitorAgent.new(MQ.new)
+      @alert_agent = AlertAgent.new(MQ.new)
+   
+      @alert_agent.redis = {"75.101.150.230" => 'fail'}
+      @alert_agent.should_not_receive(:process_alert)
+      @monitor_agent.mq.queue(MonitorAgent::QUEUE_NAME).publish(solo.to_json)
+
+
+      EM.add_timer(0.0001) { AMQP.stop{ EM.stop } }
+    end
+
+    @monitor_agent.mq.queue(MonitorAgent::QUEUE_NAME).should have(1).acked_messages
+    alerts = @alert_agent.mq.queue(AlertAgent::QUEUE_NAME).acked_messages
+    alerts.size.should == 1
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,15 @@
+$TESTING=true
+require File.join(File.dirname(__FILE__), '..', 'lib', 'monitor_agent')
+require File.join(File.dirname(__FILE__), '..', 'lib', 'alert_agent')
+gem "danielsdeleo-moqueue"
+require "moqueue"
+
+Spec::Runner.configure do |config|
+  overload_amqp
+  config.include Moqueue
+  Redis = Hash
+  config.before(:each) do
+    reset_broker
+  end
+end
+

metadata

@@ -0,0 +1,71 @@
+--- !ruby/object:Gem::Specification 
+name: ey-big-brother
+version: !ruby/object:Gem::Version 
+  version: 0.0.9
+platform: ruby
+authors: 
+- Ezra Zygmuntowicz
+autorequire: ey-big-brother
+bindir: bin
+cert_chain: []
+
+date: 2009-08-18 00:00:00 -07:00
+default_executable: 
+dependencies: []
+
+description: A gem that provides oversight of the ey-cloud
+email: ez@engineyard.com
+executables: 
+- ey-monitor-agent
+- ey-alert-agent
+- ey-drain-queues
+extensions: []
+
+extra_rdoc_files: 
+- README.rdoc
+- LICENSE
+- TODO
+files: 
+- LICENSE
+- README.rdoc
+- Rakefile
+- TODO
+- lib/alert_agent.rb
+- lib/cron_feeder.rb
+- lib/ey_config.rb
+- lib/feeder.rb
+- lib/monitor_agent.rb
+- lib/packet.rb
+- lib/utils.rb
+- spec/integration_spec.rb
+- spec/spec_helper.rb
+has_rdoc: true
+homepage: http://engineyard.com
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.5
+signing_key: 
+specification_version: 3
+summary: A gem that provides oversight of the ey-cloud
+test_files: []
+