deckard 0.4

data/README

@@ -0,0 +1,85 @@
+deckard : http monitoring system
+
+deckard is a http check monitoring system built on top of CouchDB.
+
+license: apache 2
+
+Features:
+
+* Email and SMS based alerts (through email)
+* Designated on-call sms email address
+* Basic CouchDB replication latency alerts
+* Basic content check alerts
+* Content check alerts with EC2 elastic IP failover
+* All checks are defined in CouchDB (CRUD checks with ReST)
+* Alert priorities (log, email, SMS)
+* Simple setup via cron
+* Basic scheduling to silence alerts
+
+Usage:
+
+$ deckard ./deckard.yml
+
+Setup:
+
+* Setup and configure all appropriate databases and alert documents.
+* Create a crontab entry
+
+$ crontab -e
+*/5 * * * * deckard /path/deckard.yml &> /dev/null
+
+
+Example documents:
+
+On Call document format:
+
+{
+   "_id": "on_call_person",
+   "sms_email": "8675309@jenny.net"
+}
+
+For sms_email you will need to put in the phone number and sms to email host for your phone provider.
+
+
+Failover check document format:
+
+{
+   "_id": "lb01",
+   "url": "http://somecheck.com/check.html",
+   "secondary_instance_id": "i-1234",
+   "priority": 2,
+   "elastic_ip": "127.0.0.1",
+   "content": "sometext",
+   "failover": true,
+   "primary_instance_id": "i-4321"
+}
+
+This document needs all the details to cause an elastic ip switch in the case the content is not found on the url.
+
+
+Replication check format:
+
+{
+   "_id": "node01_node02",
+	 "name": "test",
+	 "master_url": "http://node01:5984/db",
+	 "slave_url": "http://node02:5984/db"
+	 "offset": 0, 
+	 "priority": 2,
+	 "schedule": [])
+}
+
+This will test the doc counts between two databases and if they become out of sync by more or less than the thresholds specified in the config an alert is triggered.
+
+
+HTTP content check format:
+
+{
+   "_id": "deckard.com:5984/",
+   "url": "http://deckard.com:5984/",
+   "content": "couchdb",
+   "priority": 2
+}
+
+
+For all of these priority and schedule are optional fields in these documents, priority is 0, 1 and 2. 0 is log only, 1 is log and email and 2 is log, email and sms. The schedule is an array containing integers for the hours the alert should be silent.

data/bin/deckard

@@ -0,0 +1,5 @@
+#!/usr/bin/ruby
+require File.join(File.dirname(__FILE__), '..', 'lib', 'deckard')
+
+Deckard.main
+

data/config/deckard.yml

@@ -0,0 +1,26 @@
+defaults:
+  email_to: "my@email.com"
+  email_from: "some@email.com"
+  email_host: "localhost"
+  db_user:
+  db_password:
+  db_host: "localhost"
+  db_port: "5984"
+  on_call_db: "monitor_on_call"
+  on_call_doc: "on_call_person"
+  log_file:
+
+content_check:
+  retry_count: 1
+  db: "monitor_content_check"
+
+fo_check:
+  retry_count: 1
+  aws_key: "key"
+  aws_secret: "seekrat"
+  db: "monitor_fo_check"
+
+rep_check:
+  doc_behind_threshold: 5000
+  doc_ahead_threshold: -100
+  db: "monitor_rep_check"

data/lib/deckard.rb

@@ -0,0 +1,100 @@
+require 'rubygems'
+
+gem 'rest-client', '1.3.0'
+require 'rest_client'
+require 'tmail'
+require 'json'
+require 'cgi'
+require 'net/smtp'
+require 'right_aws'
+require 'mixlib/log'
+require 'mixlib/config'
+require 'yaml'
+
+__DIR__ = File.dirname(__FILE__)
+
+$LOAD_PATH.unshift __DIR__ unless
+  $LOAD_PATH.include?(__DIR__) ||
+  $LOAD_PATH.include?(File.expand_path(__DIR__))
+  
+require 'deckard/config'
+require 'deckard/log'
+require 'deckard/ec2'
+require 'deckard/monitoring'
+require 'deckard/util'
+
+
+class Deckard
+  def self.content_check
+    retry_count = Deckard::Config.content_check_retry
+    db_name = Deckard::Config.content_check_db
+    list = Array.new
+
+    nodes = Deckard::Util.get_nodes(db_name)
+
+    nodes.each do |node|
+      run = Thread.new {
+        Deckard::Monitor.content_check(node["url"], node["content"], node["priority"], retry_count, node["schedule"])
+        }
+      list << run
+    end
+
+    list.each { |x|
+     	  x.join
+    }
+  end
+
+  def self.rep_check
+    db_name = Deckard::Config.rep_check_db
+    list = Array.new
+
+    nodes = Deckard::Util.get_nodes(db_name)
+
+    nodes.each do |node|
+      run = Thread.new {
+        Deckard::Monitor.rep_check(node["name"], node["master_url"], node["slave_url"], node["offset"], node["priority"], node["schedule"])
+        }
+      list << run
+    end
+
+    list.each { |x|
+     	  x.join
+    }
+  end
+
+  def self.fo_check
+    retry_count = Deckard::Config.fo_check_retry
+    db_name = Deckard::Config.fo_check_db
+    list = Array.new
+
+    nodes = Deckard::Util.get_nodes(db_name)
+
+    nodes.each do |node|
+      run = Thread.new {
+        check = Deckard::Monitor.content_check(node["url"], node["content"], node["priority"], retry_count, node["schedule"])
+        unless check
+          Deckard::Monitor.failover(node["elastic_ip"], node["primary_instance_id"], node["secondary_instance_id"], node["priority"], node["schedule"], node["failover"])
+          Deckard::Util.flip_failover(node)
+        end
+        }
+      list << run
+    end
+
+    list.each { |x|
+      x.join
+    }
+  end
+
+  def self.main
+    list = Array.new
+
+    list << Thread.new { content_check }
+    list << Thread.new { rep_check }
+    list << Thread.new { fo_check }
+
+    list.each { |x|
+      x.join
+    }
+  end
+
+end

data/lib/deckard/config.rb

@@ -0,0 +1,30 @@
+class Deckard
+  class Config
+    monitor_config = YAML.load(File.open(ARGV[0]))
+    extend Mixlib::Config
+    configure do |c|
+      c[:email_to] = monitor_config["defaults"]["email_to"]
+      c[:email_from] = monitor_config["defaults"]["email_from"]
+      c[:email_host] = monitor_config["defaults"]["email_host"]
+      c[:db_user] = monitor_config["defaults"]["db_user"]
+      c[:db_password] = monitor_config["defaults"]["db_password"]
+      c[:db_host] = monitor_config["defaults"]["db_host"]
+      c[:db_port] = monitor_config["defaults"]["db_port"]
+      c[:on_call_db] = monitor_config["defaults"]["on_call_db"]
+      c[:on_call_doc] = monitor_config["defaults"]["on_call_doc"]
+      c[:log_file] = monitor_config["defaults"]["log_file"]
+
+      c[:content_check_retry] = monitor_config["content_check"]["retry_count"]
+      c[:content_check_db] = monitor_config["content_check"]["db"]
+
+      c[:fo_check_retry] = monitor_config["fo_check"]["retry_count"]
+      c[:aws_key] = monitor_config["fo_check"]["aws_key"]
+      c[:aws_secret] = monitor_config["fo_check"]["aws_secret"]
+      c[:fo_check_db] = monitor_config["fo_check"]["db"]
+
+      c[:doc_behind_threshold] = monitor_config["rep_check"]["doc_behind_threshold"]
+      c[:doc_ahead_threshold] = monitor_config["rep_check"]["doc_ahead_threshold"]
+      c[:rep_check_db] = monitor_config["rep_check"]["db"]
+    end
+  end
+end

data/lib/deckard/ec2.rb

@@ -0,0 +1,19 @@
+class Deckard
+  class Ec2
+    aws_key = Deckard::Config.aws_key
+    aws_secret = Deckard::Config.aws_secret
+    Ec2 = RightAws::Ec2.new(aws_key, aws_secret)
+
+    def self.get_association(elastic_ip)
+      Ec2.describe_addresses(elastic_ip)[0][:instance_id]
+    end
+
+    def self.associate_address(instance_id, elastic_ip)
+      Ec2.associate_address(instance_id, elastic_ip)
+    end
+
+    def self.disassociate_address(elastic_ip)
+      Ec2.disassociate_address(elastic_ip)
+    end
+  end
+end

data/lib/deckard/log.rb

@@ -0,0 +1,10 @@
+class Deckard
+  class Log
+    extend Mixlib::Log
+    log_file = Deckard::Config.log_file
+
+    if log_file
+      Deckard::Log.init(log_file)
+    end
+  end
+end

data/lib/deckard/monitoring.rb

@@ -0,0 +1,103 @@
+class Deckard
+  class Monitor
+
+    def self.content_check(url, content, priority, retry_count, schedule)
+      check = true
+      begin
+        retries = 1 unless retries
+        result = RestClient.get(url)
+      rescue
+        Deckard::Log.info("ALERT :: Could not connect to #{url}, retrying ...")
+        sleep(3)
+        retry if (retries += 1) < retry_count
+        if retries >= retry_count
+          subject = "ALERT :: Check Content Failed on #{url}"
+          body = "Could not connect to #{url}"
+          log = subject + " -- " + body
+          Deckard::Util.alert(priority, subject, body, log, schedule)
+          check = false
+        end
+      else
+        retries = 1
+        if result.include? content
+          Deckard::Log.info("PASS :: Found text \"#{content}\" on #{url}")
+        else
+          subject = "ALERT :: Check Content Failed on #{url}"
+          body = "Could not find text \"#{content}\" at #{url}"
+          log = subject + " -- " + body
+          Deckard::Util.alert(priority, subject, body, log, schedule)
+          check = false
+        end
+      end
+      check
+    end
+
+    def self.rep_check(name, master_url, slave_url, offset, priority, schedule)
+      begin
+        doc_behind_threshold = Deckard::Config.doc_behind_threshold
+        doc_ahead_threshold = Deckard::Config.doc_ahead_threshold
+
+        master_result = RestClient.get(master_url)
+        slave_result = RestClient.get(slave_url)
+
+        master_doc_count = JSON.parse(master_result)["doc_count"]
+        slave_doc_count = JSON.parse(slave_result)["doc_count"]
+        doc_count_diff = master_doc_count - slave_doc_count + offset
+
+        if doc_count_diff >= doc_behind_threshold || doc_count_diff < doc_ahead_threshold
+          subject = "ALERT :: Replication for #{name}"
+          body = "Master: #{master_url} => Slave: #{slave_url} : off by #{doc_count_diff}"
+          log = subject + " -- " + body
+          Deckard::Util.alert(priority, subject, body, log, schedule)
+        else
+          Deckard::Log.info("PASS :: Replication for #{name} is OK (#{doc_count_diff})")
+        end
+      rescue
+        # do nothing
+      end
+    end
+
+    def self.failover(elastic_ip, primary_instance_id, secondary_instance_id, priority, schedule, failover)
+      if failover
+        begin
+          subject = "ALERT :: #{elastic_ip} attempting failover!"
+          body = "#{elastic_ip} => #{primary_instance_id} / #{secondary_instance_id} attempting failover!"
+          log = subject + " " + body
+          Deckard::Util.alert(priority, subject, body, log, schedule)
+
+          instance_id = Deckard::Ec2.get_association(elastic_ip)
+          Deckard::Ec2.disassociate_address(elastic_ip)
+          Deckard::Log.info("ALERT :: Disassociated #{elastic_ip}")
+
+          if instance_id == primary_instance_id
+            Deckard::Ec2.associate_address(secondary_instance_id, elastic_ip)
+
+            Deckard::Log.info("ALERT :: associated #{elastic_ip} to #{secondary_instance_id}")
+            subject = "ALERT :: Failover Complete for #{elastic_ip} #{secondary_instance_id}"
+            body = "VERIFY THINGS ARE WORKING! #{elastic_ip} => #{primary_instance_id} / #{secondary_instance_id}"
+            Deckard::Util.alert(priority, subject, body, subject, schedule)
+          elsif instance_id == secondary_instance_id
+            Deckard::Ec2.associate_address(primary_instance_id, elastic_ip)
+
+            Deckard::Log.info("ALERT :: associated #{elastic_ip} to #{primary_instance_id}")
+            subject = "ALERT :: Failover Complete for #{elastic_ip} #{primary_instance_id}"
+            body = "VERIFY THINGS ARE WORKING! #{elastic_ip} => #{primary_instance_id} / #{secondary_instance_id}"
+            Deckard::Util.alert(priority, subject, body, subject, schedule)
+          else
+            error = "ALERT :: Could not a failover #{elastic_ip} => #{primary_instance_id} / #{secondary_instance_id}!!"
+            log = "ALERT :: Could not a failover #{elastic_ip} => #{primary_instance_id} / #{secondary_instance_id}!! Due to instance_id != primary and secondary"
+            Deckard::Util.alert(priority, error, error, log, schedule)
+          end
+        rescue Exception => e
+          error = "ALERT :: Could not a failover #{elastic_ip} => #{primary_instance_id} / #{secondary_instance_id}!!"
+          log = "ALERT :: Could not a failover #{elastic_ip} => #{primary_instance_id} / #{secondary_instance_id}!! Due to #{e}"
+          Deckard::Util.alert(priority, error, error, log, schedule)
+        end
+      else
+        # dont failover
+        Deckard::Log.info("ALERT :: not failing over due to failover=false")
+      end
+    end
+
+  end
+end

data/lib/deckard/util.rb

@@ -0,0 +1,123 @@
+class Deckard
+  class Util
+    def self.get_nodes(db_name)
+      doc_list = []
+      node_list = []
+
+      db_user = Deckard::Config.db_user
+      db_password = Deckard::Config.db_password
+      db_host = Deckard::Config.db_host
+      db_port = Deckard::Config.db_port
+
+      if db_user && db_password
+        db_url = "http://#{db_user}:#{db_password}@#{db_host}:#{db_port}/#{db_name}"
+      else
+        db_url = "http://#{db_host}:#{db_port}/#{db_name}"
+      end
+
+      all_docs = RestClient.get("#{db_url}/_all_docs")
+
+      all_docs_hash = JSON.parse(all_docs)
+
+      all_docs_hash["rows"].each do |doc|
+        doc_list << doc["id"]
+      end
+
+      doc_list.each do |doc|
+        escaped_doc = CGI.escape(doc)
+        node_json = RestClient.get("#{db_url}/#{escaped_doc}")
+        node = JSON.parse(node_json)
+        node_list << node
+      end
+      node_list
+    end
+
+    def self.alert(priority, subject, body, log, schedule)
+      email_to = Deckard::Config.email_to
+      sms_email = on_call
+
+      # if scheduled maintenance set to logging only
+      if schedule(schedule) == true
+        priority = 0
+      end
+
+      if priority == 0
+        Deckard::Log.info(log)
+      elsif priority == 1
+        Deckard::Log.info("sending email alert to #{email_to}")
+        send_email(email_to, subject, body)
+        Deckard::Log.info(log)
+      elsif priority == 2
+        Deckard::Log.info("sending email alert to #{email_to} and sms to #{sms_email}")
+        send_email(email_to, subject, body)
+        Deckard::Log.info(log)
+        send_email("#{sms_email}", subject, body)
+      end
+    end
+
+    def self.on_call
+      db_user = Deckard::Config.db_user
+      db_password = Deckard::Config.db_password
+      db_host = Deckard::Config.db_host
+      db_port = Deckard::Config.db_port
+      db_name = Deckard::Config.on_call_db
+      doc_name = Deckard::Config.on_call_doc
+
+      doc_url = "http://#{db_user}:#{db_password}@#{db_host}:#{db_port}/#{db_name}/#{doc_name}"
+
+      on_call = RestClient.get doc_url
+      on_call_json = JSON.parse(on_call)
+      on_call_json["sms_email"]
+    end
+
+    def self.send_email(email_addr, subject, body)
+      to = email_addr
+      from = Deckard::Config.email_from
+      host = Deckard::Config.email_host
+      mail = TMail::Mail.new
+      mail.to = to
+      mail.from = from
+      mail.subject = subject
+      mail.date = Time.now
+      mail.mime_version = '1.0'
+      mail.body = body
+
+      Net::SMTP.start( host ) do |smtp|
+        smtp.send_message(
+          mail.to_s,
+          from,
+          to
+        )
+      end
+    end
+
+    def self.schedule(schedule)
+      if schedule.nil?
+        false
+      else
+        schedule.include? Time.now.hour
+      end
+    end
+
+    def self.flip_failover(node)
+      db_user = Deckard::Config.db_user
+      db_password = Deckard::Config.db_password
+      db_host = Deckard::Config.db_host
+      db_port = Deckard::Config.db_port
+      db_name = Deckard::Config.fo_check_db
+
+      if db_user == "" || db_password == ""
+        db_url = "http://#{db_host}:#{db_port}/#{db_name}"
+      else
+        db_url = "http://#{db_user}:#{db_password}@#{db_host}:#{db_port}/#{db_name}"
+      end
+
+      doc_json = RestClient.get("#{db_url}/#{node["_id"]}")
+      doc = JSON.parse(doc_json)
+      doc.store("failover", false)
+
+      RestClient.put("#{db_url}/#{node["_id"]}?rev=#{node["_rev"]}", doc.to_json)
+    end
+
+  end
+end

metadata

@@ -0,0 +1,142 @@
+--- !ruby/object:Gem::Specification 
+name: deckard
+version: !ruby/object:Gem::Version 
+  prerelease: false
+  segments: 
+  - 0
+  - 4
+  version: "0.4"
+platform: ruby
+authors: 
+- joe williams
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-06-04 00:00:00 -07:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: mixlib-config
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: mixlib-log
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: tmail
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  name: json
+  prerelease: false
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id004
+- !ruby/object:Gem::Dependency 
+  name: right_aws
+  prerelease: false
+  requirement: &id005 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id005
+- !ruby/object:Gem::Dependency 
+  name: rest-client
+  prerelease: false
+  requirement: &id006 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - "="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 1
+        - 3
+        - 0
+        version: 1.3.0
+  type: :runtime
+  version_requirements: *id006
+description: 
+email: joe@joetify.com
+executables: 
+- deckard
+extensions: []
+
+extra_rdoc_files: 
+- README
+files: 
+- bin/deckard
+- lib/deckard/config.rb
+- lib/deckard/ec2.rb
+- lib/deckard/log.rb
+- lib/deckard/monitoring.rb
+- lib/deckard/util.rb
+- lib/deckard.rb
+- config/deckard.yml
+- README
+has_rdoc: true
+homepage: http://github.com/joewilliams/deckard
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.6
+signing_key: 
+specification_version: 3
+summary: a monitoring system built on couchdb
+test_files: []
+