deckard 0.5.10 → 0.5.12

data/README

@@ -16,6 +16,8 @@ Features:
 * Simple setup via cron
 * Basic scheduling to silence alerts
 * Adjustable delay before firing check content requests
+* Basic Chef "tag" lookup support
+* Alert stats database for trending and analysis
 
 Usage:
 
@@ -90,4 +92,35 @@ HTTP content check format:
 }
 
 
-For all of these priority and schedule are optional fields in these documents, priority is 0, 1 and 2. 0 is log only, 1 is log and email and 2 is log, email and sms. The schedule is an array containing integers for the hours the alert should be silent. Check out the replication check definition above.
+For all of these priority and schedule are optional fields in these documents, priority is 0, 1 and 2. 0 is log only, 1 is log and email and 2 is log, email and sms. The schedule is an array containing integers for the hours the alert should be silent. Check out the replication check definition above.
+
+
+For Chef "tag" support you need to install a view in your chef database and configure the url to it in your Deckard config file.
+
+function(doc) {
+    if (doc.chef_type != 'node') return;
+    
+    emit(doc.automatic.ec2 ? doc.automatic.ec2.public_hostname : doc.automatic.fqdn, doc.normal.tags[0]);
+}
+
+
+Regarding alert stats, just create a stats database and puts it's name in the config file. When alerts happen you should begin to see documents get added. By itself it isn't all that helpful, analysis is the key. Here is a basic "counts" design document to get you started, it will give you stats about the different kinds of alerts you are having.
+
+{
+   "_id": "_design/counts",
+   "language": "javascript",
+   "views": {
+       "by_type": {
+           "map": "function(doc) {\n  emit(doc.type, 1);\n}",
+           "reduce": "_count"
+       },
+       "by_url": {
+           "map": "function(doc) {\n  emit(doc.url, 1);\n}",
+           "reduce": "_count"
+       },
+       "by_error": {
+           "map": "function(doc) {\n  emit(doc.error, 1);\n}",
+           "reduce": "_count"
+       }
+   }
+}

data/config/deckard-test.yml

@@ -31,3 +31,6 @@ rep_check:
 notifo:
   user: "username"
   apikey: "password"
+
+stats:
+  db: "monitor_stats"

data/config/deckard.yml

@@ -31,3 +31,10 @@ rep_check:
 notifo:
   user: "username"
   apikey: "password"
+
+stats:
+  db: "monitor_stats"
+
+chef:
+  enabled: true
+  url:"http://localhost:5984/chef/_design/nodes/_view/hostnames"

data/lib/deckard.rb

@@ -22,7 +22,9 @@ require 'deckard/config'
 require 'deckard/log'
 require 'deckard/ec2'
 require 'deckard/monitoring'
+require 'deckard/stats'
 require 'deckard/util'
+require 'deckard/chef'
 
 
 class Deckard

data/lib/deckard/chef.rb

@@ -0,0 +1,31 @@
+class Deckard
+  class Chef
+    
+    def self.get_node_info(url)
+      
+      info = false
+      
+      if Deckard::Config.chef_enabled
+      
+        chef_url = Deckard::Config.chef_url
+      
+        begin
+          chef_nodes_json = RestClient.get(chef_url)
+          chef_nodes = JSON.parse(chef_nodes_json)
+
+          chef_nodes["rows"].each do |row|
+            if row["key"] == URI.parse(url).host
+              info = row["value"]
+            end
+          end
+
+        rescue Exception => e
+          Deckard::Log.error("Could not get chef tag for alert on #{url} due to #{e}")
+        end
+      end
+        
+      info
+    end
+    
+  end
+end

data/lib/deckard/config.rb

@@ -1,6 +1,13 @@
 class Deckard
   class Config
-    monitor_config = YAML.load(File.open(ARGV[1]))
+
+    if ARGV[0]
+      monitor_config = YAML.load(File.open(ARGV[1]))
+    else
+      puts "No config file specified"
+      abort
+    end
+
     extend Mixlib::Config
     configure do |c|
       c[:email_to] = monitor_config["defaults"]["email_to"]
@@ -31,6 +38,12 @@ class Deckard
       
       c[:notifo_user] = monitor_config["notifo"]["user"]
       c[:notifo_apikey] = monitor_config["notifo"]["apikey"]
+      
+      c[:stats_db] = monitor_config["stats"]["db"]
+      
+      c[:chef_url] = monitor_config["chef"]["url"]
+      c[:chef_enabled] = monitor_config["chef"]["enabled"]
+      
     end
   end
 end

data/lib/deckard/monitoring.rb

@@ -11,10 +11,20 @@ class Deckard
         sleep(Deckard::Config.content_check_retry_interval)
         retry if (retries += 1) < retry_count
         if retries >= retry_count
-          subject = "ALERT :: Check Content Failed on #{url}"
-          body = "#{Time.now} :: #{e} :: #{url}"
+          
+          node_info = Deckard::Chef.get_node_info(url)
+          
+          if node_info
+            subject = "ALERT :: #{node_info} :: #{e}"
+            body = "#{Time.now} :: #{url}"
+          else
+            subject = "ALERT :: #{url} :: #{e}"
+            body = "#{Time.now} :: #{url} :: #{e}"
+          end
+          
           log = subject + " -- " + body
           Deckard::Util.alert(priority, subject, body, log, schedule, url)
+          Deckard::Stats.alert(priority, e, url, "contentcheck")
           check = false
         end
       else
@@ -22,10 +32,20 @@ class Deckard
         if result.include?(content)
           Deckard::Log.info("PASS :: Found text \"#{content}\" on #{url}")
         else
-          subject = "ALERT :: Check Content Failed on #{url}"
-          body = "#{Time.now} :: Could not find text \"#{content}\" at #{url}"
+          
+          node_info = Deckard::Chef.get_node_info(url)
+          
+          if node_info
+            subject = "ALERT :: #{node_info} :: content missing"
+            body = "#{Time.now} :: Could not find text \"#{content}\" at #{url}"
+          else
+            subject = "ALERT :: #{url} :: content missing"
+            body = "#{Time.now} :: Could not find text \"#{content}\" at #{url}"
+          end
+          
           log = subject + " -- " + body
           Deckard::Util.alert(priority, subject, body, log, schedule, url)
+          Deckard::Stats.alert(priority, "nocontent", url, "contentcheck")
           check = false
         end
       end
@@ -49,6 +69,7 @@ class Deckard
           body = "Master: #{master_url} => Slave: #{slave_url} : off by #{doc_count_diff}"
           log = subject + " -- " + body
           Deckard::Util.alert(priority, subject, body, log, schedule, master_url)
+          Deckard::Stats.alert(priority, "#{doc_count_diff}", url, "replication")
         else
           Deckard::Log.info("PASS :: Replication for #{name} is OK (#{doc_count_diff})")
         end
@@ -64,7 +85,8 @@ class Deckard
           body = "#{region} : #{elastic_ip} => #{primary_instance_id} / #{secondary_instance_id} attempting failover!"
           log = subject + " " + body
           Deckard::Util.alert(priority, subject, body, log, schedule, "http://#{elastic_ip}")
-
+          Deckard::Stats.alert(priority, "unknown", "http://#{elastic_ip}", "failover")
+          
           instance_id = Deckard::Ec2.get_association(region, elastic_ip)
 
           if Deckard::Ec2.disassociate_address(region, elastic_ip)

data/lib/deckard/stats.rb

@@ -0,0 +1,24 @@
+class Deckard
+  class Stats
+    
+    def self.alert(priority, error, url, type)
+      db_name = Deckard::Config.stats_db
+      db_user = Deckard::Config.db_user
+      db_password = Deckard::Config.db_password
+      db_host = Deckard::Config.db_host
+      db_port = Deckard::Config.db_port
+      
+      db_url = "http://#{db_user}:#{db_password}@#{db_host}:#{db_port}/#{db_name}"
+      
+      alert_info = {"priority" => priority, "error" => error, "url" => url, "timestamp" => Time.now.utc.iso8601, "type" => type}
+      
+      begin
+        RestClient.post("#{db_url}", alert_info.to_json, "Content-Type" => "application/json")
+        Deckard::Log.info("Added stats for alert on #{url}")
+      rescue Exception => e
+        Deckard::Log.error("Could not add stats for alert on #{url} due to #{e}")
+      end
+    end
+    
+  end
+end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: deckard
 version: !ruby/object:Gem::Version 
-  hash: 31
+  hash: 19
   prerelease: false
   segments: 
   - 0
   - 5
-  - 10
-  version: 0.5.10
+  - 12
+  version: 0.5.12
 platform: ruby
 authors: 
 - joe williams
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-01-13 00:00:00 -08:00
+date: 2011-04-08 00:00:00 -07:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -128,10 +128,12 @@ extra_rdoc_files:
 - README
 files: 
 - bin/deckard
+- lib/deckard/chef.rb
 - lib/deckard/config.rb
 - lib/deckard/ec2.rb
 - lib/deckard/log.rb
 - lib/deckard/monitoring.rb
+- lib/deckard/stats.rb
 - lib/deckard/util.rb
 - lib/deckard.rb
 - config/deckard-test.yml