city-watch 0.5.2 → 0.5.3

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    city-watch (0.4.0)
+    city-watch (0.5.2)
       rack
       rack-mount
       redis

data/config/city_watch.yml

@@ -1 +1,2 @@
-:collector: "localhost:62001"
+:debug: true
+:watch_collector: "localhost:62001"

data/lib/city_watch/collector.rb

@@ -6,4 +6,6 @@ end
 
 ::Routes = CityWatch::Routes
 
+require 'city_watch/util/collector'
+require 'city_watch/watchmen'
 require 'city_watch/collector/receive'

data/lib/city_watch/collector/receive.rb

@@ -1,25 +1,11 @@
-class ResendRequest
+class WatchCollector
 	
 	def call(env)
 		
 		post_data = begin Yajl::Parser.new(:symbolize_keys => true).parse(env["rack.input"].read) || {} rescue {} end
 		post_data[:received_at] = Time.now.to_s
 		
-		CityWatch.redis.sadd "#{CityWatch.config[:prefix]}::known_hosts", post_data[:hostname]
-		CityWatch.redis.zadd "#{CityWatch.config[:prefix]}::#{post_data[:hostname]}::raw_stats", Time.now.to_i, Yajl::Encoder.encode(post_data)
-		
-		summary = {}
-		
-		post_data[:watchmen].each do |watchman,dat|
-			CityWatch.redis.zadd "#{CityWatch.config[:prefix]}::#{post_data[:hostname]}::#{watchman}", Time.now.to_i, Yajl::Encoder.encode(dat.merge({:received_at => post_data[:received_at]}))
-			if dat[:summary]
-				sum = dat[:summary].is_a?(Array) ? dat[:summary].inject({}) {|acc,k| acc[k.to_sym] = dat[k.to_sym]; acc} : dat[:summary]
-				CityWatch.redis.zadd "#{CityWatch.config[:prefix]}::#{post_data[:hostname]}::#{watchman}::summary", Time.now.to_i, Yajl::Encoder.encode(sum)
-				summary[watchman] = sum
-			end
-		end
-		
-		CityWatch.redis.zadd "#{CityWatch.config[:prefix]}::#{post_data[:hostname]}::summary", Time.now.to_i, Yajl::Encoder.encode(summary.merge({:received_at => post_data[:received_at]}))
+		Collector.process(post_data)
 		
 		[200,{"Content-Type" => "text/plain"},["Got it!"]]
 	end

data/lib/city_watch/commander.rb

@@ -6,5 +6,6 @@ end
 
 ::Routes = CityWatch::Routes
 
+require 'city_watch/watchmen'
 require 'city_watch/commander/server'
 require 'city_watch/commander/home'

data/lib/city_watch/commander/server.rb

@@ -5,7 +5,25 @@ class Server
 		parms = Rack::Request.new(env).params.merge(env["rack.routing_args"]).inject({}){|acc,(k,v)| acc[k.to_sym] = v; acc}
 		server = parms[:server]
 		
-		output = CityWatch.header << '<h1>' << server << "</h1><ul>"
+		output = CityWatch.header << '<h1>' << server << "</h1>"
+		Watchmen.each do |watchman|
+			flags = watchman.get_flags(server)
+			alerts = watchman.get_alerts(server,2)
+			if (flags && flags.keys.count > 0) || (alerts && alerts.count > 0)
+				output << "<h3>" << watchman.name.to_s << "</h3><ul>"
+				if flags && flags.keys.count > 0
+					output << "<li class=\"alert\"><strong>Flags:</strong> <pre><code>" << Yajl::Encoder.encode(flags,:pretty => true, :indent => "   ") << "</code></pre></li>"
+				end
+				if alerts && alerts.count > 0
+					output << "<li class=\"alert\"><strong>Alerts:</strong></li>"
+					alerts.each do |alert|
+						output << "<li><pre><code>" << Yajl::Encoder.encode(Yajl::Parser.parse(alert),:pretty => true, :indent => "   ") << "</code></pre></li>"
+					end
+				end
+				output << "</ul>"
+			end
+		end
+		output << "<ol>"
 		CityWatch.redis.zrevrange("#{CityWatch.config[:prefix]}::#{server}::raw_stats",0,10).each do |update|
 			dat = Yajl::Parser.new(:symbolize_keys => true).parse(update)
 			output << "<li><h4>" << dat[:received_at] << "</h4><ul>"
@@ -14,7 +32,7 @@ class Server
 			end
 			output << "</ul></li>"
 		end
-		output << "</ul></body></html>"
+		output << "</ol></body></html>"
 		
 		[200,{"Content-Type" => "text/html"},[output]]
 	end

data/lib/city_watch/util/collector.rb

@@ -26,4 +26,21 @@ module Collector
 		end
 	end
 	
+	def self.process(data)
+		rcv_time = Time.now.to_i
+		host = data[:hostname]
+		CityWatch.redis.sadd "#{CityWatch.config[:prefix]}::known_hosts", host
+		CityWatch.redis.zadd "#{CityWatch.config[:prefix]}::#{host}::raw_stats", rcv_time, Yajl::Encoder.encode(data)
+		
+		summary = {}
+		
+		data[:watchmen].each do |watchman,dat|
+			if watch_obj = Watchmen.get(watchman)
+				status, summary[watchman] = watch_obj.process(dat,rcv_time,host)
+			end
+		end
+		
+		CityWatch.redis.zadd "#{CityWatch.config[:prefix]}::#{host}::summary", rcv_time, Yajl::Encoder.encode(summary.merge({:received_at => data[:received_at]}))
+	end
+	
 end

data/lib/city_watch/util/watchman.rb

@@ -6,6 +6,135 @@ module Watchman
 	
 	module ClassMethods
 		
+		def process(dat,rcv,host)
+			@host = host
+			@rcv_time = rcv
+			CityWatch.redis.zadd "#{CityWatch.config[:prefix]}::#{host}::#{self.name}", rcv_time, Yajl::Encoder.encode(dat.merge({:received_at => dat[:received_at]}))
+			if dat[:summary]
+				sum = dat[:summary].is_a?(Array) ? dat[:summary].inject({}) {|acc,k| acc[k.to_sym] = dat[k.to_sym]; acc} : dat[:summary]
+				CityWatch.redis.zadd "#{CityWatch.config[:prefix]}::#{host}::#{self.name}::summary", rcv_time, Yajl::Encoder.encode(sum)
+			end
+			run_rules(dat)
+			return 0, sum || nil
+		end
+		
+		def host
+			@host || nil
+		end
+		
+		def rcv_time
+			@rcv_time || Time.now.to_i
+		end
+		
+		def add_rule(name,&block)
+			@rules ||= {}
+			@rules[name] = block
+		end
+
+		def run_rules(dat)
+			@rules.map do |(name,rule)|
+				rule.call(dat)
+			end if @rules
+		end
+		
+		def send_alert(message,dat=nil)
+			CityWatch.redis.zadd "#{CityWatch.config[:prefix]}::#{host}::#{self.name}::alerts", rcv_time, Yajl::Encoder.encode({:message => message, :data => dat, :when => rcv_time})
+		end
+		
+		def alerts
+			@alerts ||= []
+			if block_given?
+				@alerts.each do |a|
+					yield a
+				end
+			else
+				@alerts
+			end
+			nil
+		end
+		
+		def get_alerts(host=host,num=5)
+			CityWatch.redis.zrevrange "#{CityWatch.config[:prefix]}::#{host}::#{self.name}::alerts", 0, num - 1
+		end
+		
+		def send_alerts!
+			@alerts.map do |alert|
+				puts "Alert: #{alert.inspect}" #if CityWatch.debug?
+			end if @alerts
+		end
+		
+		def set_flag(name)
+			unless get_flag(name)
+				flag_flapped name, :on
+			end
+			CityWatch.redis.setbit "#{CityWatch.config[:prefix]}::#{host}::#{self.name}::flags", flag_position(name), 1
+		end
+		
+		def clear_flag(name)
+			if get_flag(name)
+				flag_flapped name, :off
+			end
+			CityWatch.redis.setbit "#{CityWatch.config[:prefix]}::#{host}::#{self.name}::flags", flag_position(name), 0
+		end
+		
+		def flag_flapped(name,new_val)
+			# should have some event to watch for a flag switching position
+			puts "Flag flipped: #{name} -> #{new_val}" #if CityWatch.debug?
+		end
+		
+		def get_flag(name,host=host)
+			@host = host
+			CityWatch.redis.getbit("#{CityWatch.config[:prefix]}::#{host}::#{self.name}::flags", flag_position(name)) ? true : false
+		end
+		
+		def get_flags(host=host)
+			@host = host
+			out = {}
+			map = flag_map
+			map.each_index do |idx|
+				out[map[idx]] = get_flag(map[idx])
+			end
+			out
+		end
+		
+		def flag_map_key
+			"#{CityWatch.config[:prefix]}::#{self.name}::flag_map"
+		end
+		
+		def flag_map
+			CityWatch.redis.lrange flag_map_key, 0, -1
+		end
+		
+		def flag_position(name)
+			if (map = flag_map) && map.include?(name.to_s)
+				map.index(name.to_s)
+			else
+				new_flag(name)
+			end
+		end
+		
+		def new_flag(name)
+			CityWatch.redis.rpush(flag_map_key, name) - 1
+		end
+		
+		def set_default(k,val)
+			opts[k] = val
+		end
+		
+		def opts
+			@options ||= {}
+		end
+		
+		def options(*args)
+			if args.count > 1
+				return args.map {|k| opts[k]}
+			else
+				return opts[args.first]
+			end
+			return nil
+		end
+		alias_method :option, :options
+		
 	end
 	
 	def self.included(base)

data/lib/city_watch/watchmen.rb

@@ -20,14 +20,13 @@ module Watchmen
 		@watchmen << cls
 	end
 	
-	def self.add_rule(&block)
-		@rules ||= []
-		@rules << block
+	def self.get(name)
+		@watchmen.select {|w| w.name.to_s == name.to_s }.first
 	end
 	
-	def self.run_rules(data)
-		@rules.map do |rule|
-			rule.call(data)
+	def self.each
+		@watchmen.each do |w|
+			yield w
 		end
 	end
 	

data/lib/city_watch/watchmen/disk_usage.rb

@@ -2,10 +2,23 @@ class DiskUsage
 	
 	include Watchman
 	
+	set_default :usage_threshold, 60
+	
 	def self.data
 		dat = DF.data
 		sum = dat.select {|d| d[:mounted]=="/"}.first
 		{:partitions => dat, :summary => sum[:capacity] || sum["use%".to_sym]}
 	end
 	
+	add_rule(:root_usage_high) do |data|
+		
+		if (usage = data[:summary].to_i) && usage > option(:usage_threshold)
+			send_alert "Root disk usage is over #{option(:usage_threshold)}% (#{usage}%)", data[:partitions].select {|d| d[:mounted]=="/"}.first
+			set_flag :root_disk_over_quota
+		else
+			clear_flag :root_disk_over_quota
+		end
+		
+	end
+	
 end

data/lib/city_watch/watchmen/unicorns.rb

@@ -19,7 +19,7 @@ class Unicorns
 		out.merge({:num_masters => out[:masters].count, :num_workers => out[:workers].count, :summary => [:num_masters, :num_workers]})
 	end
 	
-	add_rule do |dat|
+	add_rule(:high_memory_usage) do |dat|
 		
 	end
 	

data/lib/version.rb

@@ -1,3 +1,3 @@
 module CityWatch
-  VERSION = "0.5.2"
+  VERSION = "0.5.3"
 end

metadata

@@ -2,7 +2,7 @@
 name: city-watch
 version: !ruby/object:Gem::Version 
   prerelease: 
-  version: 0.5.2
+  version: 0.5.3
 platform: ruby
 authors: 
 - John Bragg
@@ -10,7 +10,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2013-02-10 00:00:00 Z
+date: 2013-02-11 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: redis