skyeye 0.0.1

data/.gitignore

@@ -0,0 +1,2 @@
+Gemfile.lock
+pkg

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+gem "aws-sdk"
+gem "open4"

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Greg Fodor
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,5 @@
+SkyEye
+======
+
+SkyEye is a gem that allows the creation and management of nagios-like alerts
+using CloudWatch.

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/bin/skyeye

@@ -0,0 +1,19 @@
+#!/usr/bin/env ruby
+
+$: << File.dirname(__FILE__) + "/../lib"
+
+require "aws-sdk"
+require "skyeye"
+
+@exec = SkyEye::Exec.new
+
+trap("INT") do
+  @exec.shutdown!
+end
+
+trap("TERM") do
+  @exec.shutdown!
+end
+
+@exec.go(*ARGV)
+

data/config/skyeye.yml.example

@@ -0,0 +1,48 @@
+:namespace: posto-dev
+
+:topics: 
+  - posto-sauron-alerts
+
+:alarm_periods:
+  :warning: 5
+  :critical: 1
+
+:watches:
+  :instance:
+    - :name: posto-mysql
+      :command: /usr/local/sbin/check_tcp -H localhost -p 3306
+      :interval: 5
+    - :name: CPUUtilization
+      :comparison_operator: GreaterThanThreshold
+      :warning: 50
+      :critical: 80
+  :elb:
+    - :name: HealthyHostCount
+      :comparison_operator: LessThanThreshold
+      :warning: 2
+      :critical: 1
+      :statistic: Minimum
+      :period: 1
+  :rds:
+    - :name: CPUUtilization
+      :comparison_operator: GreaterThanThreshold
+      :warning: 50
+      :critical: 80
+      :statistic: Average
+      :period: 2
+    - :name: DatabaseConnections
+      :comparison_operator: GreaterThanThreshold
+      :critical: 128
+      :statistic: Maximum
+      :period: 1
+  :elasticache:
+    - :name: CmdGet
+      :comparison_operator: GreaterThanThreshold
+      :critical: 6000
+      :period: 1
+    - :name: SwapUsage
+      :comparison_operator: GreaterThanThreshold
+      :critical: 0
+      :statistic: Minimum
+      :period: 5
+

data/lib/skyeye/exec.rb

@@ -0,0 +1,300 @@
+require "logger"
+require "pp"
+require "json"
+require "open4"
+
+module SkyEye
+  class Exec
+    attr_accessor :config
+
+    def instance_id
+      "i-d3b842a1"
+    end
+
+    def go(*args)
+      config_file = ENV["SKYEYE_CONFIG_FILE"] || "/etc/skyeye.yml"
+
+      unless File.exists?(config_file)
+        puts "Missing #{config_file}. You can set the path to this file by exporting SKYEYE_CONFIG_FILE."
+        return
+      end
+
+      @config = YAML.load(config_file)
+      @logger = Logger.new(STDOUT)
+      @mutex = Mutex.new
+
+      if args.size == 0
+        start!
+      elsif args[0] == "register:instance"
+        load_or_register_topics!
+        register_instance_alarms!
+      elsif args[0] == "deregister:instance"
+        deregister_instance_alarms!
+      elsif args[0] == "deregister:aws"
+        deregister_aws_alarms!
+      elsif args[0] == "register:aws"
+        load_or_register_topics!
+        register_aws_alarms!
+      else
+        puts "unknown command #{args[0]}"
+      end
+    end
+
+    def deregister_instance_alarms!
+      deregister_alarms_matching!(/^skyeye::instance::#{instance_id}::/)
+    end
+
+    def deregister_aws_alarms!
+      @config[:watches].keys.each do |resource_type|
+        unless resource_type == :instance
+          deregister_alarms_matching!(/^skyeye::#{resource_type}::/)
+        end
+      end
+    end
+
+    def deregister_alarms_matching!(pattern)
+      cw = AWS::CloudWatch.new
+      alarms = cw.client.describe_alarms
+
+      to_delete = []
+
+      alarms.data[:metric_alarms].each do |alarm|
+        if alarm[:alarm_name] =~ pattern
+          to_delete << alarm[:alarm_name]
+        end
+      end
+
+      if to_delete.size > 0
+        @logger.info "De-Register Alarms #{to_delete.inspect}"
+        cw.alarms.delete(*to_delete)
+      end
+    end
+
+    def register_aws_alarms!
+      @config[:watches].each do |resource_type, watches|
+        dimension_values = dimension_values_for_resource_type(resource_type)
+        register_alarms_for_watches!(resource_type, dimension_values) if dimension_values
+      end
+    end
+
+    def dimension_values_for_resource_type(resource_type)
+      values = []
+
+      case resource_type
+      when :elb
+        AWS::ELB.new.load_balancers.each do |load_balancer|
+          values << [{ name: "LoadBalancerName", value: load_balancer.name }]
+        end
+      when :rds
+        AWS::RDS.new.db_instances.each do |db_instance|
+          values << [{ name: "DBInstanceIdentifier", value: db_instance.db_instance_identifier }]
+        end
+      when :elasticache
+        AWS::ElastiCache.new.client.describe_cache_clusters(show_cache_node_info: true).data[:cache_clusters].each do |cluster|
+          cluster[:cache_nodes].each do |node|
+            values << [{ name: "CacheClusterId", value: cluster[:cache_cluster_id] },
+                       { name: "CacheNodeId", value: node[:cache_node_id] }]
+          end
+        end
+      when :sqs
+        AWS::SQS.new.queues.each do |queue|
+          values << [{ name: "QueueName", value: queue.arn.split(/:/)[-1] }]
+        end
+      end
+
+      values
+    end
+    
+    def namespace_for_resource_type(resource_type)
+      case resource_type
+      when :instance
+        return "AWS/EC2"
+      when :elb
+        return "AWS/ELB"
+      when :rds
+        return "AWS/RDS"
+      when :elasticache
+        return "AWS/ElastiCache"
+      when :sqs
+        return "AWS/SQS"
+      end
+    end
+
+    def register_instance_alarms!
+      cw = AWS::CloudWatch.new
+
+      watches = @config[:watches][:instance]
+      dimension_values = [[{ name: "InstanceId", value: instance_id }]]
+
+      register_alarms_for_watches!(:instance, dimension_values)
+    end
+
+    def register_alarms_for_watches!(resource_type, dimension_values)
+      cw = AWS::CloudWatch.new
+
+      dimension_values.each do |dimension_value|
+        @config[:watches][resource_type].each do |watch|
+          [:warning, :critical].each do |threshold|
+            target = dimension_value.map { |v| v[:value] }.join("_")
+            alarm_name = "skyeye::#{resource_type}::#{target}::#{watch[:name]}-#{threshold}"
+
+            alarm = cw.alarms[alarm_name]
+
+            unless alarm.exists?
+              is_command = !!watch[:command]
+              raise "Cannot specify command for non-instance watch" if is_command && resource_type != :instance
+
+              if is_command || watch[threshold]
+                @logger.info "Register Alarm #{alarm_name}"
+
+                cw.alarms.create(alarm_name, {
+                  namespace: is_command ? @config[:namespace] : namespace_for_resource_type(resource_type),
+                  metric_name: watch[:name],
+                  dimensions: dimension_value,
+                  comparison_operator: watch[:comparison_operator] || "GreaterThanThreshold",
+                  evaluation_periods: 1,
+                  period: (watch[:period] || @config[:alarm_periods][threshold] || 5) * 60,
+                  statistic: watch[:statistic] || "Maximum",
+                  threshold: is_command ? (threshold == :warning ? 0 : 1) : watch[threshold],
+                  insufficient_data_actions: (threshold == :critical && is_command ? @config[:arns] : []),
+                  actions_enabled: true,
+                  alarm_actions: @config[:arns],
+                  alarm_description: "skyeye: #{watch.to_json}",
+                })
+              end
+            end
+          end
+        end
+      end
+    end
+
+    def start!
+      @running = true
+      @logger.info "SkyEye starting."
+
+      namespace = @config[:namespace] || "skyeye"
+
+      @threads = @config[:watches][:instance].map do |w|
+        thread_for_watch(namespace, w)
+      end.compact
+
+      @threads.each(&:join)
+
+      if @killed_by
+        @logger.error(@killed_by.to_s + " " + @killed_by.backtrace.join("\n"))
+        raise @killed_by
+      end
+    end
+
+    def kill!(e)
+      @mutex.synchronize do
+        @killed_by = e
+      end
+    end
+
+    def thread_for_watch(namespace, watch)
+      return nil if watch[:type] == "aws"
+      raise "missing name" unless watch[:name]
+
+      Thread.new do 
+        last_check = nil
+
+        interval = watch[:interval] || 5
+        cw = AWS::CloudWatch.new.client
+
+        do_check = lambda do |&block|
+          if !last_check || (Time.now - last_check > interval)
+            last_check = Time.now
+            block.call
+          end
+        end
+
+        begin
+          loop do
+            running = true
+
+            @mutex.synchronize do
+              running = @running && !@killed_by
+            end
+
+            break unless running
+
+            do_check.call do
+              @logger.info "[CHECK] [#{instance_id}] #{watch[:name]}::#{watch[:type]} #{watch[:command] || ""}"
+
+              if watch[:command]
+                command = watch[:command]
+                message = nil
+
+                status = Open4::popen4(command) do |pid, stdin, stdout, stderr|
+                  message = "#{watch[:name]} :: #{stdout.readlines.join(" ").gsub(/\n/, " ").chomp}"
+                end
+
+                case status
+                when 0
+                  @logger.info message
+                when 1
+                  @logger.warn message
+                when 2
+                  @logger.error message
+                else
+                  @logger.warn message
+                end
+
+                cw.put_metric_data({
+                  namespace: namespace,
+                  metric_data: [{
+                    metric_name: watch[:name],
+                    dimensions: [{
+                      :name => "InstanceId",
+                      :value => instance_id
+                    }],
+                    value: status.exitstatus,
+                    unit: "None"
+                  }]
+                })
+              else
+                raise "Missing command or aws id for watch #{watch[:name]}"
+              end
+            end
+
+            sleep 1
+          end
+        rescue Exception => e
+          kill!(e)
+        end
+      end
+    end
+
+    def shutdown!
+      @mutex.synchronize do
+        @running = false
+      end
+    end
+
+    def load_or_register_topics!
+      sns = AWS::SNS.new
+
+      current_topics = sns.client.list_topics
+
+      arns = []
+
+      (@config[:topics] || ["skyeye-alerts"]).each do |topic_name|
+        current_topic = current_topics[:topics].find do |topic|
+          topic[:topic_arn].split(/:/)[-1] == topic_name
+        end
+
+        if current_topic
+          arns << current_topic[:topic_arn]
+        else
+          arn = sns.topics.create(topic_name).arn
+          arns << arn
+
+          @logger.info "Created Topic #{arn}"
+        end
+      end
+
+      @config[:arns] = arns
+    end
+  end
+end

data/lib/skyeye/version.rb

@@ -0,0 +1,3 @@
+module Skyeye
+  VERSION = "0.0.1"
+end

data/lib/skyeye.rb

@@ -0,0 +1,4 @@
+module SkyEye
+end
+
+require 'skyeye/exec'

data/skyeye.gemspec

@@ -0,0 +1,19 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'skyeye/version'
+
+Gem::Specification.new do |gem|
+  gem.name          = "skyeye"
+  gem.version       = Skyeye::VERSION
+  gem.authors       = ["Greg Fodor"]
+  gem.email         = ["gfodor@gmail.com"]
+  gem.description   = %q{SkyEye is a daemon that lets you have nagios-like alerts on EC2 using CloudWatch.}
+  gem.summary   = %q{SkyEye is a daemon that lets you have nagios-like alerts on EC2 using CloudWatch.}
+  gem.homepage      = "http://github.com/gfodor/skyeye"
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+end

metadata

@@ -0,0 +1,58 @@
+--- !ruby/object:Gem::Specification
+name: skyeye
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Greg Fodor
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-03-14 00:00:00.000000000 Z
+dependencies: []
+description: SkyEye is a daemon that lets you have nagios-like alerts on EC2 using
+  CloudWatch.
+email:
+- gfodor@gmail.com
+executables:
+- skyeye
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/skyeye
+- config/skyeye.yml.example
+- lib/skyeye.rb
+- lib/skyeye/exec.rb
+- lib/skyeye/version.rb
+- skyeye.gemspec
+homepage: http://github.com/gfodor/skyeye
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: SkyEye is a daemon that lets you have nagios-like alerts on EC2 using CloudWatch.
+test_files: []