interferon 0.0.1

data/lib/interferon/destinations/datadog.rb

@@ -0,0 +1,169 @@
+require 'dogapi'
+require 'set'
+
+module Interferon::Destinations
+  class Datadog
+    include Interferon::Logging
+
+    attr_accessor :concurrency
+    ALERT_KEY = 'This alert was created via the alerts framework'
+
+    def initialize(options)
+      %w{app_key api_key}.each do |req|
+        unless options[req]
+          raise ArgumentError, "missing required argument #{req}"
+        end
+      end
+
+      @dog = Dogapi::Client.new(options['api_key'], options['app_key'])
+      @dry_run = !!options['dry_run']
+      @existing_alerts = nil
+
+      # create datadog alerts 10 at a time
+      @concurrency = 10
+
+      @stats = {
+        :alerts_created => 0,
+        :alerts_updated => 0,
+        :alerts_deleted => 0,
+        :alerts_silenced => 0,
+        :api_successes => 0,
+        :api_client_errors => 0,
+        :api_unknown_errors => 0,
+        :manually_created_alerts => 0,
+      }
+    end
+
+    def existing_alerts
+      unless @existing_alerts
+        resp = @dog.get_all_alerts()
+        alerts = resp[1]['alerts']
+
+        # key alerts by name
+        @existing_alerts = Hash[alerts.map{ |a| [a['name'], a] }]
+
+        # count how many are manually created
+        @stats[:manually_created_alerts] = \
+          @existing_alerts.reject{|n,a| a['message'].include?(ALERT_KEY)}.length
+
+        log.info "datadog: found %d existing alerts; %d were manually created" % [
+          @existing_alerts.length,
+          @stats[:manually_created_alerts],
+        ]
+      end
+
+      return @existing_alerts
+    end
+
+    def create_alert(alert, people)
+      # create a message which includes the notifications
+      message = [
+        alert['message'],
+        ALERT_KEY,
+        people.map{ |p| "@#{p}" }
+      ].flatten.join("\n")
+
+      # create the hash of options to send to datadog
+      alert_opts = {
+        :name => alert['name'],
+        :message => message,
+        :silenced => alert['silenced'] || alert['silenced_until'] > Time.now,
+        :notify_no_data => alert['notify_no_data'],
+        :timeout_h => nil,
+      }
+
+      # allow an optional timeframe for "no data" alerts to be specified
+      # (this feature is supported, even though it's not documented)
+      alert_opts[:no_data_timeframe] = alert['no_data_timeframe'] if alert['no_data_timeframe']
+
+      # timeout is in seconds, but set it to 1 hour at least
+      alert_opts[:timeout_h] = [1, (alert['timeout'].to_i / 3600)].max if alert['timeout']
+
+      # new alert, create it
+      if existing_alerts[alert['name']].nil?
+        action = :creating
+        log.debug("new alert #{alert['name']}")
+
+        resp = @dog.alert(
+          alert['metric']['datadog_query'].strip,
+          alert_opts,
+        ) unless @dry_run
+
+      # existing alert, modify it
+      else
+        action = :updating
+        id = existing_alerts[alert['name']]['id']
+        log.debug("updating existing alert #{id} (#{alert['name']})")
+
+        resp = @dog.update_alert(
+          id,
+          alert['metric']['datadog_query'].strip,
+          alert_opts
+        ) unless @dry_run
+      end
+
+      # log whenever we've encountered errors
+      if resp
+        code = resp[0].to_i
+
+        # client error
+        if code == 400
+          statsd.gauge('datadog.api.unknown_error', 0, :tags => ["alert:#{alert}"])
+          statsd.gauge('datadog.api.client_error', 1, :tags => ["alert:#{alert}"])
+          statsd.gauge('datadog.api.success', 0, :tags => ["alert:#{alert}"])
+
+          @stats[:api_client_errors] += 1
+          log.error("client error while #{action} alert '#{alert['name']}';" \
+              " query was '#{alert['metric']['datadog_query'].strip}'")
+
+        # unknown (prob. datadog) error:
+        elsif code >= 400 || code == -1
+          statsd.gauge('datadog.api.unknown_error', 1, :tags => ["alert:#{alert}"])
+          statsd.gauge('datadog.api.client_error', 0, :tags => ["alert:#{alert}"])
+          statsd.gauge('datadog.api.success', 0, :tags => ["alert:#{alert}"])
+
+          @stats[:api_unknown_errors] += 1
+          log.error("unknown error while #{action} alert '#{alert['name']}':" \
+              " query was '#{alert['metric']['datadog_query'].strip}'" \
+              " response was #{resp[0]}:'#{resp[1].inspect}'")
+
+        # assume this was a success
+        else
+          statsd.gauge('datadog.api.unknown_error', 0, :tags => ["alert:#{alert}"])
+          statsd.gauge('datadog.api.client_error', 0, :tags => ["alert:#{alert}"])
+          statsd.gauge('datadog.api.success', 1, :tags => ["alert:#{alert}"])
+
+          @stats[:api_successes] += 1
+          @stats[:alerts_created] += 1 if action == :creating
+          @stats[:alerts_updated] += 1 if action == :updating
+          @stats[:alerts_silenced] += 1 if alert_opts[:silenced]
+        end
+      end
+
+      # lets key alerts by their name
+      return alert['name']
+    end
+
+    def remove_alert(alert)
+      if alert['message'].include?(ALERT_KEY)
+        log.debug("deleting alert #{alert['id']} (#{alert['name']})")
+        @dog.delete_alert(alert['id']) unless @dry_run
+        @stats[:alerts_deleted] += 1
+      else
+        log.warn("not deleting manually-created alert #{alert['id']} (#{alert['name']})")
+      end
+    end
+
+    def report_stats
+      @stats.each do |k,v|
+        statsd.gauge("datadog.#{k}", v)
+      end
+
+      log.info "datadog: created %d updated %d and deleted %d alerts" % [
+        @stats[:alerts_created],
+        @stats[:alerts_updated],
+        @stats[:alerts_deleted],
+      ]
+    end
+  end
+end

data/lib/interferon/group_sources/filesystem.rb

@@ -0,0 +1,38 @@
+
+module Interferon::GroupSources
+  class Filesystem
+    def initialize(options)
+      raise ArgumentError, "missing paths for loading groups from filesystem" \
+        unless options['paths']
+
+      @paths = options['paths']
+    end
+
+    def list_groups
+      groups = {}
+
+      @paths.each do |path|
+        path = File.expand_path(path)
+        unless Dir.exists?(path)
+          log.warn "no such directory #{path} for reading group files"
+          next
+        end
+
+        Dir.glob(File.join(path, '*.{json,yaml}')) do |group_file|
+          begin
+            group = YAML::parse(File.read(group_file))
+          rescue YAML::SyntaxError => e
+            log.error "syntax error in group file #{group_file}: #{e}"
+          rescue StandardError => e
+            log.warn "error reading group file #{group_file}: #{e}"
+          else
+            group = group.to_ruby
+            groups[group['name']] = group['people'] || []
+          end
+        end
+      end
+
+      return groups
+    end
+  end
+end

data/lib/interferon/host_sources/aws_dynamo.rb

@@ -0,0 +1,51 @@
+require 'aws'
+
+module Interferon::HostSources
+  class AwsDynamo
+    def initialize(options)
+      missing = %w{access_key_id secret_access_key}.reject{|r| options.key?(r)}
+      raise ArgumentError, "missing these required arguments for source AwsDynamo: #{missing.inspect}"\
+        unless missing.empty?
+
+      @access_key_id = options['access_key_id']
+      @secret_access_key = options['secret_access_key']
+
+      # initialize a list of regions to check
+      if options['regions'] && !options['regions'].empty?
+        @regions = options['regions']
+      else
+        @regions = AWS::regions.map(&:name)
+      end
+    end
+
+    def list_hosts
+      hosts = []
+
+      @regions.each do |region|
+        client = AWS::DynamoDB.new(
+          :access_key_id => @access_key_id,
+          :secret_access_key => @secret_access_key,
+          :region => region)
+
+        AWS.memoize do
+          client.tables.each do |table|
+            hosts << {
+              :source => 'aws_dynamo',
+              :region => region,
+              :table_name => table.name,
+
+              :read_capacity => table.read_capacity_units,
+              :write_capacity => table.write_capacity_units,
+
+              # dynamodb does not support tagging
+              :owners => [],
+              :owner_groups => [],
+            }
+          end
+        end
+      end
+
+      return hosts
+    end
+  end
+end

data/lib/interferon/host_sources/aws_elasticache.rb

@@ -0,0 +1,69 @@
+require 'aws'
+
+module Interferon::HostSources
+  class AwsElasticache
+    def initialize(options)
+      missing = %w{access_key_id secret_access_key}.reject{|r| options.key?(r)}
+      raise ArgumentError, "missing these required arguments for source AwsElasticache: #{missing.inspect}"\
+        unless missing.empty?
+
+      @access_key_id = options['access_key_id']
+      @secret_access_key = options['secret_access_key']
+
+      # initialize a list of regions to check
+      if options['regions'] && !options['regions'].empty?
+        @regions = options['regions']
+      else
+        @regions = AWS::regions.map(&:name)
+      end
+    end
+
+    def list_hosts
+      hosts = []
+
+      @regions.each do |region|
+        clusters = []
+        client = AWS::ElastiCache.new(
+          :access_key_id => @access_key_id,
+          :secret_access_key => @secret_access_key,
+          :region => region).client
+
+        AWS.memoize do
+          # read the list of cache clusters; we have to do our own pagination
+          clusters = []
+          options = {:show_cache_node_info => true}
+          loop do
+            r = client.describe_cache_clusters(options)
+            clusters += r.data[:cache_clusters]
+
+            break unless r.data[:marker]
+            options[:marker] = r.data[:marker]
+          end
+
+          # iterate over the nodes in each cluster and add each one to hosts
+          clusters.each do |cluster|
+            cluster[:cache_nodes].each do |node|
+              hosts << {
+                :source => 'aws_elasticache',
+                :region => region,
+
+                :cluster_id => cluster[:cache_cluster_id],
+                :cluster_status => cluster[:cache_cluster_status],
+                :node_type => cluster[:cache_node_type],
+                :peer_nodes => cluster[:num_cache_nodes],
+
+                :node_status => node[:cache_node_status],
+
+                # elasticache does not support tagging
+                :owners => [],
+                :owner_groups => [],
+              }
+            end
+          end
+        end
+      end
+
+      return hosts
+    end
+  end
+end

data/lib/interferon/host_sources/aws_rds.rb

@@ -0,0 +1,92 @@
+require 'aws'
+
+module Interferon::HostSources
+  class AwsRds
+    def initialize(options)
+      missing = %w{access_key_id secret_access_key}.reject{|r| options.key?(r)}
+      raise ArgumentError, "missing these required arguments for source AwsRds: #{missing.inspect}"\
+        unless missing.empty?
+
+      @access_key_id = options['access_key_id']
+      @secret_access_key = options['secret_access_key']
+
+      # initialize a list of regions to check
+      if options['regions'] && !options['regions'].empty?
+        @regions = options['regions']
+      else
+        @regions = AWS::regions.map(&:name)
+      end
+    end
+
+    def list_hosts
+      hosts = []
+
+      @regions.each do |region|
+        rds = AWS::RDS.new(
+          :access_key_id => @access_key_id,
+          :secret_access_key => @secret_access_key,
+          :region => region)
+
+        AWS.memoize do
+          rds.instances.each do |instance|
+            # get the tags for the instance
+            arn = arn(region, instance.id)
+            tag_list = rds.client.list_tags_for_resource(:resource_name => arn)[:tag_list]
+            tags = Hash[ tag_list.map { |h| [h[:key], h[:value]] } ]
+
+            tags['owners'] ||= ''
+            tags['owner_groups'] ||= ''
+
+            # build the host data for this instance
+            hosts << {
+              :source => 'aws_rds',
+              :region => region,
+              :instance_id => instance.id,
+              :db_name => instance.db_name,
+              :engine => instance.engine,
+              :engine_version => instance.engine_version,
+
+              # metrics
+              :allocated_storage => instance.allocated_storage,
+              :iops => instance.iops,
+
+              # replication info
+              :is_replica => !instance.read_replica_source_db_instance_identifier.nil?,
+              :replica_source_name => instance.read_replica_source_db_instance_identifier,
+              :replica_names => instance.read_replica_db_instance_identifiers.join(','),
+              :replicas => instance.read_replica_db_instance_identifiers.count,
+
+              :owners => tags['owners'].split(','),
+              :owner_groups => tags['owner_groups'].split(','),
+
+              :db_env => tags['db_env'],
+              :db_role => tags['db_role'],
+            }
+          end
+        end
+      end
+
+      return hosts
+    end
+
+    private
+    def arn(region, instance_id)
+      return "arn:aws:rds:#{region}:#{account_number}:db:#{instance_id}"
+    end
+
+    # unfortunately, this appears to be the only way to get your account number
+    def account_number
+      return @account_number if @account_number
+
+      begin
+        my_arn = AWS::IAM.new(
+          :access_key_id => @access_key_id,
+          :secret_access_key => @secret_access_key).client.get_user()[:user][:arn]
+      rescue AWS::IAM::Errors::AccessDenied => e
+        my_arn = e.message.split[1]
+      end
+
+      @account_number = my_arn.split(':')[4]
+    end
+  end
+end

data/lib/interferon/host_sources/optica.rb

@@ -0,0 +1,35 @@
+require 'net/http'
+require 'json'
+
+module Interferon::HostSources
+  class Optica
+    include Logging
+
+    def initialize(options)
+      raise ArgumentError, "missing host for optica source" \
+        unless options['host']
+
+      @host = options['host']
+      @port = options['port'] || 80
+    end
+
+    def list_hosts
+      con = Net::HTTP.new(@host, @port)
+      con.read_timeout = 60
+      con.open_timeout = 60
+
+      response = con.get('/')
+      data = JSON::parse(response.body)
+
+      return data['nodes'].map{|ip, host| {
+          :source => 'optica',
+          :hostname => host['hostname'],
+          :role => host['role'],
+          :environment => host['environment'],
+
+          :owners => host['ownership'] && host['ownership']['people'] || [],
+          :owner_groups => host['ownership'] && host['ownership']['groups'] || [],
+        }}
+    end
+  end
+end