sensu-plugins-aws 0.0.4 → 1.0.0

data/bin/check-rds-events.rb

@@ -16,7 +16,7 @@
 #   Linux
 #
 # DEPENDENCIES:
-#   gem: aws-sdk
+#   gem: aws-sdk-v1
 #   gem: sensu-plugin
 #
 # USAGE:
@@ -35,28 +35,41 @@ require 'aws-sdk-v1'
 
 class CheckRDSEvents < Sensu::Plugin::Check::CLI
   option :aws_access_key,
-         short: '-a AWS_ACCESS_KEY',
-         long: '--aws-access-key AWS_ACCESS_KEY',
-         description: "AWS Access Key. Either set ENV['AWS_ACCESS_KEY_ID'] or provide it as an option",
-         default: ENV['AWS_ACCESS_KEY_ID']
+         short:       '-a AWS_ACCESS_KEY',
+         long:        '--aws-access-key AWS_ACCESS_KEY',
+         description: "AWS Access Key. Either set ENV['AWS_ACCESS_KEY'] or provide it as an option",
+         default:     ENV['AWS_ACCESS_KEY']
 
   option :aws_secret_access_key,
-         short: '-s AWS_SECRET_ACCESS_KEY',
-         long: '--aws-secret-access-key AWS_SECRET_ACCESS_KEY',
-         description: "AWS Secret Access Key. Either set ENV['AWS_SECRET_ACCESS_KEY'] or provide it as an option",
-         default: ENV['AWS_SECRET_ACCESS_KEY']
+         short:       '-k AWS_SECRET_KEY',
+         long:        '--aws-secret-access-key AWS_SECRET_KEY',
+         description: "AWS Secret Access Key. Either set ENV['AWS_SECRET_KEY'] or provide it as an option",
+         default:     ENV['AWS_SECRET_KEY']
 
   option :aws_region,
-         short: '-r AWS_REGION',
-         long: '--aws-region REGION',
-         description: 'AWS Region (such as eu-west-1).',
-         default: 'us-east-1'
+         short:       '-r AWS_REGION',
+         long:        '--aws-region REGION',
+         description: 'AWS Region (defaults to us-east-1).',
+         default:     'us-east-1'
 
-  def run # rubocop:disable AbcSize
-    rds = AWS::RDS::Client.new(
-      access_key_id: config[:aws_access_key],
+  def aws_config
+    { access_key_id: config[:aws_access_key],
       secret_access_key: config[:aws_secret_access_key],
-      region: config[:aws_region])
+      region: config[:aws_region]
+    }
+  end
+
+  def run
+    clusters = maint_clusters
+    if clusters.empty?
+      ok
+    else
+      critical("Clusters w/ critical events: #{clusters.join(',')}")
+    end
+  end
+
+  def maint_clusters
+    rds = AWS::RDS::Client.new aws_config
 
     begin
       # fetch all clusters identifiers
@@ -74,11 +87,6 @@ class CheckRDSEvents < Sensu::Plugin::Check::CLI
     rescue => e
       unknown "An error occurred processing AWS RDS API: #{e.message}"
     end
-
-    if maint_clusters.empty?
-      ok
-    else
-      critical("Clusters w/ critical events: #{maint_clusters.join(',')}")
-    end
+    maint_clusters
   end
 end

data/bin/check-rds.rb

@@ -12,7 +12,7 @@
 #   Linux
 #
 # DEPENDENCIES:
-#   gem: aws-sdk
+#   gem: aws-sdk-v1
 #   gem: sensu-plugin
 #
 # USAGE:
@@ -35,6 +35,10 @@
 #   You can check multiple metrics simultaneously. Highest severity will be reported
 #   check-rds -i sensu-admin-db --cpu-warning-over 80 --cpu-critical-over 90 --memory-warning-over 60 --memory-critical-over 80
 #
+#   You can ignore accept nil values returned for a time periods from Cloudwatch as being an OK.  Amazon falls behind in their
+#   metrics from time to time and this prevents false positives
+#   check-rds -i sensu-admin-db --cpu-critical-over 90 -a
+#
 # NOTES:
 #
 # LICENSE:
@@ -48,24 +52,23 @@ require 'aws-sdk-v1'
 require 'time'
 
 class CheckRDS < Sensu::Plugin::Check::CLI
-  option :access_key_id,
-         short:       '-k N',
-         long:        '--access-key-id ID',
-         description: 'AWS access key ID',
-         default: ENV['AWS_ACCESS_KEY_ID']
-
-  option :secret_access_key,
-         short:       '-s N',
-         long:        '--secret-access-key KEY',
-         description: 'AWS secret access key',
-         default: ENV['AWS_SECRET_ACCESS_KEY']
-
-  option :region,
-         short:       '-r R',
-         long:        '--region REGION',
-         description: 'AWS region',
-         description: 'AWS Region (such as eu-west-1).',
-         default: 'us-east-1'
+  option :aws_access_key,
+         short:       '-a AWS_ACCESS_KEY',
+         long:        '--aws-access-key AWS_ACCESS_KEY',
+         description: "AWS Access Key. Either set ENV['AWS_ACCESS_KEY'] or provide it as an option",
+         default:     ENV['AWS_ACCESS_KEY']
+
+  option :aws_secret_access_key,
+         short:       '-k AWS_SECRET_KEY',
+         long:        '--aws-secret-access-key AWS_SECRET_KEY',
+         description: "AWS Secret Access Key. Either set ENV['AWS_SECRET_KEY'] or provide it as an option",
+         default:     ENV['AWS_SECRET_KEY']
+
+  option :aws_region,
+         short:       '-r AWS_REGION',
+         long:        '--aws-region REGION',
+         description: 'AWS Region (defaults to us-east-1).',
+         default:     'us-east-1'
 
   option :db_instance_id,
          short:       '-i N',
@@ -93,6 +96,12 @@ class CheckRDS < Sensu::Plugin::Check::CLI
          proc:        proc { |a| a.downcase.intern },
          description: 'CloudWatch statistics method'
 
+  option :accept_nil,
+         short: '-n',
+         long: '--accept_nil',
+         description: 'Continue if CloudWatch provides no metrics for the time period',
+         default: false
+
   %w(warning critical).each do |severity|
     option :"availability_zone_#{severity}",
            long:        "--availability-zone-#{severity} AZ",
@@ -107,10 +116,10 @@ class CheckRDS < Sensu::Plugin::Check::CLI
   end
 
   def aws_config
-    hash = {}
-    hash.update access_key_id: config[:access_key_id], secret_access_key: config[:secret_access_key] if config[:access_key_id] && config[:secret_access_key]
-    hash.update region: config[:region] if config[:region]
-    hash
+    { access_key_id: config[:aws_access_key],
+      secret_access_key: config[:aws_secret_access_key],
+      region: config[:aws_region]
+    }
   end
 
   def rds
@@ -145,8 +154,12 @@ class CheckRDS < Sensu::Plugin::Check::CLI
   def latest_value(metric, unit)
     values = metric.statistics(statistics_options.merge unit: unit).datapoints.sort_by { |datapoint| datapoint[:timestamp] }
 
-    # handle time periods that are too small to return usable values
-    values.empty? ? unknown('Requested time period did not return values from Cloudwatch. Try increasing your time period.') : values.last[config[:statistics]]
+    # handle time periods that are too small to return usable values.  # this is a cozy addition that wouldn't port upstream.
+    if values.empty?
+      config[:accept_nil] ? ok('Cloudwatch returned no results for time period. Accept nil passed so OK') : unknown('Requested time period did not return values from Cloudwatch. Try increasing your time period.') # rubocop:disable all
+    else
+      values.last[config[:statistics]]
+    end
   end
 
   def flag_alert(severity, message)

data/bin/check-redshift-events.rb

@@ -12,7 +12,7 @@
 #   Linux
 #
 # DEPENDENCIES:
-#   gem: aws-sdk
+#   gem: aws-sdk-v1
 #   gem: sensu-plugin
 #
 # USAGE:
@@ -41,19 +41,19 @@ class CheckRedshiftEvents < Sensu::Plugin::Check::CLI
   option :aws_access_key,
          short: '-a AWS_ACCESS_KEY',
          long: '--aws-access-key AWS_ACCESS_KEY',
-         description: "AWS Access Key. Either set ENV['AWS_ACCESS_KEY_ID'] or provide it as an option",
-         default: ENV['AWS_ACCESS_KEY_ID']
+         description: "AWS Access Key. Either set ENV['AWS_ACCESS_KEY'] or provide it as an option",
+         default: ENV['AWS_ACCESS_KEY']
 
   option :aws_secret_access_key,
-         short: '-s AWS_SECRET_ACCESS_KEY',
-         long: '--aws-secret-access-key AWS_SECRET_ACCESS_KEY',
-         description: "AWS Secret Access Key. Either set ENV['AWS_SECRET_ACCESS_KEY'] or provide it as an option",
-         default: ENV['AWS_SECRET_ACCESS_KEY']
+         short: '-k AWS_SECRET_KEY',
+         long: '--aws-secret-access-key AWS_SECRET_KEY',
+         description: "AWS Secret Access Key. Either set ENV['AWS_SECRET_KEY'] or provide it as an option",
+         default: ENV['AWS_SECRET_KEY']
 
   option :aws_region,
          short: '-r AWS_REGION',
          long: '--aws-region REGION',
-         description: 'AWS Region (such as eu-west-1).',
+         description: 'AWS Region (defaults to us-east-1).',
          default: 'us-east-1'
 
   option :instances,
@@ -63,12 +63,16 @@ class CheckRedshiftEvents < Sensu::Plugin::Check::CLI
          proc: proc { |a| a.split(',') },
          default: []
 
+  def aws_config
+    { access_key_id: config[:aws_access_key],
+      secret_access_key: config[:aws_secret_access_key],
+      region: config[:aws_region]
+    }
+  end
+
   # setup a redshift connection using aws-sdk
   def redshift
-    @redshift ||= AWS::Redshift::Client.new(
-      access_key_id: config[:aws_access_key],
-      secret_access_key: config[:aws_secret_access_key],
-      region: config[:aws_region])
+    @redshift ||= AWS::Redshift::Client.new aws_config
   end
 
   # fetch all clusters in the region from AWS

data/bin/check-ses-limit.rb

@@ -35,14 +35,20 @@ class CheckSESLimit < Sensu::Plugin::Check::CLI
   option :aws_access_key,
          short: '-a AWS_ACCESS_KEY',
          long: '--aws-access-key AWS_ACCESS_KEY',
-         description: "AWS Access Key. Either set ENV['AWS_ACCESS_KEY_ID'] or provide it as an option",
-         required: true
+         description: "AWS Access Key. Either set ENV['AWS_ACCESS_KEY'] or provide it as an option",
+         default: ENV['AWS_ACCESS_KEY']
 
   option :aws_secret_access_key,
-         short: '-s AWS_SECRET_ACCESS_KEY',
-         long: '--aws-secret-access-key AWS_SECRET_ACCESS_KEY',
-         description: "AWS Secret Access Key. Either set ENV['AWS_SECRET_ACCESS_KEY'] or provide it as an option",
-         required: true
+         short: '-k AWS_SECRET_KEY',
+         long: '--aws-secret-access-key AWS_SECRET_KEY',
+         description: "AWS Secret Access Key. Either set ENV['AWS_SECRET_KEY'] or provide it as an option",
+         default: ENV['AWS_SECRET_KEY']
+
+  option :aws_region,
+         short: '-r AWS_REGION',
+         long: '--aws-region REGION',
+         description: 'AWS Region (defaults to us-east-1).',
+         default: 'us-east-1'
 
   option :warn_percent,
          short: '-W WARN_PERCENT',
@@ -59,10 +65,10 @@ class CheckSESLimit < Sensu::Plugin::Check::CLI
          proc: proc(&:to_i)
 
   def aws_config
-    hash = {}
-    hash.update access_key_id: config[:aws_access_key], secret_access_key: config[:aws_secret_access_key]\
-      if config[:aws_access_key] && config[:aws_secret_access_key]
-    hash
+    { access_key_id: config[:aws_access_key],
+      secret_access_key: config[:aws_secret_access_key],
+      region: config[:aws_region]
+    }
   end
 
   def run

data/bin/check-sqs-messages.rb

@@ -12,7 +12,7 @@
 #   Linux
 #
 # DEPENDENCIES:
-#   gem: aws-sdk
+#   gem: aws-sdk-v1
 #   gem: sensu-plugin
 #
 # USAGE:
@@ -37,17 +37,19 @@ class SQSMsgs < Sensu::Plugin::Check::CLI
   option :aws_access_key,
          short: '-a AWS_ACCESS_KEY',
          long: '--aws-access-key AWS_ACCESS_KEY',
-         description: "AWS Access Key. Either set ENV['AWS_ACCESS_KEY_ID'] or provide it as an option"
+         description: "AWS Access Key. Either set ENV['AWS_ACCESS_KEY'] or provide it as an option",
+         default: ENV['AWS_ACCESS_KEY']
 
   option :aws_secret_access_key,
-         short: '-s AWS_SECRET_ACCESS_KEY',
-         long: '--aws-secret-access-key AWS_SECRET_ACCESS_KEY',
-         description: "AWS Secret Access Key. Either set ENV['AWS_SECRET_ACCESS_KEY'] or provide it as an option"
+         short: '-k AWS_SECRET_KEY',
+         long: '--aws-secret-access-key AWS_SECRET_KEY',
+         description: "AWS Secret Access Key. Either set ENV['AWS_SECRET_KEY'] or provide it as an option",
+         default: ENV['AWS_SECRET_KEY']
 
   option :aws_region,
-         description: 'AWS Region (such as us-east-1)',
          short: '-r AWS_REGION',
-         long: '--aws-region AWS_REGION',
+         long: '--aws-region REGION',
+         description: 'AWS Region (defaults to us-east-1).',
          default: 'us-east-1'
 
   option :queue,
@@ -85,11 +87,10 @@ class SQSMsgs < Sensu::Plugin::Check::CLI
          proc: proc(&:to_i)
 
   def aws_config
-    hash = {}
-    hash.update access_key_id: config[:aws_access_key], secret_access_key: config[:aws_secret_access_key]\
-      if config[:aws_access_key] && config[:aws_secret_access_key]
-    hash.update region: config[:aws_region]
-    hash
+    { access_key_id: config[:aws_access_key],
+      secret_access_key: config[:aws_secret_access_key],
+      region: config[:aws_region]
+    }
   end
 
   def run

data/bin/handler-ec2_node.rb

@@ -1,6 +1,8 @@
 #!/usr/bin/env ruby
 #
 # CHANGELOG:
+# * 0.5.0:
+#   - Adds configuration to filter by state reason
 # * 0.4.0:
 #   - Adds ability to specify a list of states an individual client can have in
 #     EC2. If none is specified, it filters out 'terminated' and 'stopped'
@@ -22,18 +24,21 @@
 # Optionally, you may specify a client attribute `ec2_states`, a list of valid
 # states an instance may have.
 #
+# You may also specify a client attribute `ec2_state_reasons`, a list of regular
+# expressions to match state reasons against. This is useful if you want to fail
+# on any `Client.*` state reason or on `Server.*` state reason. The default is
+# to match any state reason `.*` Regardless, eventually a client will be
+# deleted once AWS stops responding that the instance id exists.
+#
 # NOTE: The implementation for correlating Sensu clients to EC2 instances may
 # need to be modified to fit your organization. The current implementation
 # assumes that Sensu clients' names are the same as their instance IDs in EC2.
 # If this is not the case, you can either sub-class this handler and override
-# `ec2_node_exists?` in your own organization-specific handler, or modify this
+# `ec2_node_should_be_deleted?` in your own organization-specific handler, or modify this
 # handler to suit your needs.
 #
-# Requires the following Rubygems (`gem install $GEM`):
-#   - sensu-plugin
-#   - fog
 #
-# Requires a Sensu configuration snippet:
+# Optional a Sensu configuration snippet:
 #   {
 #     "aws": {
 #       "access_key": "adsafdafda",
@@ -47,6 +52,11 @@
 #   - AWS_SECRET_ACCESS_KEY
 #   - EC2_REGION
 #
+# If none of the settings are found it will then attempt to
+# generate temporary credentials from the IAM instance profile
+#
+# If region is not specified in either of the above 2 mechanisms
+# we will make a request for the EC2 instances current region.
 #
 # To use, you can set it as the keepalive handler for a client:
 #   {
@@ -92,17 +102,21 @@
 
 require 'timeout'
 require 'sensu-handler'
-require 'fog'
+require 'net/http'
+require 'uri'
+require 'aws-sdk'
+require 'sensu-plugins-aws'
 
 class Ec2Node < Sensu::Handler
+  include Common
+
   def filter; end
 
   def handle
-    # #YELLOW
-    unless ec2_node_exists? # rubocop:disable UnlessElse
+    if ec2_node_should_be_deleted?
       delete_sensu_client!
     else
-      puts "[EC2 Node] #{@event['client']['name']} appears to exist in EC2"
+      puts "[EC2 Node] #{@event['client']['name']} is in an invalid state"
     end
   end
 
@@ -111,46 +125,55 @@ class Ec2Node < Sensu::Handler
     deletion_status(response)
   end
 
-  def ec2_node_exists?
-    states = acquire_valid_states
-    filtered_instances = ec2.servers.select { |s| states.include?(s.state) }
-    instance_ids = filtered_instances.map(&:id)
-    instance_ids.each do |id|
-      return true if id == @event['client']['name']
+  def ec2_node_should_be_deleted?
+    ec2 = Aws::EC2::Client.new(region: region)
+    states = @event['client']['ec2_states'] || settings['ec2_node']['ec2_states'] || ['shutting-down', 'terminated', 'stopping', 'stopped']
+    begin
+      instances = ec2.describe_instances(instance_ids: [@event['client']['name']]).reservations[0]
+      if instances.nil?
+        true
+      else
+        instance = instances.instances[0]
+        state_reason = instance.state_reason.code
+        state = instance.state.name
+        states.include?(state) && state_reasons.any? { |reason| Regexp.new(reason) =~ state_reason }
+      end
+    rescue Aws::EC2::Errors::InvalidInstanceIDNotFound
+      true
     end
-    false # no match found, node doesn't exist
   end
 
-  def ec2
-    @ec2 ||= begin
-      key = settings['aws']['access_key'] || ENV['AWS_ACCESS_KEY_ID']
-      secret = settings['aws']['secret_key'] || ENV['AWS_SECRET_ACCESS_KEY']
-      region = settings['aws']['region'] || ENV['EC2_REGION']
-      Fog::Compute.new(provider: 'AWS',
-                       aws_access_key_id: key,
-                       aws_secret_access_key: secret,
-                       region: region)
+  def region
+    @region ||= begin
+      region_check = ENV['EC2_REGION']
+      region_check = settings['aws']['region'] if settings.key? 'aws'
+      if region_check.nil? || region_check.empty?
+        region_check = Net::HTTP.get(URI('http://169.254.169.254/latest/meta-data/placement/availability-zone'))
+        matches = /(\w+\-\w+\-\d+)/.match(region_check)
+        if !matches.nil? && !matches.captures.empty?
+          region_check = matches.captures[0]
+        end
+      end
+      region_check
     end
   end
 
+  def state_reasons
+    default_reasons = %w('UserInitiatedShutdown', 'SpotInstanceTermination', 'InstanceInitiatedShutdown')
+    reasons = @event['client']['ec2_state_reasons'] || settings['ec2_node']['ec2_state_reasons'] || default_reasons
+    @state_reasons ||= reasons.each { |reason| Regexp.new(reason) }
+  end
+
   def deletion_status(code)
     case code
     when '202'
-      puts "[EC2 Node] 202: Successfully deleted Sensu client: #{node}"
+      puts "[EC2 Node] 202: Successfully deleted Sensu client: #{@event['client']['name']}"
     when '404'
-      puts "[EC2 Node] 404: Unable to delete #{node}, doesn't exist!"
+      puts "[EC2 Node] 404: Unable to delete #{@event['client']['name']}, doesn't exist!"
     when '500'
-      puts "[EC2 Node] 500: Miscellaneous error when deleting #{node}"
-    else
-      puts "[EC2 Node] #{res}: Completely unsure of what happened!"
-    end
-  end
-
-  def acquire_valid_states
-    if @event['client'].key?('ec2_states')
-      return @event['client']['ec2_states']
+      puts "[EC2 Node] 500: Miscellaneous error when deleting #{@event['client']['name']}"
     else
-      return ['running']
+      puts "[EC2 Node] #{code}: Completely unsure of what happened!"
     end
   end
 end