cfn-guardian 0.6.10 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 355680f054f4e9550f1709780e0b3a5d72c8773c0222bb2307621080a76728b2
-  data.tar.gz: 840685a1d7f15409e809e71944c90e433d428123d204e38c37113444c9508d44
+  metadata.gz: cd1284d70e2862cb14e2aa9f3c97e582ef6776bc54d80f47af8d8bfb561a9386
+  data.tar.gz: 6df131dc0d56bb00041f808617dbd856501d6a131d42ab1311a53d8fb3a3e02d
 SHA512:
-  metadata.gz: 5c8bb52fbeed1bcad2b05776798e5bd21861e08631801e607b6219bb93c5c1e53993355826b9f561aa6099364af09e9e1defa0ddf93f829db325499cc359ba3d
-  data.tar.gz: eff695bda7e0e2e9325f0f7ef332f6e2c7f12a1a32ed8dabfca0a04d7c61a93c5616df8711a7daff6f64bcbbf8b3808b9d01b080334d80c163bcc2712f7431cb
+  metadata.gz: 3f63a7f508b3b2b235138840ddcf9fbdf4512b646711cc7422da94727bde967e49f7da23e7dff94f10134d15c39e661a8da6e356b1fe60c514897f4bfd7176dc
+  data.tar.gz: 9ff94d8f5eeec75778f4b13da7fccd853fd85ff3669966de381c3147a94769b2dc916644069b764d05f560ad9e0f61024c6d776bd60e355cd6a2dbe89d579e04

data/README.md

@@ -41,3 +41,4 @@ CfnGuardian is a AWS monitoring tool with a few capabilities:
 - Redshift Cluster
 - SQS Queues
 - LogGroup Metric Filters
+- ElasticSearch

data/docs/maintenance_mode.md

@@ -8,8 +8,8 @@ Alarms can be provided to the function the following ways
 Alarm names be provided by a space delimited list using the `--alarms` switch.
 
 ```bash
-cfn-guardian disable-alarms --group alarm-1 alarm-2
-cfn-guardian enable-alarms --group alarm-1 alarm-2
+cfn-guardian disable-alarms --alarms alarm-1 alarm-2
+cfn-guardian enable-alarms --alarms alarm-1 alarm-2
 ```
 
 ## Alarm Name Prefix
@@ -60,10 +60,16 @@ Resources:
     StatusCode: 200
 
 # Define the top level key
-MaintenaceGroups:
+MaintenanceGroups:
   
   # Define the group name
   AppUpdate:
+    # Optionally set a schedule for enabling/disabling
+    Schedules:
+      Disable: '30 0 * * ? *'
+      Enable: '00 1 * * ? *'
+      #Optionally specify and set to true to enable logging on lambda
+      Debug: true
     # Define the resource group
     ECSService:
       # define the alarms in the resource group
@@ -82,4 +88,6 @@ MaintenaceGroups:
 ```bash
 cfn-guardian disable-alarms --group AppUpdate
 cfn-guardian enable-alarms --group AppUpdate
-```
+```
+
+Optionally add a Schedule for disabling and enabling alarm actions as shown in the example above to deploy a lambda function that will be invoked by event rules created with the given cron expressions.

data/docs/resources.md

@@ -54,6 +54,7 @@ Resources:
 | Lambda                      | Id               |
 | CloudFrontDistribution      | Id               |
 | SQSQueue                    | Id               |
+| ElasticSearch               | Id, Domain       |
 
 
 ## Custom Resource Groups

data/lib/cfnguardian/compile.rb

@@ -16,6 +16,7 @@ require 'cfnguardian/resources/dynamodb_table'
 require 'cfnguardian/resources/ec2_instance'
 require 'cfnguardian/resources/ecs_cluster'
 require 'cfnguardian/resources/ecs_service'
+require 'cfnguardian/resources/eks_container_insights'
 require 'cfnguardian/resources/elastic_file_system'
 require 'cfnguardian/resources/elasticache_replication_group'
 require 'cfnguardian/resources/elastic_loadbalancer'
@@ -42,9 +43,11 @@ require 'cfnguardian/resources/glue'
 require 'cfnguardian/resources/step_functions'
 require 'cfnguardian/resources/vpn_tunnel'
 require 'cfnguardian/resources/vpn_connection'
+require 'cfnguardian/resources/elastic_search'
 require 'cfnguardian/version'
 require 'cfnguardian/error'
 
+
 module CfnGuardian
   class Compile
     include Logging
@@ -58,7 +61,7 @@ module CfnGuardian
       @composites = config.fetch('Composites',{})
       @templates = config.fetch('Templates',{})
       @topics = config.fetch('Topics',{})
-      @maintenance_groups = config.fetch('MaintenaceGroups', {})
+      @maintenance_groups = config.fetch('MaintenanceGroups', {})
       @event_subscriptions = config.fetch('EventSubscriptions', {})
       
       # Make sure the default topics exist if they aren't supplied in the alarms.yaml
@@ -66,7 +69,6 @@ module CfnGuardian
         @topics[topic] = '' unless @topics.has_key?(topic)
       end
 
-      @maintenance_group_list = @maintenance_groups.keys.map {|group| "#{group}MaintenanceGroup"}
       @resources = []
       @stacks = []
       @checks = []
@@ -113,6 +115,9 @@ module CfnGuardian
       
       @maintenance_groups.each do |maintenance_group,resource_groups|
         resource_groups.each do |group, alarms|
+          if group == 'Schedules' 
+            next
+          end
           alarms.each do |alarm, resources|
             resources.each do |resource|
 
@@ -187,7 +192,7 @@ module CfnGuardian
       resources = split_resources(bucket,path)
       
       main_stack = CfnGuardian::Stacks::Main.new()
-      main_stack.build_template(@stacks,@checks,@topics,@maintenance_group_list,@ssm_parameters)
+      main_stack.build_template(@stacks,@checks,@topics,@maintenance_groups,@ssm_parameters)
       valid = main_stack.template.validate
       FileUtils.mkdir_p 'out'
       File.write("out/guardian.compiled.yaml", JSON.parse(valid.to_json).to_yaml)

data/lib/cfnguardian/config/defaults.yaml

@@ -28,12 +28,20 @@ Resources:
   ECSCluster:
   - Id: Default
   ECSService:
+  - Id: Default
+    Cluster: Default
+  EKSContainerInsightsCluster:
+  - Id: Default
+  EKSContainerInsightsNamespace:
   - Id: Default
     Cluster: Default
   ElasticFileSystem:
   - Id: Default
   ElasticLoadBalancer:
   - Id: Default
+  ElasticSearch:
+  - Id: Default
+    Domain: Default
   ElastiCacheReplicationGroup:
   - Id: Default
   Http:

data/lib/cfnguardian/models/alarm.rb

@@ -203,7 +203,28 @@ module CfnGuardian
         }
       end
     end
+
+    class EKSContainerInsightsClusterAlarm < BaseAlarm
+      def initialize(resource)
+        super(resource)
+        @group = 'EKSContainerInsightsCluster'
+        @namespace = 'ContainerInsights'
+        @dimensions = { ClusterName: resource['Id'] }
+      end
+    end
     
+    class EKSContainerInsightsNamespaceAlarm < BaseAlarm
+      def initialize(resource)
+        super(resource)
+        @group = 'EKSContainerInsightsNamespace'
+        @namespace = 'ContainerInsights'
+        @dimensions = { 
+          ClusterName: resource['Cluster'],
+          Namespace: resource['Id']
+        }
+      end
+    end
+
     class ElastiCacheReplicationGroupAlarm < BaseAlarm
       def initialize(resource)
         super(resource)
@@ -212,6 +233,24 @@ module CfnGuardian
         @dimensions = { CacheClusterId: resource['Id'] }
       end
     end
+
+    class ElasticSearchAlarm < BaseAlarm
+      def initialize(resource)
+        super(resource)
+        @group = 'ElasticSearch'
+        @namespace = 'AWS/ES'
+        @dimensions = { 
+          DomainName: resource['Domain'], 
+          ClientId: resource['Id'] 
+        }
+        @comparison_operator = 'GreaterThanOrEqualToThreshold'
+        @threshold = 1
+        @evaluation_periods = 5
+        @treat_missing_data = 'breaching'
+        @period = 60
+        @data_points_to_alarm = 1
+      end
+    end
     
     class ElasticLoadBalancerAlarm < BaseAlarm
       def initialize(resource)

data/lib/cfnguardian/models/check.rb

@@ -205,5 +205,16 @@ module CfnGuardian
       end
     end
 
+    class MaintenanceGroupCheck < BaseCheck
+      def initialize(resource)
+        super(resource)
+        @name = 'MaintenanceGroupCheck'
+        @package = 'maintenance-group-check'
+        @handler = 'handler.maintenance_group_check'
+        @version = '5b795e6509068d1767e4be80f2e6868cbeb3b425'
+        @runtime = 'python3.7'
+      end 
+    end
+
   end
 end

data/lib/cfnguardian/resources/eks_container_insights.rb

@@ -0,0 +1,99 @@
+module CfnGuardian::Resource
+  class EKSContainerInsightsCluster < Base
+
+    def default_alarms
+
+      alarm = CfnGuardian::Models::EKSContainerInsightsClusterAlarm.new(@resource)
+      alarm.name = 'NodeCpuUtilisationBase'
+      alarm.metric_name = 'node_cpu_utilization'
+      alarm.comparison_operator = 'GreaterThanThreshold'
+      alarm.statistic = 'Maximum'
+      alarm.threshold = 75
+      alarm.evaluation_periods = 60
+      alarm.alarm_action = 'Warning'
+      @alarms.push(alarm)
+
+      alarm = CfnGuardian::Models::EKSContainerInsightsClusterAlarm.new(@resource)
+      alarm.name = 'NodeCpuUtilisationSpike'
+      alarm.metric_name = 'node_cpu_utilization'
+      alarm.comparison_operator = 'GreaterThanThreshold'
+      alarm.statistic = 'Maximum'
+      alarm.threshold = 95
+      alarm.evaluation_periods = 5
+      @alarms.push(alarm)
+
+      alarm = CfnGuardian::Models::EKSContainerInsightsClusterAlarm.new(@resource)
+      alarm.name = 'NodeFileSystemUtilisationCrit'
+      alarm.metric_name = 'node_filesystem_utilization'
+      alarm.comparison_operator = 'GreaterThanThreshold'
+      alarm.statistic = 'Maximum'
+      alarm.threshold = 90
+      alarm.evaluation_periods = 1
+      @alarms.push(alarm)
+
+      alarm = CfnGuardian::Models::EKSContainerInsightsClusterAlarm.new(@resource)
+      alarm.name = 'NodeFileSystemUtilisationWarning'
+      alarm.metric_name = 'node_filesystem_utilization'
+      alarm.comparison_operator = 'GreaterThanThreshold'
+      alarm.statistic = 'Maximum'
+      alarm.threshold = 75
+      alarm.evaluation_periods = 1
+      alarm.alarm_action = 'Warning'
+      @alarms.push(alarm)
+
+      alarm = CfnGuardian::Models::EKSContainerInsightsClusterAlarm.new(@resource)
+      alarm.name = 'NodeMemoryUtilisationBase'
+      alarm.metric_name = 'node_memory_utilization'
+      alarm.comparison_operator = 'GreaterThanThreshold'
+      alarm.statistic = 'Maximum'
+      alarm.threshold = 80
+      alarm.evaluation_periods = 60
+      alarm.alarm_action = 'Warning'
+      @alarms.push(alarm)
+
+      alarm = CfnGuardian::Models::EKSContainerInsightsClusterAlarm.new(@resource)
+      alarm.name = 'NodeMemoryUtilisationSpike'
+      alarm.metric_name = 'node_memory_utilization'
+      alarm.comparison_operator = 'GreaterThanThreshold'
+      alarm.statistic = 'Maximum'
+      alarm.threshold = 90
+      alarm.evaluation_periods = 5
+      @alarms.push(alarm)
+
+      alarm = CfnGuardian::Models::EKSContainerInsightsClusterAlarm.new(@resource)
+      alarm.name = 'ClusterFailedNodeCount'
+      alarm.metric_name = 'cluster_failed_node_count'
+      alarm.comparison_operator = 'GreaterThanThreshold'
+      alarm.statistic = 'Minimum'
+      alarm.threshold = 0
+      alarm.evaluation_periods = 1
+      @alarms.push(alarm)
+
+    end
+  end
+
+  class EKSContainerInsightsNamespace < Base
+
+    def default_alarms
+
+      alarm = CfnGuardian::Models::EKSContainerInsightsNamespaceAlarm.new(@resource)
+      alarm.name = 'PodCpuUtilisation'
+      alarm.metric_name = 'pod_cpu_utilization'
+      alarm.comparison_operator = 'GreaterThanThreshold'
+      alarm.statistic = 'Maximum'
+      alarm.threshold = 90
+      alarm.evaluation_periods = 5
+      @alarms.push(alarm)
+
+      alarm = CfnGuardian::Models::EKSContainerInsightsNamespaceAlarm.new(@resource)
+      alarm.name = 'PodMemoryUtilisation'
+      alarm.metric_name = 'pod_memory_utilization'
+      alarm.comparison_operator = 'GreaterThanThreshold'
+      alarm.statistic = 'Maximum'
+      alarm.threshold = 90
+      alarm.evaluation_periods = 5
+      @alarms.push(alarm)
+
+    end
+  end
+end 

data/lib/cfnguardian/resources/elastic_search.rb

@@ -0,0 +1,136 @@
+module CfnGuardian::Resource
+    class ElasticSearch < Base
+
+      def default_alarms    
+        alarm = CfnGuardian::Models::ElasticSearchAlarm.new(@resource)
+        alarm.name = 'NodeCount'
+        alarm.metric_name = 'Nodes'
+        alarm.threshold = 3
+        alarm.evaluation_periods = 1440 # 24 hours
+        alarm.datapoints_to_alarm = 1
+        alarm.comparison_operator = 'LessThanOrEqualToThreshold'
+        alarm.alarm_action = 'Critical'
+        alarm.enabled = false
+        @alarms.push(alarm)
+
+        alarm = CfnGuardian::Models::ElasticSearchAlarm.new(@resource)
+        alarm.name = 'JVMMemoryPressureWarning'
+        alarm.metric_name = 'JVMMemoryPressure'
+        alarm.threshold = 72
+        alarm.evaluation_periods = 5
+        alarm.datapoints_to_alarm = 3
+        alarm.alarm_action = 'Warning'
+        @alarms.push(alarm)
+
+        alarm = CfnGuardian::Models::ElasticSearchAlarm.new(@resource)
+        alarm.name = 'JVMMemoryPressureCrit'
+        alarm.metric_name = 'JVMMemoryPressure'
+        alarm.threshold = 92
+        alarm.evaluation_periods = 5
+        alarm.alarm_action = 'Critical'
+        @alarms.push(alarm)
+
+        alarm = CfnGuardian::Models::ElasticSearchAlarm.new(@resource)
+        alarm.name = 'ClusterIndexWritesBlocked'
+        alarm.metric_name = 'ClusterIndexWritesBlocked'
+        alarm.threshold = 1
+        alarm.evaluation_periods = 5
+        alarm.alarm_action = 'Critical'
+        @alarms.push(alarm)  
+        
+        alarm = CfnGuardian::Models::ElasticSearchAlarm.new(@resource)
+        alarm.name = 'MasterNodeCPUUtilisationWarning'
+        alarm.metric_name = 'MasterCPUUtilization'
+        alarm.threshold = 75
+        alarm.evaluation_periods = 60
+        alarm.alarm_action = 'Warning'
+        @alarms.push(alarm)  
+
+        alarm = CfnGuardian::Models::ElasticSearchAlarm.new(@resource)
+        alarm.name = 'MasterNodeCPUUtilisationCrit'
+        alarm.metric_name = 'MasterCPUUtilization'
+        alarm.threshold = 95
+        alarm.evaluation_periods = 10
+        alarm.alarm_action = 'Critical'
+        @alarms.push(alarm)  
+
+        alarm = CfnGuardian::Models::ElasticSearchAlarm.new(@resource)
+        alarm.name = 'FreeStorageSpaceWarning'
+        alarm.metric_name = 'FreeStorageSpace'
+        alarm.threshold = 50000
+        alarm.evaluation_periods = 1
+        alarm.alarm_action = 'Warning'
+        alarm.statistic = 'Minimum'
+        alarm.comparison_operator = 'LessThanOrEqualToThreshold'
+        @alarms.push(alarm)  
+
+        alarm = CfnGuardian::Models::ElasticSearchAlarm.new(@resource)
+        alarm.name = 'FreeStorageSpaceCrit'
+        alarm.metric_name = 'FreeStorageSpace'
+        alarm.threshold = 25000
+        alarm.evaluation_periods = 1
+        alarm.alarm_action = 'Critical'
+        alarm.statistic = 'Minimum'
+        alarm.comparison_operator = 'LessThanOrEqualToThreshold'
+        @alarms.push(alarm)  
+       
+        alarm = CfnGuardian::Models::ElasticSearchAlarm.new(@resource)
+        alarm.name = 'CPUUtilizationWarning'
+        alarm.metric_name = 'CPUUtilization'
+        alarm.threshold = 75
+        alarm.evaluation_periods = 15
+        alarm.datapoints_to_alarm = 3
+        alarm.alarm_action = 'Warning'
+        alarm.statistic = 'Average'
+        @alarms.push(alarm)  
+
+        alarm = CfnGuardian::Models::ElasticSearchAlarm.new(@resource)
+        alarm.name = 'CPUUtilizationCrit'
+        alarm.metric_name = 'CPUUtilization'
+        alarm.threshold = 95
+        alarm.evaluation_periods = 5
+        alarm.datapoints_to_alarm = 3
+        alarm.alarm_action = 'Critical'
+        alarm.statistic = 'Average'
+        @alarms.push(alarm)  
+
+        alarm = CfnGuardian::Models::ElasticSearchAlarm.new(@resource)
+        alarm.name = 'KMSKeyError'
+        alarm.metric_name = 'KMSKeyError'
+        alarm.threshold = 1
+        alarm.evaluation_periods = 1
+        alarm.alarm_action = 'Warning'
+        alarm.statistic = 'Minimum'
+        @alarms.push(alarm)  
+
+        alarm = CfnGuardian::Models::ElasticSearchAlarm.new(@resource)
+        alarm.name = 'KMSKeyInaccessible'
+        alarm.metric_name = 'KMSKeyInaccessible'
+        alarm.threshold = 1
+        alarm.evaluation_periods = 1
+        alarm.alarm_action = 'Critical'
+        alarm.statistic = 'Minimum'
+        alarm.enabled = false
+        @alarms.push(alarm)  
+
+        alarm = CfnGuardian::Models::ElasticSearchAlarm.new(@resource)
+        alarm.name = 'ClusterStatusRed'
+        alarm.metric_name = 'ClusterStatus.red'
+        alarm.threshold = 1
+        alarm.evaluation_periods = 1
+        alarm.alarm_action = 'Critical'
+        alarm.statistic = 'Minimum'
+        @alarms.push(alarm)  
+
+        alarm = CfnGuardian::Models::ElasticSearchAlarm.new(@resource)
+        alarm.name = 'ClusterStatusYellow'
+        alarm.metric_name = 'ClusterStatus.yellow'
+        alarm.threshold = 1
+        alarm.evaluation_periods = 1
+        alarm.alarm_action = 'Warning'
+        alarm.statistic = 'Minimum'
+        @alarms.push(alarm)  
+
+    end
+  end
+end

data/lib/cfnguardian/stacks/main.rb

@@ -4,6 +4,7 @@ module CfnGuardian
   module Stacks
     class Main
       include CfnDsl::CloudFormation
+      include Logging
       
       attr_reader :parameters, :template
       
@@ -22,12 +23,10 @@ module CfnGuardian
           parameter.Default sns
           parameters[name] = Ref(name)
         end
-        
-        maintenance_groups.each do |group|
-          topic = @template.SNS_Topic(group)
-          topic.TopicName group
-          topic.Tags([{ Key: 'Environment', Value: 'guardian' }])
-          parameters[group] = Ref(group)
+
+        if maintenance_groups.any?
+          add_lambda(CfnGuardian::Models::MaintenanceGroupCheck.new(maintenance_groups))
+          maintenance_groups.each {|group,config| add_maintenance_group(group,config,parameters)}
         end
         
         add_iam_role(ssm_parameters)
@@ -73,6 +72,17 @@ module CfnGuardian
             }]
           }
         }
+        policies << {
+          PolicyName: 'maintenance-group-actions',
+          PolicyDocument: {
+            Version: '2012-10-17',
+            Statement: [{
+              Effect: 'Allow',
+              Action: [ 'cloudwatch:DescribeAlarms', 'cloudwatch:DisableAlarmActions', 'cloudwatch:EnableAlarmActions', 'cloudwatch:SetAlarmState' ],
+              Resource: FnSub("arn:aws:cloudwatch:${AWS::Region}:${AWS::AccountId}:alarm:*")
+            }]
+          }
+        }
         if ssm_parameters.any?
           policies << {
             PolicyName: 'ssm-parameters',
@@ -165,7 +175,37 @@ module CfnGuardian
           end
         end
       end
-      
+
+      def add_maintenance_group(group,config,parameters)
+        group_name = "#{group}MaintenanceGroup"
+        schedules = config.fetch('Schedules', {})
+        logging = config.dig('Schedules', 'Debug').to_s
+
+        topic = @template.SNS_Topic(group_name)
+        topic.TopicName group_name
+        topic.Tags([{ Key: 'Environment', Value: 'guardian' }])
+        parameters[group_name] = Ref(group_name)
+
+        if schedules.any?
+          event = @template.Events_Rule("#{group_name}EnableEvent")
+          event.Name "#{group_name}EnableEvent"
+          event.ScheduleExpression "cron(#{schedules['Enable']})"
+          event.Targets([{ 
+            Arn: FnGetAtt('MaintenanceGroupCheckFunction', 'Arn'), 
+            Id: "#{group_name}EnableTarget", 
+            Input: {action:"enable_alarms", maintenance_group: group_name, logging: logging}.to_json
+          }])
+
+          event = @template.Events_Rule("#{group_name}DisableEvent")
+          event.Name "#{group_name}DisableEvent"
+          event.ScheduleExpression "cron(#{schedules['Disable']})"            
+          event.Targets([{ 
+            Arn: FnGetAtt('MaintenanceGroupCheckFunction', 'Arn'), 
+            Id: "#{group_name}DisableTarget", 
+            Input: {action:"disable_alarms", maintenance_group: group_name, logging: logging}.to_json
+          }])
+        end
+      end
     end
   end
 end

data/lib/cfnguardian/version.rb

@@ -1,4 +1,4 @@
 module CfnGuardian
-  VERSION = "0.6.10"
+  VERSION = "0.7.0"
   CHANGE_SET_VERSION = VERSION.gsub('.', '-').freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: cfn-guardian
 version: !ruby/object:Gem::Version
-  version: 0.6.10
+  version: 0.7.0
 platform: ruby
 authors:
 - Guslington
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-06-25 00:00:00.000000000 Z
+date: 2021-08-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -285,8 +285,10 @@ files:
 - lib/cfnguardian/resources/ec2_instance.rb
 - lib/cfnguardian/resources/ecs_cluster.rb
 - lib/cfnguardian/resources/ecs_service.rb
+- lib/cfnguardian/resources/eks_container_insights.rb
 - lib/cfnguardian/resources/elastic_file_system.rb
 - lib/cfnguardian/resources/elastic_loadbalancer.rb
+- lib/cfnguardian/resources/elastic_search.rb
 - lib/cfnguardian/resources/elasticache_replication_group.rb
 - lib/cfnguardian/resources/glue.rb
 - lib/cfnguardian/resources/http.rb