cfn_monitor 0.1.0

data/lib/config/templates.yml

@@ -0,0 +1,428 @@
+templates:
+  HttpCheck:
+    EndpointAvailable:
+      AlarmActions: crit
+      Namespace: HttpCheck
+      ComparisonOperator: LessThanThreshold
+      Dimensions: [ { Name: 'Endpoint', Value: { 'Fn::Sub': ['${endpoint}', {'env': { Ref: 'EnvironmentName' } } ] } } ]
+      Statistic: Maximum
+      Threshold: 1
+      Period: 60
+      EvaluationPeriods: 2
+      MetricName: Available
+    EndpointTimeTaken:
+      AlarmActions: crit
+      Namespace: HttpCheck
+      ComparisonOperator: GreaterThanThreshold
+      Dimensions: [ { Name: 'Endpoint', Value: { 'Fn::Sub': ['${endpoint}', {'env': { Ref: 'EnvironmentName' } } ] } } ]
+      Statistic: Minimum
+      Threshold: 1000
+      Period: 300
+      EvaluationPeriods: 1
+      MetricName: TimeTaken
+    EndpointStatusCodeMatch:
+      AlarmActions: crit
+      Namespace: HttpCheck
+      ComparisonOperator: LessThanThreshold
+      Dimensions: [ { Name: 'Endpoint', Value: { 'Fn::Sub': ['${endpoint}', {'env': { Ref: 'EnvironmentName' } } ] } } ]
+      Statistic: Maximum
+      Threshold: 1
+      Period: 60
+      EvaluationPeriods: 2
+      MetricName: StatusCodeMatch
+  HttpCheckBody:
+    EndpointResponseBodyRegexMatch:
+      AlarmActions: crit
+      Namespace: HttpCheck
+      ComparisonOperator: LessThanThreshold
+      Dimensions: [ { Name: 'Endpoint', Value: { 'Fn::Sub': ['${endpoint}', {'env': { Ref: 'EnvironmentName' } } ] } } ]
+      Statistic: Maximum
+      Threshold: 1
+      Period: 60
+      EvaluationPeriods: 2
+      MetricName: ResponseBodyRegexMatch
+  Services:
+    FailedService:
+      AlarmActions: warn
+      Namespace: ServiceChecks
+      ComparisonOperator: GreaterThanThreshold
+      Dimensions: [ { Name: 'Service', Value: '${resource}' }, { Name: 'Environment', Value: { Ref: 'EnvironmentName' } } ]
+      Statistic: Maximum
+      Threshold: 0
+      Period: 60
+      EvaluationPeriods: 2
+      MetricName: Failed_Service
+      AlarmDescription: { 'Fn::Join': [ ' ', [ Ref: 'MonitoredStack', { 'Fn::Sub': ['${resource}', {'env': { Ref: 'EnvironmentName' } } ] }, '${alarmName}' ] ] }
+      TreatMissingData: notBreaching
+    # FailedSSMAgent:
+    #   AlarmActions: warn
+    #   Namespace: ServiceChecks
+    #   ComparisonOperator: GreaterThanThreshold
+    #   Dimensions: [ { Name: 'Environment', Value: { Ref: 'EnvironmentName' } } ]
+    #   Statistic: Maximum
+    #   Threshold: 0
+    #   Period: 60
+    #   EvaluationPeriods: 2
+    #   MetricName: Failed_SSM_Agent
+    #   AlarmDescription: { 'Fn::Join': [ ' ', [ Ref: 'MonitoredStack', { 'Fn::Sub': ['${resource}', {'env': { Ref: 'EnvironmentName' } } ] }, '${alarmName}' ] ] }
+    #   TreatMissingData: notBreaching
+  Nrpe:
+    Warn:
+      AlarmActions: warn
+      Namespace: NRPE
+      ComparisonOperator: GreaterThanThreshold
+      Dimensions: [ { Name: 'Host', Value: { 'Fn::Join': [ '-', [ { 'Fn::Sub': ['${env}', {'env': { Ref: 'EnvironmentName' } } ] }, '${resource}' ] ] } } ]
+      Statistic: Maximum
+      Threshold: 0
+      Period: 60
+      EvaluationPeriods: 2
+      MetricName: ${cmd}
+      TreatMissingData: breaching
+      AlarmDescription: { 'Fn::Join': [ ' ', [ Ref: 'MonitoredStack', { 'Fn::Sub': ['${resource}', {'env': { Ref: 'EnvironmentName' } } ] }, '${templateName}', '${cmd}', '${alarmName}' ] ] }
+    Crit:
+      AlarmActions: crit
+      Namespace: NRPE
+      ComparisonOperator: GreaterThanThreshold
+      Dimensions: [ { Name: 'Host', Value: { 'Fn::Join': [ '-', [ { 'Fn::Sub': ['${env}', {'env': { Ref: 'EnvironmentName' } } ] }, '${resource}' ] ] } } ]
+      Statistic: Maximum
+      Threshold: 1
+      Period: 60
+      EvaluationPeriods: 2
+      MetricName: ${cmd}
+      AlarmDescription: { 'Fn::Join': [ ' ', [ Ref: 'MonitoredStack', { 'Fn::Sub': ['${resource}', {'env': { Ref: 'EnvironmentName' } } ] }, '${templateName}', '${cmd}', '${alarmName}' ] ] }
+  LambdaMetrics: # Lambda Functions deployed with a Function Name
+    Errors:
+      AlarmActions: crit
+      Namespace: AWS/Lambda
+      ComparisonOperator: GreaterThanThreshold
+      Dimensions: [ { Name: 'FunctionName', Value: { 'Fn::Sub': ['${metric}', {'env': { Ref: 'EnvironmentName' } } ] } } ]
+      Statistic: Average
+      Threshold: 0.5
+      Period: 60
+      EvaluationPeriods: 5
+      MetricName: Errors
+      TreatMissingData: notBreaching
+  ElasticLoadBalancer: # AWS::ElasticLoadBalancing::LoadBalancer
+    HealthyHosts:
+      AlarmActions: crit
+      Namespace: AWS/ELB
+      MetricName: HealthyHostCount
+      ComparisonOperator: LessThanThreshold
+      DimensionsName: LoadBalancerName
+      Statistic: Minimum
+      Threshold: 2
+      Threshold.development: 1
+      EvaluationPeriods: 1
+      TreatMissingData: notBreaching
+    UnHealthyHosts:
+      AlarmActions: warn
+      Namespace: AWS/ELB
+      MetricName: UnHealthyHostCount
+      ComparisonOperator: GreaterThanThreshold
+      DimensionsName: LoadBalancerName
+      Statistic: Maximum
+      Threshold: 0
+      EvaluationPeriods: 10
+      TreatMissingData: notBreaching
+  ApplicationELBTargetGroup: # AWS::ElasticLoadBalancingV2::TargetGroup / AWS::ElasticLoadBalancingV2::LoadBalancer where type: application
+    HealthyHosts:
+      AlarmActions: crit
+      Namespace: AWS/ApplicationELB
+      MetricName: HealthyHostCount
+      ComparisonOperator: LessThanThreshold
+      DimensionsName: TargetGroup/LoadBalancer
+      Statistic: Minimum
+      Threshold: 2
+      Threshold.development: 1
+      EvaluationPeriods: 1
+      TreatMissingData: breaching
+    UnHealthyHosts:
+      AlarmActions: warn
+      Namespace: AWS/ApplicationELB
+      MetricName: UnHealthyHostCount
+      ComparisonOperator: GreaterThanThreshold
+      DimensionsName: TargetGroup/LoadBalancer
+      Statistic: Maximum
+      Threshold: 0
+      EvaluationPeriods: 10
+      TreatMissingData: notBreaching
+    TargetResponseTime:
+      AlarmActions: crit
+      Namespace: AWS/ApplicationELB
+      MetricName: TargetResponseTime
+      ComparisonOperator: GreaterThanThreshold
+      DimensionsName: TargetGroup/LoadBalancer
+      Statistic: Average
+      Threshold: 5
+      EvaluationPeriods: 5
+      TreatMissingData: notBreaching
+  NetworkELBTargetGroup: # AWS::ElasticLoadBalancingV2::TargetGroup / AWS::ElasticLoadBalancingV2::LoadBalancer where type: network
+    HealthyHosts:
+      AlarmActions: crit
+      Namespace: AWS/NetworkELB
+      MetricName: HealthyHostCount
+      ComparisonOperator: LessThanThreshold
+      DimensionsName: TargetGroup/LoadBalancer
+      Statistic: Minimum
+      Threshold: 2
+      Threshold.development: 1
+      EvaluationPeriods: 1
+      TreatMissingData: breaching
+    UnHealthyHosts:
+      AlarmActions: warn
+      Namespace: AWS/NetworkELB
+      MetricName: UnHealthyHostCount
+      ComparisonOperator: GreaterThanThreshold
+      DimensionsName: TargetGroup/LoadBalancer
+      Statistic: Maximum
+      Threshold: 0
+      EvaluationPeriods: 10
+      TreatMissingData: notBreaching
+    TargetResponseTime:
+      AlarmActions: crit
+      Namespace: AWS/NetworkELB
+      MetricName: TargetResponseTime
+      ComparisonOperator: GreaterThanThreshold
+      DimensionsName: TargetGroup/LoadBalancer
+      Statistic: Average
+      Threshold: 5
+      EvaluationPeriods: 5
+      TreatMissingData: notBreaching
+  AutoScalingGroup: # AWS::AutoScaling::AutoScalingGroup
+    CPUUtilizationHighBase:
+      AlarmActions: warn
+      Namespace: AWS/EC2
+      MetricName: CPUUtilization
+      ComparisonOperator: GreaterThanThreshold
+      DimensionsName: AutoScalingGroupName
+      Statistic: Minimum
+      Threshold: 75
+      EvaluationPeriods: 60
+      TreatMissingData: notBreaching
+    StatusCheckFailed:
+      AlarmActions: warn
+      Namespace: AWS/EC2
+      MetricName: StatusCheckFailed
+      ComparisonOperator: GreaterThanThreshold
+      DimensionsName: AutoScalingGroupName
+      Statistic: Maximum
+      Threshold: 0
+      EvaluationPeriods: 10
+  Ec2Instance: # AWS::EC2::Instance
+    CPUUtilizationHigh:
+      AlarmActions: crit
+      Namespace: AWS/EC2
+      MetricName: CPUUtilization
+      ComparisonOperator: GreaterThanThreshold
+      DimensionsName: InstanceId
+      Statistic: Minimum
+      Threshold: 90
+      EvaluationPeriods: 10
+    StatusCheckFailed:
+      AlarmActions: crit
+      Namespace: AWS/EC2
+      MetricName: StatusCheckFailed
+      ComparisonOperator: GreaterThanThreshold
+      DimensionsName: InstanceId
+      Statistic: Maximum
+      Threshold: 0
+      EvaluationPeriods: 10
+  RDSInstance: # AWS::RDS::DBInstance
+    FreeStorageSpaceCrit:
+      AlarmActions: crit
+      Namespace: AWS/RDS
+      MetricName: FreeStorageSpace
+      ComparisonOperator: LessThanThreshold
+      DimensionsName: DBInstanceIdentifier
+      Statistic: Minimum
+      Threshold: 50000000000
+      Threshold.development: 10000000000
+      EvaluationPeriods: 1
+    FreeStorageSpaceTask:
+      AlarmActions: task
+      Namespace: AWS/RDS
+      MetricName: FreeStorageSpace
+      ComparisonOperator: LessThanThreshold
+      DimensionsName: DBInstanceIdentifier
+      Statistic: Minimum
+      Threshold: 100000000000
+      Threshold.development: 20000000000
+      EvaluationPeriods: 1
+    CPUUtilizationHighSpike:
+      AlarmActions: crit
+      Namespace: AWS/RDS
+      MetricName: CPUUtilization
+      ComparisonOperator: GreaterThanThreshold
+      DimensionsName: DBInstanceIdentifier
+      Statistic: Minimum
+      Threshold: 95
+      EvaluationPeriods: 10
+    CPUUtilizationHighBase:
+      AlarmActions: warn
+      Namespace: AWS/RDS
+      MetricName: CPUUtilization
+      ComparisonOperator: GreaterThanThreshold
+      DimensionsName: DBInstanceIdentifier
+      Statistic: Minimum
+      Threshold: 75
+      EvaluationPeriods: 60
+  DBCluster: # AWS::RDS::DBCluster
+    CPUUtilizationHighSpike:
+      AlarmActions: crit
+      Namespace: AWS/RDS
+      MetricName: CPUUtilization
+      ComparisonOperator: GreaterThanThreshold
+      DimensionsName: DBClusterIdentifier
+      Statistic: Minimum
+      Threshold: 95
+      EvaluationPeriods: 10
+    CPUUtilizationHighBase:
+      AlarmActions: warn
+      Namespace: AWS/RDS
+      MetricName: CPUUtilization
+      ComparisonOperator: GreaterThanThreshold
+      DimensionsName: DBClusterIdentifier
+      Statistic: Minimum
+      Threshold: 75
+      EvaluationPeriods: 60
+  RedshiftCluster: # AWS::Redshift::Cluster
+    CPUUtilizationHighSpike:
+      AlarmActions: crit
+      Namespace: AWS/Redshift
+      MetricName: CPUUtilization
+      ComparisonOperator: GreaterThanThreshold
+      DimensionsName: ClusterIdentifier
+      Statistic: Minimum
+      Threshold: 95
+      EvaluationPeriods: 10
+    CPUUtilizationHighBase:
+      AlarmActions: warn
+      Namespace: AWS/Redshift
+      MetricName: CPUUtilization
+      ComparisonOperator: GreaterThanThreshold
+      DimensionsName: ClusterIdentifier
+      Statistic: Minimum
+      Threshold: 75
+      EvaluationPeriods: 60
+    UnHealthyCluster:
+      AlarmActions: crit
+      Namespace: AWS/Redshift
+      MetricName: HealthStatus
+      ComparisonOperator: GreaterThanThreshold
+      DimensionsName: ClusterIdentifier
+      Statistic: Maximum
+      Threshold: 0
+      EvaluationPeriods: 10
+      TreatMissingData: notBreaching
+  ECSCluster: # AWS::ECS::Cluster
+    MemoryUtilizationWarn:
+      AlarmActions: warn
+      Namespace: AWS/ECS
+      MetricName: MemoryUtilization
+      ComparisonOperator: GreaterThanThreshold
+      DimensionsName: ClusterName
+      Statistic: Maximum
+      Threshold: 75
+      EvaluationPeriods: 10
+    MemoryUtilizationCrit:
+      AlarmActions: crit
+      Namespace: AWS/ECS
+      MetricName: MemoryUtilization
+      ComparisonOperator: GreaterThanThreshold
+      DimensionsName: ClusterName
+      Statistic: Maximum
+      Threshold: 95
+      EvaluationPeriods: 10
+    CPUUtilizationHighBase:
+      AlarmActions: warn
+      Namespace: AWS/ECS
+      MetricName: CPUUtilization
+      ComparisonOperator: GreaterThanThreshold
+      DimensionsName: ClusterName
+      Statistic: Minimum
+      Threshold: 75
+      EvaluationPeriods: 10
+  ECSService: #AWS::ECS::Service
+    UnHealthyTask:
+      AlarmActions: crit
+      MetricName: MemoryUtilization
+      Namespace: AWS/ECS
+      Statistic: SampleCount
+      DimensionsName: ServiceName/ClusterName
+      Period: 60
+      EvaluationPeriods: 10
+      DatapointsToAlarm: 8
+      Threshold: 0.0
+      ComparisonOperator: LessThanOrEqualToThreshold
+      TreatMissingData: breaching
+  ECSTasks:
+    UnHealthyTaskWarn:
+      AlarmActions: warn
+      MetricName: MemoryUtilization
+      Namespace: AWS/ECS
+      Statistic: SampleCount
+      DimensionsName: ServiceName/ClusterName
+      Period: 60
+      EvaluationPeriods: 10
+      DatapointsToAlarm: 10
+      Threshold: 1.0
+      ComparisonOperator: LessThanOrEqualToThreshold
+      TreatMissingData: breaching
+  ElastiCacheReplicationGroup: # AWS::ElastiCache::ReplicationGroup
+    FreeableMemoryWarn:
+      AlarmActions: warn
+      Namespace: AWS/ElastiCache
+      MetricName: FreeableMemory
+      ComparisonOperator: LessThanThreshold
+      DimensionsName: CacheClusterId
+      Statistic: Maximum
+      Threshold: 100000000
+      EvaluationPeriods: 10
+    CPUUtilizationHighBase:
+      AlarmActions: warn
+      Namespace: AWS/ElastiCache
+      MetricName: CPUUtilization
+      ComparisonOperator: GreaterThanThreshold
+      DimensionsName: CacheClusterId
+      Statistic: Minimum
+      Threshold: 75
+      EvaluationPeriods: 10
+  ElasticFileSystem: # AWS::EFS::FileSystem
+    PercentIOLimitHigh:
+      AlarmActions: crit
+      Namespace: AWS/EFS
+      MetricName: PercentIOLimit
+      ComparisonOperator: GreaterThanThreshold
+      DimensionsName: FileSystemId
+      Statistic: Minimum
+      Threshold: 90
+      EvaluationPeriods: 5
+  ApiGateway: #AWS:ApiGateway
+    ApiEndpoint5xx:
+      AlarmActions: crit
+      Namespace: AWS/ApiGateway
+      MetricName: 5XXError
+      ComparisonOperator: GreaterThanThreshold
+      DimensionsName: ApiName
+      Statistic: Sum
+      Threshold: 5
+      EvaluationPdyneriods: 2
+  DynamoDBTable: #AWS::DynamoDB::Table
+    DynamoDBReadUsage:
+      AlarmActions: warn
+      Namespace: AWS/DynamoDB
+      MetricName: ConsumedReadCapacityUnits
+      ComparisonOperator: GreaterThanThreshold
+      DimensionsName: TableName
+      Statistic: Sum
+      Threshold: 80
+      EvaluationPeriods: 2
+    DynamoDBWriteUsage:
+      AlarmActions: warn
+      Namespace: AWS/DynamoDB
+      MetricName: ConsumedWriteCapacityUnits
+      ComparisonOperator: GreaterThanThreshold
+      DimensionsName: TableName
+      Statistic: Sum
+      Threshold: 80
+      EvaluationPeriods: 2

data/lib/ext/alarms.rb

@@ -0,0 +1,33 @@
+def replace_vars(hash,find,replace)
+  if hash.is_a?(Hash)
+    hash.each do |k, v|
+      replace_vars(v,find,replace)
+    end
+  elsif hash.is_a?(Array)
+    hash.each do |e|
+      replace_vars(e,find,replace)
+    end
+  elsif hash.is_a?(String) && hash == find
+    hash.replace replace
+  end
+  hash
+end
+
+def create_param_mappings(params,template_envs,alarmHash)
+  mappings = {}
+  params.each do |key,value|
+    if !key.include? '.'
+      if [String, Integer, Float, Fixnum, TrueClass].member?(value.class)
+        mappings[key] = {}
+        template_envs.each do |env|
+          if !params["#{key}.#{env}"].nil? && [String, Integer, Float, Fixnum, TrueClass].member?(params["#{key}.#{env}"].class)
+            mappings[key][env] = params["#{key}.#{env}"]
+          else
+            mappings[key][env] = value
+          end
+        end
+      end
+    end
+  end
+  Mapping("#{alarmHash}", mappings)
+end

data/lib/lambda/getEnvironmentName.py

@@ -0,0 +1,39 @@
+import json
+import cfnresponse
+import boto3
+from botocore.exceptions import ClientError
+import time
+
+def getEnvironmentName(client,stackName):
+    time.sleep(5)
+    max_retries = 5
+    sleep_time = 1
+    for i in range(max_retries):
+        try:
+            response = client.describe_stacks(StackName=stackName)
+        except ClientError as e:
+            print e
+            time.sleep(sleep_time)
+            sleep_time += sleep_time
+            continue
+        else:
+            break
+    else:
+        return stackName
+    if 'Parameters' in response['Stacks'][0]:
+        for r in response['Stacks'][0]['Parameters']:
+            if r['ParameterKey'] == 'EnvironmentName':
+                return r['ParameterValue']
+        return stackName
+    else:
+        return stackName
+
+
+def handler(event, context):
+    stackName = event['ResourceProperties']['StackName']
+    region = event['ResourceProperties']['Region']
+    client = boto3.client('cloudformation',region_name=region)
+
+    responseData = {}
+    responseData['EnvironmentName'] = getEnvironmentName(client,stackName)
+    cfnresponse.send(event, context, cfnresponse.SUCCESS, responseData, "CustomResourcePhysicalID")

data/lib/lambda/getPhysicalId.py

@@ -0,0 +1,67 @@
+import json
+import cfnresponse
+import boto3
+from botocore.exceptions import ClientError
+import time
+
+def findNestedStack(client,resource):
+    if len(resource) == 2:
+        return resource
+    else:
+        max_retries = 20
+        sleep_time = 1
+        for i in range(max_retries):
+            try:
+                response = client.list_stack_resources(StackName=resource[0])
+            except ClientError as e:
+                print e
+                time.sleep(sleep_time)
+                sleep_time += 1
+                continue
+            else:
+                break
+        else:
+            return
+        for r in response['StackResourceSummaries']:
+            if r['ResourceType'] == 'AWS::CloudFormation::Stack':
+                if r['LogicalResourceId'] == resource[1]:
+                    resource[1] = r['PhysicalResourceId']
+        del resource[0]
+        return findNestedStack(client,resource)
+
+def findResource(client,resource):
+    time.sleep(5)
+    if resource:
+        max_retries = 20
+        sleep_time = 1
+        for i in range(max_retries):
+            try:
+                paginator = client.get_paginator('list_stack_resources')
+                page_iterator = paginator.paginate(StackName=resource[0])
+                response = []
+                for page in page_iterator:
+                    response = response + page['StackResourceSummaries']
+            except ClientError as e:
+                print e
+                time.sleep(sleep_time)
+                sleep_time += 1
+                continue
+            else:
+                break
+        else:
+            return
+        for r in response:
+            if r['LogicalResourceId'] == resource[1]:
+                return r['PhysicalResourceId']
+    else:
+        return
+
+def handler(event, context):
+    resource = event['ResourceProperties']['LogicalResourceId'].split(".")
+    region = event['ResourceProperties']['Region']
+    client = boto3.client('cloudformation',region_name=region)
+
+    responseData = {}
+    responseData['PhysicalResourceId'] = findResource(client,findNestedStack(client,resource))
+    cfnresponse.send(event, context, cfnresponse.SUCCESS, responseData, "CustomResourcePhysicalID")
+