cfn_monitor 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 34238326509c8f6011a05378837272162e03c4e0b203a740f7cc9b5969aec650
+  data.tar.gz: 771cffe9fcd1a9191d183cc2a4b2414429ed1fe599ff9be1f79ff3f01c08ea13
+SHA512:
+  metadata.gz: 43a4608773c8c40f26b328f36f2d3ece7dc88aab0ad67e56388ad114f252f1ef37a9cc7ad60c4e4b73c3077cf7dce21de247d40646146b7b1f7e76b81439d11b
+  data.tar.gz: c9edf51029dd38f6b31c30b0ed132375e62a62200d2dd3703a6e0a8acb934483f7dc3616f52687759d897251fb5c3f932bef9c2245b3cf8e9712fc6128ef1303

data/.gitignore

@@ -0,0 +1,10 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+*.gem

data/.rspec

@@ -0,0 +1 @@
+--require spec_helper

data/.travis.yml

@@ -0,0 +1,18 @@
+sudo: required
+dist: trusty
+language: ruby
+rvm:
+  - 2.5
+script:
+  - bundle install
+  - bundle exec rake spec
+  - gem build cfn_monitor.gemspec
+  - gem install cfn_monitor-*.gem
+  - cfn_monitor --version
+deploy:
+  provider: rubygems
+  api_key: "${RUBYGEMS_API_KEY}"
+  gem: cfn_monitor
+  on:
+   all_branches: true
+   condition: $TRAVIS_BRANCH =~ ^develop|master &&  $TRAVIS_EVENT_TYPE =~ ^push|api$ && $TRAVIS_REPO_SLUG == "base2Services/cfn-monitor"

data/Dockerfile

@@ -0,0 +1,7 @@
+FROM ruby:2.5-alpine
+
+RUN gem install cfn_monitor
+
+WORKDIR /src
+
+CMD ["cfn_monitor"]

data/Gemfile

@@ -0,0 +1,6 @@
+source "https://rubygems.org"
+
+git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
+
+# Specify your gem's dependencies in cfn_monitor.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2018 Guslington
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,424 @@
+# cfn-monitor
+
+[![Build Status](https://travis-ci.org/base2Services/cfn-monitor.svg?branch=master)](https://travis-ci.org/base2Services/cfn-monitor)
+
+## About
+
+CloudWatch monitoring tool can query a cloudformation stack and return
+monitorable resources that can be placed into a config file. This config
+can then be used to generate a cloudformation stack to create and manage
+cloudwatch alarms.
+
+It is packaged as a docker container `base2/cfn_monitor` and
+can be run by volume mounting in a local directory to access the config
+or by using within AWS CodePipeline.
+
+## Commands
+
+```bash
+Commands:
+  cfn_monitor --version, -v   # print the version
+  cfn_monitor deploy          # Deploys gerenated cfn templates to S3 bucket
+  cfn_monitor generate        # Generate monitoring cloudformation templates
+  cfn_monitor help [COMMAND]  # Describe available commands or one specific command
+  cfn_monitor query           # Queries a cloudformation stack for monitorable resources
+```
+
+## Run With Docker
+
+The docker image will manage the runtime environment and dependencies.
+You can pass in your AWS credentials and set the region and profile with environment variables.
+
+Example
+```bash
+docker run -it --rm \
+  -v $(pwd):/src \
+  -v $HOME/.aws:/root/.aws \
+  -e AWS_REGION=us-east-1 \
+  -e AWS_PROFILE=default \
+  base2/cfn_monitor cfn_monitor <command> [parameters]
+```
+
+## Configuration files
+
+There are 2 config files that can be utilised to configure your cloudwatch alarms.
+
+- alarms.yaml - Configure the resources you want to monitor
+- template.yaml - Create or override alarm templates
+
+The bellow config structure allows for multiple monitoring stacks to be kept with
+a single code repository separated by directories parameterised as `<application>`.
+
+```
+.
+├── _application_1
+|   ├── alarms.yaml
+|   └── template.yaml
+└── _application_2
+    ├── alarms.yaml
+    └── template.yaml
+```
+
+## Generate
+
+The generate command takes you alarm configuration and turns it into
+cloudformation templates to be deployed into AWS.
+
+```bash
+Usage:
+  cfn_monitor generate
+
+Options:
+  a, [--application=APPLICATION]  # application name
+
+Description:
+  Generates cloudformation templates from the alarm configuration and output to the output/ directory.
+```
+
+### alarms.yml
+
+This file is used to configure the AWS resources you want to monitor with CloudWatch.
+
+```YAML
+source_bucket: [Name of S3 bucket where CloudFormation templates will be deployed]
+source_region: [Region of source_bucket]
+
+resources:
+  [nested stack name].[resource name]: [template name]
+```
+
+Example:
+
+```YAML
+source_bucket: source.example.com
+
+resources:
+  RDSStack.RDS: RDSInstance
+```
+
+#### Resources
+Resources are referenced by the CloudFormation logical resource ID used to create them. Nested stacks are also referenced by their CloudFormation logical resource ID. See example above.
+
+#### Target group configuration:
+Target group alarms in CloudWatch require dimensions for both the target group and its associated load balancer.
+To configure a target group alarm provide the logical ID of the target group (including any stacks it's nested under) followed by "/", followed by the logical ID of the load balancer (also including any stacks it's nested under).
+
+Example:
+```YAML
+resources:
+  LoadBalancerStack.WebDefTargetGroup/LoadBalancerStack.WebLoadBalancer: ApplicationELBTargetGroup
+```
+
+#### Custom Metrics
+Custom metrics are configured with a similar syntax to resources. Use `metrics` instead of `resources`.
+
+Example:
+
+```YAML
+metrics:
+  MyCustomMetric: MyCustomMetricTemplate
+```
+
+#### Endpoints
+HTTP endpoint monitoring and alerting is enabled by configuring resources under `endpoints`. Each endpoint will create a cloudwatch event, scheduled to trigger the `aws-lambda-http-check` lambda function deployed with this stack. Alarms will be configured (based on the specified template) to alert on the cloudwatch metrics generated by the lambda function.
+
+Example:
+
+```YAML
+endpoints:
+  http://www.base2services.com:
+    template: HttpCheck
+    statusCode: 200
+    bodyRegex: 'DevOps'
+```
+
+```YAML
+endpoints:
+  http://www.base2services.com:
+    template: HttpCheck
+    statusCode: 200
+    bodyRegex: 'DevOps'
+    payload: id_=123
+    method: POST
+```
+
+Supported parameters:
+
+Key | Value | Default
+--- | --- | ---
+statusCode | The expected response code | 200
+bodyRegex | A regex expected in the response body | Disabled
+timeOut | A timeout value for the endpoint monitoring | 120 seconds
+scheduleExpression | A cron expression used to schedule the endpoint monitoring | Every minute
+environments | A string or array of environment names. Monitoring will only be deployed for these environments (if specified) | All environments
+
+
+#### Multiple templates
+You can specify multiple templates for the resource by providing a list/array. You may want to do this if you want to deploy some custom alarms in addition to the default alarms for a resource.
+
+Example:
+```YAML
+resources:
+  RDSStack.RDS: [ 'RDSInstance', 'MyRDSInstance' ]
+```
+or
+```YAML
+resources:
+  RDSStack.RDS:
+    - RDSInstance
+    - MyRDSInstance
+```
+
+#### Auto generate alarms config for resources
+You can query an existing stack for monitorable resources using the `query` command.
+This will provide a list of resources in the correct config syntax,
+including the nested stacks and the default templates for those resources.
+
+Example:
+
+```bash
+Usage:
+  cfn_monitor query
+
+Options:
+  a, [--application=APPLICATION]  # application name
+  s, [--stack=STACK]              # cfn stack name
+
+Description:
+  This will provide a list of resources in the correct config syntax,
+  including the nested stacks and the default templates for those resources.
+```
+
+Make sure you query a prod sized stack so that all conditional resources are included.
+The output will list all monitorable resources found in the stack, the coverage your current `alarms.yml` config provides, and a list of any resources missing from your current `alarms.yml` config.
+
+#### Templates
+The "template" value you specify for a resource refers to either a default templates, or a custom/override template in your own `templates.yml`. This template can contain multiple alarms. The example below shows the default `RDSInstance` template, which has 2 alarms (`FreeStorageSpaceCrit` and `FreeStorageSpaceTask`). Using the `RDSInstance` template in this example will create 2 CloudWatch alarms for the `RDS` resource in the `RDSStack` nested stack.
+
+Example: `alarms.yml`
+```YAML
+resources:
+  RDSStack.RDS: RDSInstance
+```
+Example: `templates.yml`
+```YAML
+templates:
+  RDSInstance: # AWS::RDS::DBInstance
+    FreeStorageSpaceCrit:
+      AlarmActions: crit
+      Namespace: AWS/RDS
+      MetricName: FreeStorageSpace
+      ComparisonOperator: LessThanThreshold
+      DimensionsName: DBInstanceIdentifier
+      Statistic: Minimum
+      Threshold: 50000000000
+      Threshold.development: 10000000000
+      EvaluationPeriods: 1
+    FreeStorageSpaceTask:
+      AlarmActions: task
+      Namespace: AWS/RDS
+      MetricName: FreeStorageSpace
+      ComparisonOperator: LessThanThreshold
+      DimensionsName: DBInstanceIdentifier
+      Statistic: Minimum
+      Threshold: 100000000000
+      Threshold.development: 20000000000
+      EvaluationPeriods: 1
+```
+
+#### Globally overriding a template
+
+You can override a default template in your own `templates.yml` file if all instances of a particular resource require a non standard configuration.
+
+Example:
+```YAML
+templates:
+  RDSInstance:
+    FreeStorageSpaceCrit:
+      Threshold: 80000000000
+```
+
+This configuration will be merged over the default `RDSInstance` template resulting in the following:
+
+```YAML
+templates:
+  RDSInstance:
+    FreeStorageSpaceCrit:
+      AlarmActions: crit
+      Namespace: AWS/RDS
+      MetricName: FreeStorageSpace
+      ComparisonOperator: LessThanThreshold
+      DimensionsName: DBInstanceIdentifier
+      Statistic: Minimum
+      Threshold: 80000000000
+      Threshold.development: 10000000000
+      EvaluationPeriods: 1
+    FreeStorageSpaceTask:
+      AlarmActions: task
+      Namespace: AWS/RDS
+      MetricName: FreeStorageSpace
+      ComparisonOperator: LessThanThreshold
+      DimensionsName: DBInstanceIdentifier
+      Statistic: Minimum
+      Threshold: 100000000000
+      Threshold.development: 20000000000
+      EvaluationPeriods: 1
+```
+
+#### Create a custom template
+If the default template for your resource is completely inappropriate, you can create your own custom template in the `monitoring/templates.yml` file.
+
+Example:
+
+```YAML
+templates:
+  MyRDSInstance:
+    DatabaseConnections:
+      AlarmActions: crit
+      Namespace: AWS/RDS
+      MetricName: DatabaseConnections
+      ComparisonOperator: MoreThanThreshold
+      DimensionsName: DBInstanceIdentifier
+      Statistic: Average
+      Threshold: 20
+      EvaluationPeriods: 5
+```
+
+#### Inherit a template
+If you have multiple instances of a particular resource and you want to adjust the configuration for only some of them, you can create your own custom template which inherits the configuration of a default template.
+
+Example:
+```YAML
+templates:
+  MyRDSInstance:
+    template: RDSInstance
+    FreeStorageSpaceCrit:
+      Threshold: 80000000000
+```
+The above example creates a new template `MyRDSInstance` which can now be used by one or many resources. The `MyRDSInstance` template inherits all of the alarms and configuration from `RDSInstance`, but sets `Threshold` to `80000000000` for the `FreeStorageSpaceCrit` alarm.
+
+#### Environment type mappings
+You can create environment type mappings if alarm configurations need to differ between different environment types. This may be useful in situations where development type environments are running different resource quantities or sizes.
+
+Example:
+```YAML
+templates:
+  RDSInstance:
+    FreeStorageSpaceCrit:
+      Threshold: 40000000000
+      Threshold.development: 20000000000
+      Threshold.staging: 30000000000
+      EvaluationPeriods: 5
+```
+The above example shows different `Threshold` values for `EnvironmentType` values of `production` (default), `development` or `staging`.
+Any value can be specified using the `.envType` syntax and the necessary mappings and `EnvironmentType` will be generated when rendered.
+The `EvaluationPeriods` value for `development` and `staging` type environments will be `5` in the above example as no `.envType` values where provided for this parameter.
+
+Supported Parameters:
+Parameter | Mapping support
+--- | ---
+ActionsEnabled | true
+AlarmActions | false
+AlarmDescription | false
+ComparisonOperator | true
+Dimensions | false
+EvaluateLowSampleCountPercentile | false
+EvaluationPeriods | true
+ExtendedStatistic | false
+InsufficientDataActions | false
+MetricName | true
+Namespace | true
+OKActions | false
+Period | true
+Statistic | true
+Threshold | true
+TreatMissingData | true
+Unit | false
+
+#### Template variables
+
+The following variables can be used in templates:
+
+Variable Key | Variable Value
+--- | ---
+${name} | Metric/Resource Name (from alarms.yml)
+${metric} | Metric Name (from alarms.yml)
+${resource} | Resource Name (from alarms.yml)
+${templateName} | Template Name (from templates.yml)
+${alarmName} | Alarm Name (from templates.yml)
+
+Example:
+
+`alarms.yml`
+```YAML
+metrics:
+  Metric1: MyCustomMetric
+```
+`templates.yml`
+```YAML
+templates:
+  MyCustomMetric:
+    ItemCountHigh:
+      MetricName: ${metric}
+      AlarmDescription: '#{templateName} #{alarmName} - #{name}'
+```
+Result:
+```YAML
+templates:
+  MyCustomMetric:
+    ItemCountHigh:
+      MetricName: Metric1
+      AlarmDescription: 'MyCustomMetric ItemCountHigh - Metric1'
+```
+
+#### Alarm Actions
+There are 3 classes of alarm actions: `crit`, `warn` and `task`.
+
+Action | Process
+--- | ---
+crit | Alert on-call technician
+warn | Create alarm in pager service but do not alert on-call technician
+task | Create support ticket for investigation
+
+An SNS topic is required per alarm action, these topics and their subscriptions are managed outside this stack
+
+### Deployment
+
+The rendered CloudFormation templates should be deployed to `[source_bucket]/cloudformation/monitoring/`.
+
+```bash
+Usage:
+  cfn_monitor deploy
+
+Options:
+  a, [--application=APPLICATION]  # application name
+
+Description:
+  Deploys gerenated cloudformation templates to the specified S3 source_bucket
+```
+
+Launch the Monitoring stack in the desired account with the following CloudFormation parameters:
+
+Parameter Key | Parameter Value
+--- | ---
+EnvironmentType | `production` / `development` / `custom env type`
+MonitoredStack | The name of the stack you want monitored. EG `prod`
+MonitoringDisabled | `true` for disables alerts, `false` for enabled alerts
+SnsTopicCrit | SNS topic used by crit type alarms
+SnsTopicTask | SNS topic used by task type alarms
+SnsTopicWarn | SNS topic used by warn type alarms
+
+### Disabling Monitoring
+It is possible to globally disable / snooze / downtime all alarms by setting the `MonitoringDisabled` CloudFormation parameter to `true`.
+This will disable alarm actions without removing removing them.
+
+### Disabling and excluding alarms
+To disable or prevent creation of a specific alarm, specify either of the following parameters:
+```YAML
+templates:
+  MyAutoScalingGroup:
+    template: AutoScalingGroup
+    CPUUtilizationHighBase:
+      CreateAlarm: false    # Don't create the alarm
+      DisableAlarm: true    # Create the alarm but disable it
+```

data/Rakefile

@@ -0,0 +1,9 @@
+require "rake"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec) do |t|
+  t.pattern = Dir.glob("spec/**/*_spec.rb")
+  t.rspec_opts = "--format documentation"
+end
+
+task default: :spec

data/cfn_monitor.gemspec

@@ -0,0 +1,39 @@
+
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "cfn_monitor/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "cfn_monitor"
+  spec.version       = CfnMonitor::VERSION
+  spec.authors       = ["Base2Services", "Jared Brook", "Angus Vine"]
+  spec.email         = ["itsupport@base2services.com"]
+
+  spec.summary       = %q{Configure and generate a cloudwatch monitoring cloudformation stack}
+  spec.description   = %q{CloudWatch monitoring tool can query a cloudformation stack and return
+                          monitorable resources that can be placed into a config file. This config
+                          can then be used to generate a cloudformation stack to create and manage
+                          cloudwatch alarms.}
+  spec.homepage      = "https://github.com/base2Services/cfn-monitor/blob/master/README.md"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "thor", "~> 0.19"
+  spec.add_dependency "cfndsl", "~> 0.16"
+  spec.add_dependency "aws-sdk-cloudformation", "~> 1", "<2"
+  spec.add_dependency "aws-sdk-s3", "~> 1", "<2"
+  spec.add_dependency "aws-sdk-elasticloadbalancingv2", "~> 1", "<2"
+
+  spec.add_development_dependency "bundler", "~> 1.16"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec", "~> 0.9"
+  spec.add_development_dependency "rspec-core", "~> 3.8"
+  spec.add_development_dependency "rspec-expectations", "~> 3.8"
+  spec.add_development_dependency "rspec-mocks", "~> 3.8"
+end

data/exe/cfn_monitor

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+require "cfn_monitor"
+
+CfnMonitor::Commands.start( ARGV )

data/lib/cfn_monitor/deploy.rb

@@ -0,0 +1,47 @@
+require 'aws-sdk-s3'
+require 'yaml'
+
+module CfnMonitor
+  class Deploy
+
+    def self.run(options)
+
+      if !options['application']
+        raise "No application specified"
+      end
+
+      application = options['application']
+
+      custom_alarms_config_file = "#{application}/alarms.yml"
+      output_path = "output/#{application}"
+      upload_path = "cloudformation/monitoring/#{application}"
+
+      # Load custom config files
+      if File.file?(custom_alarms_config_file)
+        custom_alarms_config = YAML.load(File.read(custom_alarms_config_file)) if File.file?(custom_alarms_config_file)
+      else
+        puts "Failed to load #{custom_alarms_config_file}"
+        exit 1
+      end
+
+      puts "-----------------------------------------------"
+      s3 = Aws::S3::Client.new(region: custom_alarms_config['source_region'])
+      ["#{output_path}/*.json"].each { |path|
+        Dir.glob(path) do |file|
+          template = File.open(file, 'rb')
+          filename = file.gsub("#{output_path}/", "")
+          s3.put_object({
+              body: template,
+              bucket: "#{custom_alarms_config['source_bucket']}",
+              key: "#{upload_path}/#{filename}",
+          })
+          puts "INFO: Copied #{file} to s3://#{custom_alarms_config['source_bucket']}/#{upload_path}/#{filename}"
+        end
+      }
+      puts "-----------------------------------------------"
+      puts "Master stack: https://s3.#{custom_alarms_config['source_region']}.amazonaws.com/#{custom_alarms_config['source_bucket']}/#{upload_path}/master.json"
+      puts "-----------------------------------------------"
+    end
+
+  end
+end