cfn_monitor 0.1.0

data/lib/cfn_monitor/generate.rb

@@ -0,0 +1,186 @@
+require "cfndsl"
+require 'fileutils'
+require 'tempfile'
+require 'yaml'
+
+require "cfn_monitor/utils"
+
+module CfnMonitor
+  class Generate
+
+    def self.run(options)
+
+      if !options['application']
+        raise "No application specified"
+      end
+
+      application = options['application']
+
+      template_path = File.join(File.dirname(__FILE__),'../config/templates.yml')
+      config_path = File.join(File.dirname(__FILE__),'../config/config.yml')
+      # Load global config files
+      global_templates_config = YAML.load(File.read(template_path))
+      config = YAML.load(File.read(config_path))
+
+      custom_alarms_config_file = "#{application}/alarms.yml"
+      custom_templates_config_file = "#{application}/templates.yml"
+      output_path = "output/#{application}"
+      upload_path = "cloudformation/monitoring/#{application}"
+
+      # Load custom config files
+      if File.file?(custom_alarms_config_file)
+        custom_alarms_config = YAML.load(File.read(custom_alarms_config_file))
+      else
+        puts "Failed to load #{custom_alarms_config_file}"
+        exit 1
+      end
+
+      # Merge custom template configs over global template configs
+      if File.file?(custom_templates_config_file)
+        custom_templates_config = YAML.load(File.read(custom_templates_config_file))
+        templates = CommonHelper.deep_merge(global_templates_config, custom_templates_config)
+      else
+        templates = global_templates_config
+      end
+
+      # Create an array of alarms based on the templates associated with each resource
+      alarms = []
+      resources = custom_alarms_config['resources']
+      metrics = custom_alarms_config['metrics']
+      hosts = custom_alarms_config['hosts']
+      hosts ||= {}
+      services = custom_alarms_config['services']
+      services ||= {}
+      endpoints = custom_alarms_config['endpoints']
+      endpoints ||= {}
+      rme = { resources: resources, metrics: metrics, endpoints: endpoints, hosts: hosts, services: services }
+      source_bucket = custom_alarms_config['source_bucket']
+
+      rme.each do | k,v |
+        if !v.nil?
+          v.each do | resource,attributes |
+            # set environments to 'all' by default
+            environments = ['all']
+            # Support config hashs for additional parameters
+            params = {}
+            if attributes.kind_of?(Hash)
+              attributes.each do | a,b |
+                environments = b if a == 'environments'
+                # Convert strings to arrays for consistency
+                if !environments.kind_of?(Array) then environments = environments.split end
+                params[a] = b if !['template','environments'].member? a
+              end
+              templatesEnabled = attributes['template']
+            else
+              templatesEnabled = attributes
+            end
+            # Convert strings to arrays for looping
+            if !templatesEnabled.kind_of?(Array) then templatesEnabled = templatesEnabled.split end
+            templatesEnabled.each do | templateEnabled |
+              if !templates['templates'][templateEnabled].nil?
+                # If a template is provided, inherit that template
+                if !templates['templates'][templateEnabled]['template'].nil?
+                  template_from = Marshal.load( Marshal.dump(templates['templates'][templates['templates'][templateEnabled]['template']]) )
+                  template_to = templates['templates'][templateEnabled].without('template')
+                  template_merged = CommonHelper.deep_merge(template_from, template_to)
+                  templates['templates'][templateEnabled] = template_merged
+                end
+                templates['templates'][templateEnabled].each do | alarm,parameters |
+                  resourceParams = parameters.clone
+                  # Override template params if overrides provided
+                  params.each do | x,y |
+                    resourceParams[x] = y
+                  end
+                  if k == :hosts
+                    resourceParams['cmds'].each do |cmd|
+                      hostParams = resourceParams.clone
+                      hostParams['cmd'] = cmd
+                      # Construct alarm object per cmd
+                      alarms << {
+                        resource: resource,
+                        type: k[0...-1],
+                        template: templateEnabled,
+                        alarm: alarm,
+                        parameters: hostParams,
+                        environments: environments
+                      }
+                    end
+                  else
+                    # Construct alarm object
+                    alarms << {
+                      resource: resource,
+                      type: k[0...-1],
+                      template: templateEnabled,
+                      alarm: alarm,
+                      parameters: resourceParams,
+                      environments: environments
+                    }
+                  end
+                end
+              end
+            end
+          end
+        end
+      end
+
+      # Create temp alarms file for CfnDsl
+      temp_file = Tempfile.new(["alarms-",'.yml'])
+      temp_file_path = temp_file.path
+      temp_file.write({'alarms' => alarms}.to_yaml)
+      temp_file.rewind
+
+      # Split resources for mulitple templates to avoid CloudFormation template resource limits
+      split = []
+      template_envs = ['production']
+      alarms.each_with_index do |alarm,index|
+        split[index/config['resource_limit']] ||= {}
+        split[index/config['resource_limit']]['alarms'] ||= []
+        split[index/config['resource_limit']]['alarms'] << alarm
+        template_envs |= get_alarm_envs(alarm[:parameters])
+      end
+
+      # Create temp files for split resources for CfnDsl input
+      temp_files=[]
+      temp_file_paths=[]
+      (alarms.count/config['resource_limit'].to_f).ceil.times do | i |
+        temp_files[i] = Tempfile.new(["alarms-#{i}-",'.yml'])
+        temp_file_paths[i] = temp_files[i].path
+        temp_files[i].write(split[i].to_yaml)
+        temp_files[i].rewind
+      end
+
+      write_cfdndsl_template(temp_file_path, temp_file_paths, custom_alarms_config_file, source_bucket, template_envs, output_path, upload_path)
+
+    end
+
+    def self.write_cfdndsl_template(alarms_config,configs,custom_alarms_config_file,source_bucket,template_envs,output_path,upload_path)
+      template_path = File.expand_path("../../templates", __FILE__)
+      FileUtils::mkdir_p output_path
+      configs.each_with_index do |config,index|
+        File.open("#{output_path}/resources#{index}.json", 'w') { |file|
+          file.write(JSON.pretty_generate( CfnDsl.eval_file_with_extras("#{template_path}/resources.rb",[[:yaml, config],[:raw, "template_number=#{index}"],[:raw, "source_bucket='#{source_bucket}'"],[:raw, "upload_path='#{upload_path}'"]],STDOUT)))}
+        File.open("#{output_path}/alarms#{index}.json", 'w') { |file|
+          file.write(JSON.pretty_generate( CfnDsl.eval_file_with_extras("#{template_path}/alarms.rb",[[:yaml, config],[:raw, "template_number=#{index}"],[:raw, "template_envs=#{template_envs}"]],STDOUT)))}
+      end
+      File.open("#{output_path}/endpoints.json", 'w') { |file|
+        file.write(JSON.pretty_generate( CfnDsl.eval_file_with_extras("#{template_path}/endpoints.rb",[[:yaml, alarms_config],[:raw, "template_envs=#{template_envs}"]],STDOUT)))}
+      File.open("#{output_path}/hosts.json", 'w') { |file|
+        file.write(JSON.pretty_generate( CfnDsl.eval_file_with_extras("#{template_path}/hosts.rb",[[:yaml, alarms_config],[:raw, "template_envs=#{template_envs}"]],STDOUT)))}
+      File.open("#{output_path}/services.json", 'w') { |file|
+        file.write(JSON.pretty_generate( CfnDsl.eval_file_with_extras("#{template_path}/services.rb",[[:yaml, alarms_config],[:raw, "template_envs=#{template_envs}"]],STDOUT)))}
+      File.open("#{output_path}/master.json", 'w') { |file|
+        file.write(JSON.pretty_generate( CfnDsl.eval_file_with_extras("#{template_path}/master.rb",[[:yaml, custom_alarms_config_file],[:raw, "templateCount=#{configs.count}"],[:raw, "template_envs=#{template_envs}"],[:raw, "upload_path='#{upload_path}'"]],STDOUT)))}
+    end
+
+    def self.get_alarm_envs(params)
+      envs = []
+      params.each do | key,value |
+        if key.include? '.'
+          envs << key.split('.').last
+        end
+      end
+      return envs
+    end
+
+  end
+end

data/lib/cfn_monitor/query.rb

@@ -0,0 +1,135 @@
+require 'aws-sdk-cloudformation'
+require 'aws-sdk-elasticloadbalancingv2'
+
+module CfnMonitor
+  class Query
+
+    def self.run(options)
+
+      if !options['application'] || !options['stack']
+        raise "No application specified"
+      end
+
+      if !options['stack']
+        raise "No stack specified"
+      end
+
+      config_path = File.join(File.dirname(__FILE__),'../config/config.yml')
+      # Load global config files
+      config = YAML.load(File.read(config_path))
+
+      custom_alarms_config_file = "#{options['application']}/alarms.yml"
+
+      # Load custom config files
+      custom_alarms_config = YAML.load(File.read(custom_alarms_config_file)) if File.file?(custom_alarms_config_file)
+      custom_alarms_config ||= {}
+      custom_alarms_config['resources'] ||= {}
+
+      puts "-----------------------------------------------"
+      puts "stack: #{options['stack']}"
+      puts "application: #{options['application']}"
+      puts "-----------------------------------------------"
+      puts "Searching Stacks for Monitorable Resources"
+      puts "-----------------------------------------------"
+
+      cfClient = Aws::CloudFormation::Client.new()
+      elbClient = Aws::ElasticLoadBalancingV2::Client.new()
+
+      stackResourceQuery = query_stacks(config,cfClient,elbClient,options['stack'])
+      stackResourceCount = stackResourceQuery[:stackResourceCount]
+      stackResources = stackResourceQuery[:stackResources]
+
+      configResourceCount = custom_alarms_config['resources'].keys.count
+      configResources = []
+      keyUpdates = []
+
+      stackResources[:template].each do | k,v |
+        if stackResources[:physical_resource_id].key? k.partition('/').last
+          keyUpdates << k
+        end
+      end
+
+      keyUpdates.each do | k |
+        stackResources[:template]["#{k.partition('/').first}/#{stackResources[:physical_resource_id][k.partition('/').last]}"] = stackResources[:template].delete(k)
+      end
+
+      stackResources[:template].each do | k,v |
+        if !custom_alarms_config['resources'].any? {|x, y| x == k}
+          configResources.push("#{k}: #{v}")
+        end
+      end
+
+      puts "-----------------------------------------------"
+      puts "Monitorable Resources (with default templates)"
+      puts "-----------------------------------------------"
+      stackResources[:template].each do | k,v |
+        puts "#{k}: #{v}"
+      end
+      puts "-----------------------------------------------"
+      if configResourceCount < stackResourceCount
+        puts "Missing resources (with default templates)"
+        puts "-----------------------------------------------"
+        configResources.each do | r |
+          puts r
+        end
+        puts "-----------------------------------------------"
+      end
+      puts "Monitorable resources in #{options['stack']} stack: #{stackResourceCount}"
+      puts "Resources in #{options['application']} alarms config: #{configResourceCount}"
+      if stackResourceCount > 0
+        puts "Coverage: #{100-(configResources.count*100/stackResourceCount)}%"
+      end
+      puts "-----------------------------------------------"
+
+    end
+
+    def self.query_stacks (config,cfClient,elbClient,stack,stackResources={template:{},physical_resource_id:{}},location='')
+      stackResourceCount = 0
+      stackResourceCountLocal = 0
+      begin
+        resp = cfClient.list_stack_resources({
+          stack_name: stack
+        })
+      rescue Aws::CloudFormation::Errors::ServiceError => e
+        puts "Error: #{e}"
+        exit 1
+      end
+
+      resp.stack_resource_summaries.each do | resource |
+        if resource['resource_type'] == 'AWS::CloudFormation::Stack'
+          query = query_stacks(config,cfClient,elbClient,resource['physical_resource_id'],stackResources,"#{location}.#{resource['logical_resource_id']}")
+          stackResourceCount += query[:stackResourceCount]
+        end
+        if config['resource_defaults'].key? resource['resource_type']
+          if resource['resource_type'] == 'AWS::ElasticLoadBalancingV2::TargetGroup'
+            begin
+              tg = elbClient.describe_target_groups({
+                target_group_arns: [ resource['physical_resource_id'] ]
+              })
+            rescue Aws::ElasticLoadBalancingV2::Errors::ServiceError => e
+              puts "Error: #{e}"
+              exit 1
+            end
+            stackResources[:template]["#{location[1..-1]}.#{resource['logical_resource_id']}/#{tg['target_groups'][0]['load_balancer_arns'][0]}"] = config['resource_defaults'][resource['resource_type']]
+          else
+            stackResources[:template]["#{location[1..-1]}.#{resource['logical_resource_id']}"] = config['resource_defaults'][resource['resource_type']]
+          end
+          stackResourceCount += 1
+          stackResourceCountLocal += 1
+          print "#{location[1..-1]}: Found #{stackResourceCount} resource#{"s" if stackResourceCount != 1}\r"
+          sleep 0.2
+        elsif resource['resource_type'] == 'AWS::ElasticLoadBalancingV2::LoadBalancer'
+          stackResources[:physical_resource_id][resource['physical_resource_id']] = "#{location[1..-1]}.#{resource['logical_resource_id']}"
+        end
+      end
+      stackResourceQuery = {
+        stackResourceCount: stackResourceCount,
+        stackResources: stackResources
+      }
+      sleep 0.2
+      puts "#{stack if location == ''}#{location[1..-1]}: Found #{stackResourceCountLocal} resource#{"s" if stackResourceCountLocal != 1}"
+      stackResourceQuery
+    end
+
+  end
+end

data/lib/cfn_monitor/utils.rb

@@ -0,0 +1,42 @@
+module CfnMonitor
+  class Utils
+
+    # Merge Hash B into hash A. If any values or hashes as well, merge will be performed recursively
+    # Returns Hash A with updated values
+    def self.deep_merge(a, b)
+
+      # Loop over key/value pairs
+      b.each { |key, value|
+
+        # If key from B present in map A
+        if a.key? key
+
+          # If both are hashes call recursively
+          if (a[key].class == Hash and b[key].class == Hash)
+            a[key] = deep_merge(a[key], value)
+          else
+            # Overwrite value with value from B
+            a[key] = value
+          end
+        else
+          # Add key from B
+          a[key] = value
+        end
+      }
+
+      # Return hash a
+      return a
+    end
+
+  end
+end
+
+class Hash
+  def without(*keys)
+    dup.without!(*keys)
+  end
+
+  def without!(*keys)
+    reject! { |key| keys.include?(key) }
+  end
+end

data/lib/cfn_monitor/version.rb

@@ -0,0 +1,3 @@
+module CfnMonitor
+  VERSION = "0.1.0".freeze
+end

data/lib/cfn_monitor.rb

@@ -0,0 +1,66 @@
+require "thor"
+require "cfn_monitor/version"
+require "cfn_monitor/query"
+require "cfn_monitor/generate"
+require "cfn_monitor/deploy"
+
+module CfnMonitor
+  class Commands < Thor
+
+    map %w[--version -v] => :__print_version
+    desc "--version, -v", "print the version"
+    def __print_version
+      puts CfnMonitor::VERSION
+    end
+
+    # class_option :verbose,
+    #   aliases: :V,
+    #   type: :boolean,
+    #   default: false,
+    #   lazy_default: true,
+    #   desc: "Extra logging"
+    #
+    # class_option :region,
+    #   group: :aws,
+    #   aliases: :r,
+    #   type: :string,
+    #   desc: "AWS Region"
+    #
+    # class_option :profile,
+    #   group: :aws,
+    #   aliases: :p,
+    #   type: :string,
+    #   desc: "Profile name in AWS credentials file"
+
+    desc "generate", "Generate monitoring cloudformation templates"
+    long_desc <<-LONG
+    Generates cloudformation templates from the alarm configuration and output to the output/ directory.
+    LONG
+    method_option :application, aliases: :a, type: :string, desc: "application name"
+    # method_option :validate, aliases: :v, type: :boolean, default: true, desc: "validate cfn templates"
+    def generate
+      CfnMonitor::Generate.run(options)
+    end
+
+    desc "query", "Queries a cloudformation stack for monitorable resources"
+    long_desc <<-LONG
+    This will provide a list of resources in the correct config syntax,
+    including the nested stacks and the default templates for those resources.
+    LONG
+    method_option :application, aliases: :a, type: :string, desc: "application name"
+    method_option :stack, aliases: :s, type: :string, desc: "cfn stack name"
+    def query
+      CfnMonitor::Query.run(options)
+    end
+
+    desc "deploy", "Deploys gerenated cfn templates to S3 bucket"
+    long_desc <<-LONG
+    Deploys gerenated cloudformation templates to the specified S3 source_bucket
+    LONG
+    method_option :application, aliases: :a, type: :string, desc: "application name"
+    def deploy
+      CfnMonitor::Deploy.run(options)
+    end
+
+  end
+end

data/lib/config/config.yml

@@ -0,0 +1,14 @@
+resource_limit: 50
+resource_defaults:
+  AWS::ElasticLoadBalancing::LoadBalancer: ElasticLoadBalancer
+  AWS::ElasticLoadBalancingV2::TargetGroup: ApplicationELBTargetGroup
+  AWS::AutoScaling::AutoScalingGroup: AutoScalingGroup
+  AWS::RDS::DBInstance: RDSInstance
+  AWS::RDS::DBCluster: DBCluster
+  AWS::ECS::Cluster: ECSCluster
+  AWS::ElastiCache::ReplicationGroup: ElastiCacheReplicationGroup
+  AWS::EFS::FileSystem: ElasticFileSystem
+  AWS::EC2::Instance: Ec2Instance
+  AWS::Redshift::Cluster: RedshiftCluster
+  AWS::DynamoDB::Table: DynamoDBTable
+  AWS::ApiGateway::RestApi: ApiGateway