sport_ngin_aws_auditor 0.2.0

data/lib/sport_ngin_aws_auditor/google.rb

@@ -0,0 +1,62 @@
+require "google/api_client"
+require "google_drive"
+
+module SportNginAwsAuditor
+	class GoogleConfig < Hash
+    include Hashie::Extensions::IndifferentAccess
+  end
+
+  class Google
+    FILE_NAMES = %w[.google.yml]
+
+    def self.configuration
+      GoogleDrive.login_with_oauth(get_authorization)
+    end
+
+    def self.get_authorization
+      creds = load_config[:credentials]
+      client = ::Google::APIClient.new
+      auth = client.authorization
+      auth.client_id = creds[:client_id]
+      auth.client_secret = creds[:client_secret]
+      auth.scope =
+          "https://www.googleapis.com/auth/drive " +
+          "https://docs.google.com/feeds/ " +
+          "https://docs.googleusercontent.com/ " +
+          "https://spreadsheets.google.com/feeds/"
+      auth.redirect_uri = "urn:ietf:wg:oauth:2.0:oob"
+      print("1. If it doesn't automatically open, open this page:\n%s\n\n" % auth.authorization_uri)
+      `open "#{auth.authorization_uri}"`
+      print("2. Enter the authorization code shown in the page: ")
+      auth.code = $stdin.gets.chomp
+      auth.fetch_access_token!
+      access_token = auth.access_token
+    end
+
+    def self.file
+      load_config[:file]
+    end
+
+    def self.load_config
+      return @config if @config
+      @config = GoogleConfig[YAML.load_file(config_path)]
+    end
+
+    def self.config_path
+      if filepath = FILE_NAMES.detect {|filename| File.exists?(filename)}
+        File.join(Dir.pwd, filepath)
+      else
+        old_dir = Dir.pwd
+        Dir.chdir('..')
+        if old_dir != Dir.pwd
+          config_path
+        else
+          puts "Could not find #{FILE_NAMES.join(' or ')}"
+          exit
+        end
+      end
+    end
+
+  end
+end
+

data/lib/sport_ngin_aws_auditor/google_sheet.rb

@@ -0,0 +1,80 @@
+require 'google_drive'
+
+module SportNginAwsAuditor
+  class GoogleSheet
+    extend GoogleWrapper
+
+    attr_accessor :sheet, :worksheet, :path
+    def initialize(title, path, environment)
+      @sheet = self.class.create_sheet(title, path)
+      @worksheet = self.class.worksheet(sheet, environment)
+    end
+
+    def write_header(header)
+      worksheet.list.keys = header.unshift('name')
+      worksheet.save
+    end
+
+    def write_row(value_hash)
+      worksheet.list.push(value_hash)
+      worksheet.save
+    end
+
+    def self.first_or_create(title)
+      spreadsheet = google.root_collection.files("title" => title, "title-exact" => true).first
+      spreadsheet ? spreadsheet : google.create_spreadsheet(title)
+    end
+
+    #returns a spreadsheet object
+    def self.create_sheet(title, path)
+      folder = go_to_collection(path) if path
+      if folder
+        spreadsheet = folder.files("title" => title, "title-exact" => true).first
+        if spreadsheet
+          return spreadsheet
+        else
+          file = first_or_create(title)
+          folder.add(file)
+          google.root_collection.remove(file)
+          return folder.files("title" => title, "title-exact" => true).first
+        end
+      else
+        first_or_create(title)
+      end
+    end
+
+    #returns a worksheet object
+    def self.worksheet(spreadsheet, title)
+      worksheet = spreadsheet.worksheet_by_title(title)
+      worksheet ? delete_all_rows(worksheet) : spreadsheet.add_worksheet(title)
+    end
+
+    #returns a collection object
+    def self.go_to_collection(directory)
+      if directory
+        path = directory.split('/')
+        go_to_subcollection(google.collection_by_title(path.first),path[1..-1])
+      end
+    end
+
+    #returns a collection object
+    def self.go_to_subcollection(base, subs)
+      puts "Folder doesn't exist in specified path" and exit if base.nil?
+      if subs.empty?
+        return base
+      else
+        base = base.subcollection_by_title(subs.first)
+        go_to_subcollection(base,subs[1..-1])
+      end
+    end
+
+    def self.delete_all_rows(worksheet)
+      worksheet.list.each do |row|
+        row.clear
+      end
+      worksheet.save
+      worksheet
+    end
+
+  end
+end

data/lib/sport_ngin_aws_auditor/instance_helper.rb

@@ -0,0 +1,71 @@
+module SportNginAwsAuditor
+	module InstanceHelper
+
+		def instance_hash
+      Hash[get_instances.map { |instance| instance.nil? ? next : [instance.id, instance]}.compact]
+    end
+
+    def reserved_instance_hash
+      Hash[get_reserved_instances.map { |instance| instance.nil? ? next : [instance.id, instance]}.compact]
+    end
+
+    def instance_count_hash(instances)
+      instance_hash = Hash.new()
+      instances.each do |instance|
+        next if instance.nil?
+        instance_hash[instance.to_s] = instance_hash.has_key?(instance.to_s) ? instance_hash[instance.to_s] + instance.count : instance.count
+      end if instances
+      instance_hash
+    end
+
+    def add_instances_with_tag_to_hash(instances_to_add, instance_hash)
+      instances_to_add.each do |instance|
+        next if instance.nil?
+        key = instance.to_s << " with tag"
+        instance_hash[key] = instance_hash.has_key?(key) ? instance_hash[key] + 1 : 1
+      end if instances_to_add
+      instance_hash
+    end
+
+    def compare(tag_name)
+      differences = Hash.new()
+      instances = get_instances(tag_name)
+      instances_with_tag = filter_instances_with_tags(instances)
+      instances_without_tag = filter_instance_without_tags(instances)
+      instance_hash = instance_count_hash(instances_without_tag)
+      ris = instance_count_hash(get_reserved_instances)
+      instance_hash.keys.concat(ris.keys).uniq.each do |key|
+        instance_count = instance_hash.has_key?(key) ? instance_hash[key] : 0
+        ris_count = ris.has_key?(key) ? ris[key] : 0
+        differences[key] = ris_count - instance_count
+      end
+      add_instances_with_tag_to_hash(instances_with_tag, differences)
+      differences
+    end
+
+    # assuming the value of the tag is in the form: 01/01/2000 like a date
+    def filter_instances_with_tags(instances)
+      instances.select do |instance|
+        value = gather_instance_tag_date(instance)
+        value && (Date.today.to_s < value.to_s)
+      end
+    end
+
+    # assuming the value of the tag is in the form: 01/01/2000 like a date
+    def filter_instance_without_tags(instances)
+      instances.select do |instance|
+        value = gather_instance_tag_date(instance)
+        value.nil? || (Date.today.to_s >= value.to_s)
+      end
+    end
+
+    def gather_instance_tag_date(instance)
+      value = instance.no_reserved_instance_tag_value
+      unless value.nil?
+        date_hash = Date._strptime(value, '%m/%d/%Y')
+        value = Date.new(date_hash[:year], date_hash[:mon], date_hash[:mday]) if date_hash
+      end
+      value
+    end
+	end
+end

data/lib/sport_ngin_aws_auditor/notify_slack.rb

@@ -0,0 +1,31 @@
+require 'slack-notifier'
+
+module SportNginAwsAuditor
+  class NotifySlack
+    attr_accessor :text, :channel, :webhook, :username, :icon_url, :icon_emoji
+
+    def initialize(text)
+      self.text = text
+      if SportNginAwsAuditor::Config.slack
+        self.channel = SportNginAwsAuditor::Config.slack[:channel]
+        self.username = SportNginAwsAuditor::Config.slack[:username]
+        self.webhook = SportNginAwsAuditor::Config.slack[:webhook]
+        self.icon_url = SportNginAwsAuditor::Config.slack[:icon_url]
+      else
+        puts "To use Slack, you must provide a separate config file. See the README for more information."
+      end
+    end
+
+    def perform
+      if SportNginAwsAuditor::Config.slack
+        options = {webhook: webhook,
+                   channel: channel,
+                   username: username,
+                   icon_url: icon_url,
+                   http_options: {open_timeout: 10}
+                  }
+        Slack::Notifier.new(webhook, options).ping(text)
+      end
+    end
+  end
+end

data/lib/sport_ngin_aws_auditor/output.rb

@@ -0,0 +1,13 @@
+require 'highline'
+
+module SportNginAwsAuditor
+  module Output
+    def self.terminal
+      @terminal ||= HighLine.new
+    end
+
+    def self.ask(*args, &block)
+      terminal.ask(*args, &block)
+    end
+  end
+end

data/lib/sport_ngin_aws_auditor/rds_instance.rb

@@ -0,0 +1,78 @@
+require_relative './instance_helper'
+
+module SportNginAwsAuditor
+  class RDSInstance
+    extend InstanceHelper
+    extend RDSWrapper
+    extend AWSWrapper
+
+    class << self
+      attr_accessor :instances, :reserved_instances
+    end
+
+    attr_accessor :id, :name, :multi_az, :instance_type, :engine, :count, :tag_value
+    def initialize(rds_instance, account_id=nil, tag_name=nil, rds=nil)
+      if rds_instance.class.to_s == "Aws::RDS::Types::ReservedDBInstance"
+        self.id = rds_instance.reserved_db_instances_offering_id
+        self.multi_az = rds_instance.multi_az ? "Multi-AZ" : "Single-AZ"
+        self.instance_type = rds_instance.db_instance_class
+        self.engine = rds_instance.product_description
+        self.count = 1
+      elsif rds_instance.class.to_s == "Aws::RDS::Types::DBInstance"
+        self.id = rds_instance.db_instance_identifier
+        self.multi_az = rds_instance.multi_az ? "Multi-AZ" : "Single-AZ"
+        self.instance_type = rds_instance.db_instance_class
+        self.engine = rds_instance.engine
+        self.count = 1
+
+        if tag_name
+          region = rds_instance.availability_zone.split(//).first(9).join
+          region = "us-east-1" if region == "Multiple"
+          arn = "arn:aws:rds:#{region}:#{account_id}:db:#{self.id}"
+
+           # go through to see if the tag we're looking for is one of them
+          rds.list_tags_for_resource(resource_name: arn).tag_list.each do |tag|
+            if tag.key == tag_name
+              self.tag_value = tag.value
+            end
+          end
+        end
+      end
+    end
+
+    def to_s
+      "#{engine_helper} #{multi_az} #{instance_type}"
+    end
+
+    def self.get_instances(tag_name=nil)
+      return @instances if @instances
+      account_id = get_account_id
+      @instances = rds.describe_db_instances.db_instances.map do |instance|
+        next unless instance.db_instance_status.to_s == 'available'
+        new(instance, account_id, tag_name, rds)
+      end.compact
+    end
+
+    def self.get_reserved_instances
+      return @reserved_instances if @reserved_instances
+      @reserved_instances = rds.describe_reserved_db_instances.reserved_db_instances.map do |instance|
+        next unless instance.state.to_s == 'active'
+        new(instance)
+      end.compact
+    end
+
+    def no_reserved_instance_tag_value
+      @tag_value
+    end
+
+    def engine_helper
+      if engine.downcase.include? "post"
+        return "PostgreSQL"
+      elsif engine.downcase.include? "mysql"
+        return "MySQL"
+      end
+    end
+    private :engine_helper
+
+  end
+end

data/lib/sport_ngin_aws_auditor/scripts/audit.rb

@@ -0,0 +1,124 @@
+require 'highline/import'
+require_relative "../notify_slack"
+
+module SportNginAwsAuditor
+  module Scripts
+    class Audit
+      extend AWSWrapper
+
+      class << self
+        attr_accessor :options
+      end
+
+      def self.execute(environment, options=nil)
+        aws(environment)
+        @options = options
+        slack = options[:slack]
+        no_selection = !(options[:ec2] || options[:rds] || options[:cache])
+
+        if options[:no_tag]
+          tag_name = nil
+        else
+          tag_name = options[:tag]
+        end
+
+        if !slack
+          print "Gathering info, please wait..."; print "\r"
+        else
+          puts "Condensed results from this audit will print into Slack instead of directly to an output."
+        end
+
+        data = gather_data("EC2Instance", tag_name) if options[:ec2] || no_selection
+        print_data(slack, environment, data, "EC2Instance") if options[:ec2] || no_selection
+
+        data = gather_data("RDSInstance", tag_name) if options[:rds] || no_selection
+        print_data(slack, environment, data, "RDSInstance") if options[:ec2] || no_selection
+
+        data = gather_data("CacheInstance", tag_name) if options[:cache] || no_selection
+        print_data(slack, environment, data, "CacheInstance") if options[:ec2] || no_selection
+      end
+
+      def self.gather_data(class_type, tag_name)
+        klass = SportNginAwsAuditor.const_get(class_type)
+
+        if options[:instances]
+          instances = klass.get_instances(tag_name)
+          instances_with_tag = klass.filter_instances_with_tags(instances)
+          instances_without_tag = klass.filter_instance_without_tags(instances)
+          instance_hash = klass.instance_count_hash(instances_without_tag)
+          klass.add_instances_with_tag_to_hash(instances_with_tag, instance_hash)
+          return instance_hash
+        elsif options[:reserved]
+          return klass.instance_count_hash(klass.get_reserved_instances)
+        else
+          return klass.compare(tag_name)
+        end
+      end
+
+      def self.print_data(slack, environment, data, class_type)
+        if slack
+          print_to_slack(data, class_type, environment)
+        elsif options[:reserved] || options[:instances]
+          puts header(class_type)
+          data.each{ |key, value| say "<%= color('#{key}: #{value}', :white) %>" }
+        else
+          puts header(class_type)
+          data.each{ |key, value| colorize(key, value) }
+        end
+      end
+
+      def self.colorize(key, value)
+        if key.include?(" with tag")
+          k = key.dup # because key is a frozen string right now
+          k.slice!(" with tag")
+          say "<%= color('#{k}: #{"*" << value.to_s}', :blue) %>"
+        elsif value < 0
+          say "<%= color('#{key}: #{value}', :yellow) %>"
+        elsif value == 0
+          say "<%= color('#{key}: #{value}', :green) %>"
+        elsif value > 0 
+          say "<%= color('#{key}: #{value}', :red) %>"
+        end
+      end
+
+      def self.print_to_slack(instances_hash, class_type, environment)
+        discrepancy_hash = Hash.new
+        instances_hash.each do |key, value|
+          if !(value == 0) && !(key.include?(" with tag"))
+            discrepancy_hash[key] = value
+          end
+        end
+
+        if discrepancy_hash.empty?
+          slack_job = NotifySlack.new("All #{class_type} instances for #{environment} are up to date.")
+          slack_job.perform
+        else
+          print_discrepancies(discrepancy_hash, class_type, environment)
+        end
+      end
+
+      def self.print_discrepancies(discrepancy_hash, class_type, environment)
+        to_print = "Some #{class_type} instances for #{environment} are out of sync:\n"
+        to_print << "#{header(class_type)}\n"
+
+        discrepancy_hash.each do |key, value|
+          to_print << "#{key}: #{value}\n"
+        end
+
+        slack_job = NotifySlack.new(to_print)
+        slack_job.perform
+      end
+
+      def self.header(type, length = 50)
+        type.upcase!.slice! "INSTANCE"
+        half_length = (length - type.length)/2.0 - 1
+        [
+          "*" * length,
+          "*" * half_length.floor + " #{type} " + "*" * half_length.ceil,
+          "*" * length
+        ].join("\n")
+      end
+
+    end
+  end
+end

data/lib/sport_ngin_aws_auditor/scripts/export.rb

@@ -0,0 +1,146 @@
+require 'csv'
+require_relative "../google"
+
+module SportNginAwsAuditor
+  module Scripts
+    class Export
+      extend GoogleWrapper
+      extend AWSWrapper
+
+      class << self
+        attr_accessor :ec2_instances, :rds_instances, :cache_instances, :options, :file, :keys_hash, :environment
+      end
+
+      CLASS_TYPES = %w[EC2Instance RDSInstance CacheInstance]
+
+      def self.execute(environment, options = nil)
+        @environment = environment
+        (puts "Must specify either --drive or --csv"; exit) unless options[:csv] || options[:drive]
+        aws(environment)
+        print "Gathering info, please wait..."
+        all_keys = get_all_keys
+        all_info = prepare
+        print "\r" + " " * 50 + "\r"
+
+        create_csv(all_keys,all_info) if options[:csv]
+        upload_to_drive(all_keys,all_info) if options[:drive]
+      end
+
+      def self.create_csv(keys, info)
+        CSV.open("#{environment}.csv", "wb") do |csv|
+          csv << ["name",keys].flatten
+          info.each do |hash|
+            csv << all_keys_hash.merge(hash).values
+          end
+        end
+
+        `open "#{environment}.csv"`
+      end
+
+      def self.upload_to_drive(keys, info)
+        @file = GoogleSheet.new(Google.file[:name], Google.file[:path], environment)
+        print "Exporting to Google Drive, please wait..."
+        file.write_header(keys)
+        info.each do |value_hash|
+          response = file.worksheet.list.push(value_hash)
+          puts response unless response.is_a? GoogleDrive::ListRow
+        end
+        file.worksheet.save
+        print "\r" + " " * 50 + "\r"
+        puts "Exporting Complete."
+        `open #{file.sheet.human_url}`
+      end
+
+      def self.prepare
+        [get_all_arrays,get_all_counts].flatten
+      end
+
+      def self.get_all_keys
+        return @keys if @keys
+        @keys = [
+          [ec2_reserved_instances.values,ec2_instances.values].flatten.map{ |x| x.to_s }.uniq.sort! { |a,b| a.downcase <=> b.downcase },
+          [rds_reserved_instances.values,rds_instances.values].flatten.map{ |x| x.to_s }.uniq.sort! { |a,b| a.downcase <=> b.downcase },
+          [cache_reserved_instances.values,cache_instances.values].flatten.map{ |x| x.to_s }.uniq.sort! { |a,b| a.downcase <=> b.downcase }
+        ].flatten
+      end
+
+      def self.all_keys_hash(name = nil, value = nil)
+        return @keys_hash if @keys_hash && @keys_hash[:name] == name
+        @keys_hash = {:name => name}
+        get_all_keys.each{ |key| @keys_hash[key] = value }
+        @keys_hash
+      end
+
+      def self.get_all_arrays
+        return @all_array if @all_array
+        @all_array = [ec2_array,rds_array,cache_array].flatten
+      end
+
+      def self.ec2_array
+        instance_array = [{name: "OPSWORKS"}]
+        EC2Instance.bucketize.map do |stack_name, stack_instances|
+          instance_array << {:name => stack_name}.merge(EC2Instance.instance_count_hash(stack_instances))
+        end
+        instance_array
+      end
+
+      def self.rds_array
+        instance_array = [{name: "RDS"}]
+        rds_instances.each do |db_name, db|
+          instance_array << Hash({:name => db_name, "#{db.to_s}" => "#{db.count}"})
+        end
+        instance_array
+      end
+
+      def self.cache_array
+        instance_array = [{name: "CACHE"}]
+        cache_instances.each do |cache_name, cache|
+          instance_array << Hash({:name => cache_name, "#{cache.to_s}" => "#{cache.count}"})
+        end
+        instance_array
+      end
+
+      def self.get_all_counts
+        total_array = [{:name => "TOTALS"}]
+        total_array << all_keys_hash("Running Instances").merge(counts(:instance => true))
+        total_array << all_keys_hash("Reserved Instances", 0).merge(counts(:reserved => true))
+        total_array << all_keys_hash("Differences").merge(counts(:compare => true))
+      end
+
+      def self.counts(options = {:instance => false, :reserved => false, :compare => false })
+        CLASS_TYPES.map do |class_type|
+          klass = SportNginAwsAuditor.const_get(class_type)
+          instances = klass.instance_count_hash(klass.get_instances) if options[:instance]
+          instances = klass.instance_count_hash(klass.get_reserved_instances) if options[:reserved]
+          instances = klass.compare if options[:compare]
+          instances
+        end.inject(:merge)
+      end
+
+      def self.ec2_instances
+        @ec2_instances ||= EC2Instance.instance_hash
+      end
+
+      def self.ec2_reserved_instances
+        @ec2_reserved_instances ||= EC2Instance.reserved_instance_hash
+      end
+
+      def self.rds_instances
+        @rds_instances ||= RDSInstance.instance_hash
+      end
+
+      def self.rds_reserved_instances
+        @rds_reserved_instances ||= RDSInstance.reserved_instance_hash
+      end
+      
+      def self.cache_instances
+        @cache_instances ||= CacheInstance.instance_hash
+      end
+
+      def self.cache_reserved_instances
+        @cache_reserved_instances ||= CacheInstance.reserved_instance_hash
+      end
+
+    end
+  end
+end

data/lib/sport_ngin_aws_auditor/scripts/inspect.rb

@@ -0,0 +1,44 @@
+module SportNginAwsAuditor
+  module Scripts
+    class Inspect
+      extend AWSWrapper
+      extend OpsWorksWrapper
+
+      def self.execute(environment, options=nil)
+        aws(environment)
+        no_selection = options.values.uniq == [false]
+        output("EC2Instance") if options[:ec2] || no_selection
+        output("RDSInstance") if options[:rds] || no_selection 
+        output("CacheInstance") if options[:cache] || no_selection
+      end
+
+      def self.output(class_type)
+        klass = SportNginAwsAuditor.const_get(class_type)
+        print "Gathering info, please wait..."; print "\r"
+        instances = class_type == "EC2Instance" ? klass.bucketize : klass.instance_hash
+        say "<%= color('#{header(class_type)}', :white) %>"
+        instances.each do |key, value|
+          pretty_print(key, klass.instance_count_hash(Array(value)))
+        end
+      end
+
+      def self.header(type, length = 50)
+        type.upcase!.slice! "INSTANCE"
+        half_length = (length - type.length)/2.0 - 1
+        [
+          "*" * length,
+          "*" * half_length.floor + " #{type} " + "*" * half_length.ceil,
+          "*" * length
+        ].join("\n")
+      end
+
+      def self.pretty_print(title, body)
+        puts "======================================="
+        puts "#{title}"
+        puts "======================================="
+        body.each{ |key, value| say "<%= color('#{key}: #{value}', :white) %>" }
+        puts "\n"
+      end
+    end
+  end
+end