aws-reporting 0.9.1

data/bin/aws-reporting

@@ -0,0 +1,54 @@
+#!/usr/bin/env ruby
+
+require File.expand_path('../../lib/aws-reporting', __FILE__)
+
+slop = Slop.new(:strict => true) do
+  command 'config' do
+    banner "Usage: aws-reporting config [-h] [-f] [--access-key-id ACCESS_KEY_ID --secret-access-key SECRET_ACCESS_KEY]"
+    on 'h', 'help', "Show this messages"
+    on 'f', 'force', "Force output config file"
+    on 'access-key-id=', "AWS Access Key ID"
+    on 'secret-access-key=', "AWS Secret Access Key"
+
+    run do |opts, args|
+      AwsReporting::Command::Config.run(opts, args) 
+    end
+  end
+
+  command 'run' do
+    banner "Usage: aws-reporting run [-f] REPORT_PATH"
+    on 'h', 'help', "Show this messages"
+    on 'f', 'force', "Force output report"
+
+    run do |opts, args|
+      AwsReporting::Command::Run.run(opts, args) 
+    end
+  end
+
+  command 'serve' do
+    banner "Usage: aws-reporting serve [-p PORT] REPORT_PATH"
+    on 'h', 'help', "Show this messages"
+    on 'p=', 'port=', 'Port to listen on'
+
+    run do |opts, args|
+      AwsReporting::Command::Serve.run(opts, args) 
+    end
+  end
+
+  command 'version' do
+    run do |opts, args|
+      AwsReporting::Command::Version.run(opts, args) 
+    end
+  end
+
+  run do |opts, args|
+    puts opts.help
+  end
+end
+
+begin
+  slop.parse
+rescue Slop::Error => e
+  puts e.message
+  puts slop
+end

data/lib/aws-reporting.rb

@@ -0,0 +1,30 @@
+$LOAD_PATH.push File.expand_path('../aws-reporting', __FILE__)
+
+# standard libraries
+require 'yaml'
+require 'fileutils'
+
+# 3rd party libraries
+require 'slop'
+require 'aws-sdk'
+require 'formatador'
+require 'parallel'
+
+# aws-reporting files
+require 'error'
+require 'version'
+require 'server'
+require 'generator'
+require 'plan'
+require 'config'
+require 'command/config'
+require 'command/run'
+require 'command/serve'
+require 'command/version'
+require 'resolvers'
+require 'resolver/ec2'
+require 'resolver/ebs'
+require 'helper'
+require 'alarm'
+require 'statistics'
+require 'store'

data/lib/aws-reporting/alarm.rb

@@ -0,0 +1,36 @@
+module AwsReporting
+  module Alarm
+    def serialize(dimensions)
+      dimensions.sort_by{|dimension| dimension[:name]}.map{|dimension| dimension[:name] + "=>" + dimension[:value]}.join(',')
+    end
+
+    def get_alarm_info(region, alarm)
+      info = {:name => alarm.name,
+              :region => region,
+              :namespace => alarm.namespace,
+              :dimensions => alarm.dimensions,
+              :metric_name => alarm.metric_name,
+              :status => get_status(alarm)}
+    end
+
+    def get_status(alarm)
+      return :ALARM if alarm.state_value == 'ALARM'
+      return :ALARM if alarm.history_items.to_a.select{|history| history.history_item_type == 'StateUpdate'}.length > 0
+      return :OK
+    end
+
+    def get_alarms()
+      alarms = []
+      AWS.regions.each{|r|
+        Config.update_region(r.name)
+        cw = AWS::CloudWatch.new
+        cw.alarms.each do |alarm|
+          alarms << get_alarm_info(r.name, alarm)
+        end
+      }
+      alarms.sort_by{|alarm| [alarm[:namespace], serialize(alarm[:dimensions]), alarm[:metric_name], alarm[:name]].join(' ')}
+    end
+
+    module_function :get_alarms
+  end
+end

data/lib/aws-reporting/command/config.rb

@@ -0,0 +1,75 @@
+module AwsReporting
+  module Command
+    module Config
+      DEFAULT_PATH = '~/.aws-reporting/config.yaml'
+      MESSAGE_ALREADY_EXIST = " already exists. If you want to overwrite this, use '-f' option."
+
+      def run(opts, args)
+        Signal.trap(:INT){
+          puts
+          puts "interrupted."
+          exit
+        }
+
+        help = opts['h']
+        if help
+          puts opts.help
+          return
+        end
+
+        begin
+          access = opts['access-key-id']
+          secret = opts['secret-access-key']
+          force = opts['f']
+
+          if access and secret
+            run_batch(access, secret, force)
+          elsif !!!access and !!!secret
+            run_interactive(force)
+          else
+            raise CommandArgumentError.new
+          end
+        rescue AwsReporting::Error::CommandArgumentError
+          puts opts.help
+        end
+      end
+
+      def run_batch(access, secret, force)
+        path = File.expand_path(DEFAULT_PATH)
+        if force or !File.exist?(path)
+          update_config(access, secret, path)
+          puts 'done.'
+        else
+          puts path + MESSAGE_ALREADY_EXIST
+        end
+      end
+
+      def run_interactive(force)
+        path = File.expand_path(DEFAULT_PATH)
+        if force or !File.exist?(path)
+          print 'Access Key ID    :'
+          access = $stdin.gets.chomp
+          print 'Secret Access Key:'
+          secret = $stdin.gets.chomp
+
+          update_config(access, secret, path)
+          puts 'done.'
+        else
+          puts path + MESSAGE_ALREADY_EXIST
+        end
+      end
+
+      def update_config(access, secret, path)
+        yaml = YAML.dump(:access_key_id => access, :secret_access_key => secret)
+
+        FileUtils.mkdir_p(File.dirname(path))
+
+        open(path, 'w'){|f|
+          f.print yaml
+        }
+      end
+
+      module_function :run, :run_batch, :run_interactive, :update_config
+    end
+  end
+end

data/lib/aws-reporting/command/run.rb

@@ -0,0 +1,35 @@
+module AwsReporting
+  module Command
+    module Run
+      MESSAGE_ALREADY_EXIST = " already exists. If you want to overwrite this, use '-f' option."
+      MESSAGE_NOT_CONFIGURED = "Can not access config file. Run `aws-reporting config` command first."
+
+      def run(opts, args)
+        begin
+          help = opts['h']
+          if help
+            puts opts.help
+            return
+          end
+
+          force = opts['f']
+          raise AwsReporting::Error::CommandArgumentError.new unless args.length == 1
+          path = args[0]
+
+          generator = AwsReporting::Generator.new
+          generator.path = path
+          generator.force = force
+          generator.generate
+        rescue AwsReporting::Error::CommandArgumentError
+          puts opts.help
+        rescue AwsReporting::Error::OverwriteError => e
+          puts e.path + MESSAGE_ALREADY_EXIST
+        rescue AwsReporting::Error::ConfigFileLoadError
+          puts MESSAGE_NOT_CONFIGURED
+        end
+      end
+
+      module_function :run
+    end
+  end
+end

data/lib/aws-reporting/command/serve.rb

@@ -0,0 +1,31 @@
+module AwsReporting
+  module Command
+    module Serve
+      def run(opts, args)
+        begin
+          help = opts['h']
+          if help
+            puts opts.help
+            return
+          end
+
+          port = opts['port'] || 23456
+          raise AwsReporting::Error::CommandArgumentError.new unless args.length == 1
+          path = args[0]
+
+          server = AwsReporting::Server.new(path, port)
+
+          Signal.trap(:INT){
+            server.stop
+          }
+
+          server.start
+        rescue AwsReporting::Error::CommandArgumentError
+          puts opts.help
+        end
+      end
+
+      module_function :run
+    end
+  end
+end

data/lib/aws-reporting/command/version.rb

@@ -0,0 +1,11 @@
+module AwsReporting
+  module Command
+    module Version
+      def run(opts, args)
+        puts AwsReporting::Version.get
+      end
+
+      module_function :run
+    end
+  end
+end

data/lib/aws-reporting/config.rb

@@ -0,0 +1,24 @@
+module AwsReporting
+  class Config
+    def self.config_file_path()
+      File.expand_path('~/.aws-reporting/config.yaml')
+    end
+
+    def self.load()
+      @@config = YAML.load(open(config_file_path()){|f| f.read})
+    rescue
+      raise AwsReporting::Error::ConfigFileLoadError.new
+    end
+
+    def self.update_region(region)
+      AWS.config(:access_key_id => @@config[:access_key_id],
+                 :secret_access_key => @@config[:secret_access_key],
+                 :region => region)
+    end
+
+    def self.get()
+      AWS.config(:access_key_id => @@config[:access_key_id],
+                 :secret_access_key => @@config[:secret_access_key])
+    end
+  end
+end

data/lib/aws-reporting/error.rb

@@ -0,0 +1,9 @@
+module AwsReporting
+  module Error
+    class CommandArgumentError < StandardError; end
+    class OverwriteError < StandardError
+      attr_accessor :path
+    end
+    class ConfigFileLoadError < StandardError; end
+  end
+end

data/lib/aws-reporting/generator.rb

@@ -0,0 +1,186 @@
+module AwsReporting
+  class Generator
+    attr_accessor :path, :force
+
+    def format_path(path)
+      path.split('/')[-2..-1]
+          .join('/')
+    end
+
+    def transform(metrics, name_resolvers)
+      temp = Hash.new{|h, k| h[k] = Hash.new{|h, k| h[k] = [] }}
+      metrics.each{|namespace, value|
+        value.each{|identification, files|
+          element =  {:region => identification[:region],
+                      :namespace => namespace,
+                      :dimensions => identification[:dimensions],
+                      :files => files.sort_by{|entry| entry[:info][:metric_name]}.map{|entry| format_path(entry[:path])}}
+
+          element[:name] = name_resolvers.get_name(element)
+
+          dimension_type = identification[:dimensions].map{|item| item[:name]}.sort
+
+          temp[namespace][dimension_type] << element
+        }
+      }
+
+      temp2 = Hash.new{|h, k| h[k] = [] }
+      temp.each{|namespace, dimension_table|
+        dimension_table.each{|dimension_type, elements| 
+          temp2[namespace] << {:dimension_type => dimension_type,
+                               :elements => elements}
+        }
+      }
+
+      result = []
+      temp2.each{|namespace, dimension_table|
+        result << {:namespace => namespace, :dimension_table => dimension_table}
+      }
+
+      sort_result!(result)
+
+      result
+    end
+
+    def sort_result!(result)
+      result.sort_by!{|namespace_table| namespace_table[:namespace]}
+      result.each{|namespace_table|
+        namespace_table[:dimension_table].sort_by!{|dimension_table| dimension_table[:dimension_type].join(' ')}
+      }
+
+      result.each{|namespace_table|
+        namespace_table[:dimension_table].each{|dimension_table|
+          dimension_type = dimension_table[:dimension_type]
+          dimension_table[:elements].sort_by!{|element| expand(dimension_type, element[:dimensions]) }
+        }
+      }
+    end
+
+    def expand(dimension_type, dimensions)
+      dimension_hash = {}
+
+      dimensions.each{|dimension|
+        dimension_hash[dimension[:name]] = dimension[:value]
+      }
+
+      values = []
+      dimension_type.each{|dimension_name|
+        values << dimension_hash[dimension_name] 
+      }
+      values.join(' ')
+    end
+
+    def merge_status(s0, s1)
+      if s0 == nil or s1 == nil
+        s0 || s1
+      else
+        if s0 == :ALARM or s1 == :ALARM
+          'ALARM'
+        else
+          'OK'
+        end
+      end
+    end
+
+    def build_alarm_tree(alarms)
+      alarm_tree = {}
+      alarms.each{|alarm|
+        key = {:region => alarm[:region],
+               :namespace => alarm[:namespace],
+               :dimensions => alarm[:dimensions],
+               :metric_name => alarm[:metric_name]}
+        alarm_tree[key] = merge_status(alarm_tree[key], alarm[:status])
+      }
+      alarm_tree
+    end
+
+    def set_status(data, alarm_tree)
+      key = {:region => data[:info][:region],
+             :namespace => data[:info][:namespace],
+             :dimensions => data[:info][:dimensions],
+             :metric_name => data[:info][:metric_name]}
+      data[:info][:status] = alarm_tree[key] if alarm_tree[key]
+    end
+
+    def download(base_path, plan, start_time, end_time, period, name_resolvers, timestamp)
+      metrics = Hash.new{|h, k| h[k] = Hash.new{|h, k| h[k] = []}}
+
+      alarms = AwsReporting::Alarm.get_alarms()
+      alarm_tree = build_alarm_tree(alarms)
+
+      mutex = Mutex.new
+      started_at = Time.now
+      num_of_metrics = plan.length
+      num_of_downloaded = 0
+      Parallel.each_with_index(plan, :in_threads => 8){|entry, i|
+        namespace = entry[:namespace]
+        metric_name = entry[:metric_name]
+        dimensions = entry[:dimensions]
+        statistics = entry[:statistics]
+        region = entry[:region]
+
+        mutex.synchronize do
+          num_of_downloaded += 1
+          Formatador.redisplay_progressbar(num_of_downloaded, num_of_metrics, {:started_at => started_at})
+        end
+
+        data = AwsReporting::Statistics.get(region, namespace, metric_name, start_time, end_time, period, dimensions, statistics)
+        set_status(data, alarm_tree)
+        file = AwsReporting::Store.save(base_path, data)
+
+        identification = {:dimensions => dimensions, :region => region}
+        metrics[namespace][identification] << file
+      }
+
+      report_info = {:start_time => start_time.to_s,
+                     :end_time => end_time.to_s,
+                     :period => period.to_s,
+                     :timestamp => timestamp.to_s,
+                     :num_of_metrics => plan.length.to_s,
+                     :version => Version.get}
+
+      open(base_path + '/metrics.json', 'w'){|f|
+        f.print JSON.dump({:report_info => report_info, :metrics => transform(metrics, name_resolvers), :alarms => alarms})
+      }
+    end
+
+    def copy_template(path, force)
+      report_path = File.expand_path(path)
+      if force != true and File.exist?(path)
+        error = AwsReporting::Error::OverwriteError.new
+        error.path = report_path
+        raise error
+      end
+
+      template_path = File.expand_path('../../../template', __FILE__)
+      template_files = Dir.glob(template_path + '/*')
+      FileUtils.mkdir_p(report_path)
+      FileUtils.cp_r(template_files, report_path)
+    end
+
+    def generate
+      timestamp = Time.now
+
+      AwsReporting::Config.load()
+
+      puts 'Report generating started.'
+      copy_template(@path, @force)
+      data_dir = @path + '/data'
+
+      puts '(1/3) Planning...'
+      plan = Plan.generate
+      puts "Planning complete."
+      start_time = Time.now - 24 * 60 * 60 * 14
+      end_time = Time.now
+      period = 60 * 60
+      puts '(2/3) Building name tables...'
+      name_resolvers = AwsReporting::Resolvers.new
+      name_resolvers.init
+      puts 'Name tables were builded.'
+      puts '(3/3) Downloading metrics...'
+      download(data_dir, plan, start_time, end_time, period, name_resolvers, timestamp)
+      puts 'Downloading metrics done.'
+      puts 'Report generating complete!'
+    end
+  end
+end