aws-reporting 0.9.1

data/lib/aws-reporting/helper.rb

@@ -0,0 +1,3 @@
+def get_value(dimension, name)
+  dimension.find{|d| d[:name] == name}[:value]
+end

data/lib/aws-reporting/plan.rb

@@ -0,0 +1,57 @@
+module AwsReporting
+  module Plan
+    def self.to_array(yaml_array)
+      array = []
+      yaml_array.each{|i|
+        array << i.to_s
+      }
+      array
+    end
+
+    def self.build_template(yaml)
+      template = Hash.new{|h, k| h[k] = {}}
+      yaml.each{|d|
+        template[d["namespace"]][d["metric_name"]] = to_array(d["statistics"])
+      }
+      template
+    end
+
+    DEFAULT_STATISTICS = ['Maximum', 'Minimum', 'Average']
+
+    def self.create_plan(template, metrics)
+      plan = []
+      metrics.map{|m|
+        namespace = m[:namespace]
+        metric_name = m[:metric_name]
+        dimensions = m[:dimensions]
+        statiestics = nil
+        if template[namespace][metric_name]
+          statistics = template[namespace][metric_name]
+        else
+          $stderr.puts "Warning: Metric #{namespace} #{metric_name} is not defined. Default statistics are used."
+          statistics = DEFAULT_STATISTICS
+        end
+        region = m[:region]
+
+        {:namespace => namespace, :metric_name => metric_name, :dimensions => dimensions, :statistics => statistics, :region => region}
+      }
+    end
+
+    def self.metrics_file_path()
+      File.expand_path('../../../metrics/metrics.yaml', __FILE__)
+    end
+
+    def self.generate()
+      yaml = YAML.load(open(metrics_file_path()){|f| f.read })
+      template = build_template(yaml)
+
+      metrics = []
+      AWS.regions.each{|r|
+        AwsReporting::Config.update_region(r.name)
+        AWS::CloudWatch::MetricCollection.new.each{|m| metrics << {:namespace => m.namespace, :metric_name => m.metric_name, :dimensions => m.dimensions, :region => r.name}}
+      }
+
+      plan = create_plan(template, metrics)
+    end
+  end
+end

data/lib/aws-reporting/resolver/ebs.rb

@@ -0,0 +1,32 @@
+module AwsReporting
+  module Resolver
+    class EBSResolver
+      def initialize
+        @ebs_name_table = {}
+      end
+
+      def self.namespace
+        "AWS/EBS"
+      end
+
+      def self.dimension_type
+        ["VolumeId"].sort
+      end
+
+      def init
+        AWS.regions.each{|r|
+          Config.update_region(r.name)
+          ec2 = AWS::EC2.new
+          ec2.volumes.each{|volume|
+            @ebs_name_table[volume.id] = volume.tags["Name"]
+          }
+        }
+      end
+
+      def get_name(element)
+        id = get_value(element[:dimensions], "VolumeId")
+        @ebs_name_table[id]
+      end
+    end
+  end
+end

data/lib/aws-reporting/resolver/ec2.rb

@@ -0,0 +1,32 @@
+module AwsReporting
+  module Resolver
+    class EC2Resolver
+      def initialize
+        @ec2_name_table = {}
+      end
+
+      def self.namespace
+        "AWS/EC2"
+      end
+
+      def self.dimension_type
+        ["InstanceId"].sort
+      end
+
+      def init
+        AWS.regions.each{|r|
+          Config.update_region(r.name)
+          ec2 = AWS::EC2.new
+          ec2.instances.each{|instance|
+            @ec2_name_table[instance.id] = instance.tags["Name"]
+          }
+        }
+      end
+
+      def get_name(element)
+        id = get_value(element[:dimensions], "InstanceId")
+        @ec2_name_table[id]
+      end
+    end
+  end
+end

data/lib/aws-reporting/resolvers.rb

@@ -0,0 +1,29 @@
+module AwsReporting
+  class Resolvers
+    def initialize
+      @resolvers = Hash.new{|h, k| h[k] = {}}
+    end
+
+    def init
+      resolvers = [AwsReporting::Resolver::EC2Resolver,
+                   AwsReporting::Resolver::EBSResolver]
+      
+      resolvers.each{|resolver_class|
+        resolver = resolver_class.new
+        resolver.init
+        @resolvers[resolver_class.namespace][resolver_class.dimension_type] = resolver
+      }
+    end
+
+    def get_name(element)
+      namespace = element[:namespace]
+      dimension_type = element[:dimensions].map{|d| d[:name]}.sort
+
+      resolver = @resolvers[namespace][dimension_type]
+
+      return nil unless resolver
+
+      resolver.get_name(element)
+    end
+  end
+end

data/lib/aws-reporting/server.rb

@@ -0,0 +1,18 @@
+require 'webrick'
+
+module AwsReporting
+  class Server
+    def initialize(path, port)
+      @server = WEBrick::HTTPServer.new(:DocumentRoot => path, :Port => port, :BindAddress => "0.0.0.0")
+    end
+
+    def start()
+      @server.start
+    end
+
+    def stop()
+      puts 'server stopped.'
+      @server.stop
+    end
+  end
+end

data/lib/aws-reporting/statistics.rb

@@ -0,0 +1,43 @@
+module AwsReporting
+  module Statistics
+    def get(region, namespace, metric_name, start_time, end_time, period, dimensions, statistics)
+      client = AWS::CloudWatch::Client.new(:config => Config.get, :region => region)
+
+      datapoints = client.get_metric_statistics(:namespace => namespace,
+                                                :metric_name => metric_name,
+                                                :start_time => start_time.iso8601,
+                                                :end_time => end_time.iso8601,
+                                                :period => period,
+                                                :dimensions => dimensions,
+                                                :statistics => statistics)[:datapoints]
+
+      info = {:region => region,
+              :namespace => namespace,
+              :metric_name => metric_name,
+              :dimensions => dimensions,
+              :start_time => start_time.iso8601,
+              :end_time => end_time.iso8601,
+              :period => period,
+              :statistics => statistics,
+              :unit => datapoints[0] ? datapoints[0][:unit] : nil }
+
+      data = datapoints.sort_by{|d| d[:timestamp] }
+                       .map{|d| 
+                          datapoint = {}
+
+                          datapoint[:timestamp] = d[:timestamp].iso8601
+                          datapoint[:maximum] = d[:maximum] if d[:maximum]
+                          datapoint[:minimum] = d[:minimum] if d[:minimum]
+                          datapoint[:average] = d[:average] if d[:average]
+                          datapoint[:sum] = d[:sum] if d[:sum]
+                          datapoint[:sample_count] = d[:sample_count] if d[:sample_count]
+
+                          datapoint
+                       }
+
+      {:info => info, :datapoints => data}
+    end
+
+    module_function :get
+  end
+end

data/lib/aws-reporting/store.rb

@@ -0,0 +1,35 @@
+module AwsReporting
+  module Store
+    class Counter
+      def initialize
+        @counter = 0
+        @mutex = Mutex.new
+      end
+
+      def get
+        @mutex.synchronize{
+          @counter += 1
+          return @counter
+        }
+      end
+    end
+
+    FILE_COUNTER = Counter.new
+
+    def save(dir, data)
+      FileUtils.mkdir_p(dir)
+
+      path = dir + '/' + FILE_COUNTER.get.to_s + '.json'
+
+      json = JSON.dump(data)
+
+      open(path, 'w'){|f|
+        f.print JSON.dump(data)
+      }
+
+      {:info => data[:info], :path => path}
+    end
+
+    module_function :save
+  end
+end

data/lib/aws-reporting/version.rb

@@ -0,0 +1,9 @@
+module AwsReporting
+  module Version
+    def get
+      "AWS-Reporting v0.9.1"
+    end
+
+    module_function :get
+  end
+end

data/metrics/metrics.csv

@@ -0,0 +1,220 @@
+AWS/AutoScaling,GroupDesiredCapacity,AVG
+AWS/AutoScaling,GroupInServiceInstances,AVG
+AWS/AutoScaling,GroupMaxSize,AVG
+AWS/AutoScaling,GroupMinSize,AVG
+AWS/AutoScaling,GroupPendingInstances,AVG
+AWS/AutoScaling,GroupTerminatingInstances,AVG
+AWS/AutoScaling,GroupTotalInstances,AVG
+AWS/Billing,EstimatedCharges,AVG
+AWS/DynamoDB,ConsumedReadCapacityUnits,SUM
+AWS/DynamoDB,ProvisionedReadCapacityUnits,AVG
+AWS/DynamoDB,ProvisionedWriteCapacityUnits,AVG
+AWS/DynamoDB,ReadThrottleEvents,SUM
+AWS/DynamoDB,ReturnedItemCount,SUM
+AWS/DynamoDB,SuccessfulRequestLatency,AVG
+AWS/DynamoDB,SystemErrors,SUM
+AWS/DynamoDB,ThrottledRequests,SUM
+AWS/DynamoDB,UserErrors,SUM
+AWS/DynamoDB,WriteThrottleEvents,SUM
+AWS/EBS,VolumeConsumedReadWriteOps,AVG
+AWS/EBS,VolumeIdleTime,SUM
+AWS/EBS,VolumeQueueLength,AVG
+AWS/EBS,VolumeReadBytes,SUM
+AWS/EBS,VolumeReadOps,SUM
+AWS/EBS,VolumeThroughputPercentage,AVG
+AWS/EBS,VolumeTotalReadTime,SUM
+AWS/EBS,VolumeTotalWriteTime,SUM
+AWS/EBS,VolumeWriteBytes,SUM
+AWS/EBS,VolumeWriteOps,SUM
+AWS/EC2,CPUCreditBalance,AVG
+AWS/EC2,CPUCreditUsage,AVG
+AWS/EC2,CPUUtilization,AVG
+AWS/EC2,DiskReadBytes,SUM
+AWS/EC2,DiskReadOps,SUM
+AWS/EC2,DiskWriteBytes,SUM
+AWS/EC2,DiskWriteOps,SUM
+AWS/EC2,NetworkIn,SUM
+AWS/EC2,NetworkOut,SUM
+AWS/EC2,StatusCheckFailed_Instance,SUM
+AWS/EC2,StatusCheckFailed,SUM
+AWS/EC2,StatusCheckFailed_System,SUM
+AWS/ElastiCache,BytesReadIntoMemcached,SUM
+AWS/ElastiCache,BytesUsedForCache,AVG
+AWS/ElastiCache,BytesUsedForCacheItems,AVG
+AWS/ElastiCache,BytesUsedForHash,AVG
+AWS/ElastiCache,BytesWrittenOutFromMemcached,SUM
+AWS/ElastiCache,CacheHits,SUM
+AWS/ElastiCache,CacheMisses,SUM
+AWS/ElastiCache,CasBadval,SUM
+AWS/ElastiCache,CasHits,SUM
+AWS/ElastiCache,CasMisses,SUM
+AWS/ElastiCache,CmdConfigGet,SUM
+AWS/ElastiCache,CmdConfigSet,SUM
+AWS/ElastiCache,CmdFlush,SUM
+AWS/ElastiCache,CmdGet,SUM
+AWS/ElastiCache,CmdSet,SUM
+AWS/ElastiCache,CmdTouch,SUM
+AWS/ElastiCache,CPUUtilization,AVG
+AWS/ElastiCache,CurrConfig,AVG
+AWS/ElastiCache,CurrConnections,AVG
+AWS/ElastiCache,CurrItems,AVG
+AWS/ElastiCache,DecrHits,SUM
+AWS/ElastiCache,DecrMisses,SUM
+AWS/ElastiCache,DeleteHits,SUM
+AWS/ElastiCache,DeleteMisses,SUM
+AWS/ElastiCache,EvictedUnfetched,SUM
+AWS/ElastiCache,Evictions,AVG
+AWS/ElastiCache,Evictions,SUM
+AWS/ElastiCache,ExpiredUnfetched,SUM
+AWS/ElastiCache,FreeableMemory,AVG
+AWS/ElastiCache,GetHits,SUM
+AWS/ElastiCache,GetMisses,SUM
+AWS/ElastiCache,GetTypeCmds,SUM
+AWS/ElastiCache,HashBasedCmds,SUM
+AWS/ElastiCache,IncrHits,SUM
+AWS/ElastiCache,IncrMisses,SUM
+AWS/ElastiCache,KeyBasedCmds,SUM
+AWS/ElastiCache,ListBasedCmds,SUM
+AWS/ElastiCache,NetworkBytesIn,AVG
+AWS/ElastiCache,NetworkBytesOut,AVG
+AWS/ElastiCache,NewConnections,AVG
+AWS/ElastiCache,NewConnections,SUM
+AWS/ElastiCache,NewItems,SUM
+AWS/ElastiCache,Reclaimed,AVG
+AWS/ElastiCache,Reclaimed,SUM
+AWS/ElastiCache,ReplicationLag,AVG
+AWS/ElastiCache,SetBasedCmds,SUM
+AWS/ElastiCache,SetTypeCmds,SUM
+AWS/ElastiCache,SlabsMoved,SUM
+AWS/ElastiCache,SortedSetBasedCmds,SUM
+AWS/ElastiCache,StringBasedCmds,SUM
+AWS/ElastiCache,SwapUsage,AVG
+AWS/ElastiCache,TouchHits,SUM
+AWS/ElastiCache,TouchMisses,SUM
+AWS/ElastiCache,UnusedMemory,AVG
+AWS/ElasticMapReduce,CoreNodesPending,AVG
+AWS/ElasticMapReduce,CoreNodesRunning,AVG
+AWS/ElasticMapReduce,HbaseBackupFailed,SUM
+AWS/ElasticMapReduce,HbaseMostRecentBackupDuration,AVG
+AWS/ElasticMapReduce,HbaseTimeSinceLastSuccessfulBackup,AVG
+AWS/ElasticMapReduce,HDFSBytesRead,SUM
+AWS/ElasticMapReduce,HDFSBytesWritten,SUM
+AWS/ElasticMapReduce,HDFSUtilization,AVG
+AWS/ElasticMapReduce,IsIdle,SUM
+AWS/ElasticMapReduce,JobsFailed,SUM
+AWS/ElasticMapReduce,JobsRunning,SUM
+AWS/ElasticMapReduce,LiveDataNodes,AVG
+AWS/ElasticMapReduce,LiveTaskTrackers,AVG
+AWS/ElasticMapReduce,MapSlotsOpen,AVG
+AWS/ElasticMapReduce,MissingBlocks,AVG
+AWS/ElasticMapReduce,ReduceSlotsOpen,AVG
+AWS/ElasticMapReduce,RemainingMapTasks,AVG
+AWS/ElasticMapReduce,RemainingMapTasksPerSlot,AVG
+AWS/ElasticMapReduce,RemainingReduceTasks,AVG
+AWS/ElasticMapReduce,RunningMapTasks,AVG
+AWS/ElasticMapReduce,RunningReduceTasks,AVG
+AWS/ElasticMapReduce,S3BytesRead,SUM
+AWS/ElasticMapReduce,S3BytesWritten,SUM
+AWS/ElasticMapReduce,TaskNodesPending,AVG
+AWS/ElasticMapReduce,TaskNodesRunning,AVG
+AWS/ElasticMapReduce,TotalLoad,AVG
+AWS/ELB,BackendConnectionErrors,SUM
+AWS/ELB,HealthyHostCount,AVG
+AWS/ELB,HTTPCode_Backend_2XX,SUM
+AWS/ELB,HTTPCode_Backend_3XX,SUM
+AWS/ELB,HTTPCode_Backend_4XX,SUM
+AWS/ELB,HTTPCode_Backend_5XX,SUM
+AWS/ELB,HTTPCode_ELB_4XX,SUM
+AWS/ELB,HTTPCode_ELB_5XX,SUM
+AWS/ELB,Latency,AVG
+AWS/ELB,RequestCount,SUM
+AWS/ELB,SpilloverCount,SUM
+AWS/ELB,SurgeQueueLength,AVG
+AWS/ELB,UnHealthyHostCount,AVG
+AWS/OpsWorks,cpu_idle,AVG
+AWS/OpsWorks,cpu_nice,AVG
+AWS/OpsWorks,cpu_system,AVG
+AWS/OpsWorks,cpu_user,AVG
+AWS/OpsWorks,cpu_waitio,AVG
+AWS/OpsWorks,Load_15,AVG
+AWS/OpsWorks,Load_1,AVG
+AWS/OpsWorks,Load_5,AVG
+AWS/OpsWorks,memory_buffers,AVG
+AWS/OpsWorks,memory_cached,AVG
+AWS/OpsWorks,memory_free,AVG
+AWS/OpsWorks,memory_swap,AVG
+AWS/OpsWorks,memory_total,AVG
+AWS/OpsWorks,memory_used,AVG
+AWS/OpsWorks,procs,AVG
+AWS/RDS,BinLogDiskUsage,AVG
+AWS/RDS,CPUUtilization,AVG
+AWS/RDS,DatabaseConnections,AVG
+AWS/RDS,DiskQueueDepth,AVG
+AWS/RDS,FreeableMemory,AVG
+AWS/RDS,FreeStorageSpace,AVG
+AWS/RDS,NetworkReceiveThroughput,AVG
+AWS/RDS,NetworkTransmitThroughput,AVG
+AWS/RDS,ReadIOPS,AVG
+AWS/RDS,ReadLatency,AVG
+AWS/RDS,ReadThroughput,AVG
+AWS/RDS,ReplicaLag,AVG
+AWS/RDS,SwapUsage,AVG
+AWS/RDS,WriteIOPS,AVG
+AWS/RDS,WriteLatency,AVG
+AWS/RDS,WriteThroughput,AVG
+AWS/Redshift,CPUUtilization,AVG
+AWS/Redshift,DatabaseConnections,AVG
+AWS/Redshift,HealthStatus,AVG
+AWS/Redshift,MaintenanceMode,AVG
+AWS/Redshift,NetworkReceiveThroughput,AVG
+AWS/Redshift,NetworkTransmitThroughput,AVG
+AWS/Redshift,PercentageDiskSpaceUsed,AVG
+AWS/Redshift,ReadIOPS,AVG
+AWS/Redshift,ReadLatency,AVG
+AWS/Redshift,ReadThroughput,AVG
+AWS/Redshift,WriteIOPS,AVG
+AWS/Redshift,WriteLatency,AVG
+AWS/Redshift,WriteThroughput,AVG
+AWS/Route53,HealthCheckStatus,AVG
+AWS/SNS,NumberOfMessagesPublished,SUM
+AWS/SNS,NumberOfNotificationsDelivered,SUM
+AWS/SNS,NumberOfNotificationsFailed,SUM
+AWS/SNS,PublishSize,AVG
+AWS/SQS,ApproximateNumberOfMessagesDelayed,AVG
+AWS/SQS,ApproximateNumberOfMessagesNotVisible,AVG
+AWS/SQS,ApproximateNumberOfMessagesVisible,AVG
+AWS/SQS,NumberOfEmptyReceives,SUM
+AWS/SQS,NumberOfMessagesDeleted,SUM
+AWS/SQS,NumberOfMessagesReceived,SUM
+AWS/SQS,NumberOfMessagesSent,SUM
+AWS/SQS,SentMessageSize,AVG
+AWS/StorageGateway,CacheHitPercent,AVG
+AWS/StorageGateway,CachePercentDirty,AVG
+AWS/StorageGateway,CachePercentUsed,AVG
+AWS/StorageGateway,CloudBytesDownloaded,SUM
+AWS/StorageGateway,CloudBytesUploaded,SUM
+AWS/StorageGateway,CloudDownloadLatency,AVG
+AWS/StorageGateway,QueuedWrites,AVG
+AWS/StorageGateway,ReadBytes,SUM
+AWS/StorageGateway,ReadTime,AVG
+AWS/StorageGateway,TotalCacheSize,AVG
+AWS/StorageGateway,UploadBufferFree,AVG
+AWS/StorageGateway,UploadBufferPercentUsed,AVG
+AWS/StorageGateway,UploadBufferUsed,AVG
+AWS/StorageGateway,WorkingStorageFree,AVG
+AWS/StorageGateway,WorkingStoragePercentageUsed,AVG
+AWS/StorageGateway,WorkingStoragePercentUsed,AVG
+AWS/StorageGateway,WorkingStorageUsed,AVG
+AWS/StorageGateway,WriteBytes,SUM
+AWS/StorageGateway,WriteTime,AVG
+AWS/SWF,DecisionTaskScheduleToStartTime,AVG
+AWS/SWF,DecisionTasksCompleted,SUM
+AWS/SWF,DecisionTaskStartToCloseTime,AVG
+AWS/SWF,StartedDecisionTasksTimedOutOnClose,SUM
+AWS/SWF,WorkflowsCanceled,SUM
+AWS/SWF,WorkflowsCompleted,SUM
+AWS/SWF,WorkflowsContinuedAsNew,SUM
+AWS/SWF,WorkflowsFailed,SUM
+AWS/SWF,WorkflowStartToCloseTime,AVG
+AWS/SWF,WorkflowsTerminated,SUM
+AWS/SWF,WorkflowsTimedOut,SUM