riemann-aws 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 4e51ebb1396ffd7f7f8f3b7a5dad6079c4d91729
+  data.tar.gz: cb449ca629c9a39103869733739be2cd19c5e817
+SHA512:
+  metadata.gz: 466cd5dd681b6f3a3c91b7bfb2120eb5753da93c23f5230b4e44ddaf6fe41b3f6ddaf80d897e7b60a702ddf31bf0df6cc6266b9c85e0b0c4d576092d2ca6c9e4
+  data.tar.gz: ff6dfb5aadf7b91c44d745eb4ff667eee19bacfef9854d85400ec30598c2bc2c5528adb4e25e25bafcb77a29208522452dd2b1b2018c6b1f6526599857078dd5

data/LICENSE

@@ -0,0 +1,21 @@
+The MIT License
+
+Copyright (c) 2011 Kyle Kingsbury
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,42 @@
+# Riemann AWS tools
+
+Gathers various AWS services statistics and submits them to Riemann.
+
+## Getting started
+
+```
+gem install riemann-aws
+```
+
+## AWS Commands
+
+
+### Riemann AWS Billing
+
+```
+riemann-aws-billing --help
+```
+
+### Riemann AWS RDS Status
+
+```
+riemann-aws-rds-status --help
+```
+
+### Riemann AWS SQS Satus
+
+```
+riemann-aws-sqs-status --help
+```
+
+### Riemann AWS Status
+
+```
+riemann-aws-status --help
+```
+
+### Riemann ELB Metrics
+
+```
+riemann-elb-metrics --help
+```

data/bin/riemann-aws-billing

@@ -0,0 +1,78 @@
+#!/usr/bin/env ruby
+
+require 'riemann/tools'
+
+$0 = __FILE__
+
+class Riemann::Tools::AWSBilling
+  include Riemann::Tools
+  require 'fog'
+
+  opt :fog_credentials_file, "Fog credentials file", :type => String
+  opt :fog_credential, "Fog credentials to use", :type => String
+
+  opt :access_key, "AWS access key", :type => String
+  opt :secret_key, "Secret access key", :type => String
+  opt :services, "AWS services: AmazonEC2  AmazonS3  AWSDataTransfer", :type => :strings, :multi => true, :default => ["AmazonEC2", "AmazonS3", "AWSDataTransfer"]
+
+  opt :time_start, "Start time in seconds of the metrics period (2hrs ago default)", :type => Integer, :default => 7200
+  opt :time_end, "End time in seconds of the metrics period ", :type => Integer, :default => 60
+
+
+  def initialize
+    if options[:fog_credentials_file]
+      Fog.credentials_path = opts[:fog_credentials_file]
+      Fog.credential = opts[:fog_credential].to_sym
+      @cloudwatch = Fog::AWS::CloudWatch.new
+    else
+      @cloudwatch = Fog::AWS::CloudWatch.new(:aws_secret_access_key => opts[:secret_key], :aws_access_key_id => opts[:access_key])
+      @start_time = (Time.now.utc - opts[:time_start]).iso8601
+      @end_time = (Time.now.utc - opts[:time_end]).iso8601
+    end
+  end
+
+  def tick
+    opts[:services].each do |service|
+      data = @cloudwatch.get_metric_statistics({
+                                                 'Statistics' => ["Maximum"],
+                                                 'StartTime' => @start_time,
+                                                 'EndTime' => @end_time,
+                                                 'Period' => 3600,
+                                                 'Unit' => "None",
+                                                 'MetricName' => "EstimatedCharges",
+                                                 'Namespace' => "AWS/Billing",
+                                                 'Dimensions' => [
+                                                                  {
+                                                                    'Name' => "ServiceName",
+                                                                    'Value' => service
+                                                                  },
+                                                                  {
+                                                                    'Name' => "Currency",
+                                                                    'Value' => "USD"
+                                                                  }
+                                                                 ]
+                                               }).body['GetMetricStatisticsResult']['Datapoints']
+
+
+      data.each do |metrics|
+        name = "AWScloudwatch.Billing." + service
+        value = metrics["Maximum"]
+        timestamp = metrics["Timestamp"].to_i
+
+        event = {
+          host: nil,
+          service: name,
+          time: timestamp,
+          description: "AWS Estimate Charges for #{service}",
+          tags: ["aws_billing"],
+          state: "ok",
+          metric: value
+        }
+
+        report event
+      end
+    end
+  end
+end
+
+Riemann::Tools::AWSBilling.run

data/bin/riemann-aws-rds-status

@@ -0,0 +1,47 @@
+#!/usr/bin/env ruby
+
+require 'riemann/tools'
+
+$0 = __FILE__ # Let's not expose our AWS keys in the process list
+
+class Riemann::Tools::AWS
+  include Riemann::Tools
+  require 'fog'
+  require 'date'
+  require 'time'
+  require 'json'
+
+  opt :access_key, "AWS access key", :type => String
+  opt :secret_key, "Secret access key", :type => String
+  opt :region, "AWS region", :type => String, :default => 'eu-west-1'
+  opt :dbinstance_identifier, "DBInstanceIdentifier", :type => String
+  def initialize
+    abort "FATAL: specify a DB instance name, see --help for usage" unless opts[:dbinstance_identifier]
+    @cloudwatch = Fog::AWS::CloudWatch.new(:aws_access_key_id => opts[:access_key],
+                                           :aws_secret_access_key => opts[:secret_key],
+                                           :region => opts[:region])
+  end
+
+  def tick
+    time = Time.new
+    ['DatabaseConnections', 'FreeableMemory', 'FreeStorageSpace', 'NetworkReceiveThroughput', 'NetworkTransmitThroughput', 'ReadThroughput', 'CPUUtilization'].each do |metric|
+      result = @cloudwatch.get_metric_statistics({"Namespace" => 'AWS/RDS', "MetricName" => "#{metric}", "Statistics" => 'Average', "Dimensions" => [{"Name" => "DBInstanceIdentifier", "Value" => "#{opts[:dbinstance_identifier]}"}], "StartTime" => (time-120).to_time.iso8601, "EndTime" => time.to_time.iso8601, "Period" => 60})
+      metricsResult = result.data[:body]['GetMetricStatisticsResult']
+      puts JSON.dump(metricsResult)
+      if (metricsResult['Datapoints'].length>0)
+        datapoint = metricsResult['Datapoints'][0]
+        ev = {:metric => datapoint['Average'],
+              :service => "#{opts[:dbinstance_identifier]}.#{metric} (#{datapoint['Unit']})",
+              :description => JSON.dump(metricsResult),
+              :state => "ok",
+              :ttl => 300}
+
+
+        report ev
+      end
+
+    end
+  end
+end
+
+Riemann::Tools::AWS.run

data/bin/riemann-aws-sqs-status

@@ -0,0 +1,36 @@
+#!/usr/bin/env ruby
+
+require 'riemann/tools'
+
+$0 = __FILE__ # Let's not expose our AWS keys in the process list
+
+class Riemann::Tools::AWS
+  include Riemann::Tools
+  require 'fog'
+
+  opt :access_key, "AWS access key", :type => String
+  opt :secret_key, "Secret access key", :type => String
+  opt :region, "AWS region", :type => String, :default => 'us-east-1'
+  opt :queue, "SQS Queue name", :type => String
+  def initialize
+    @sqs = Fog::AWS::SQS.new(:aws_access_key_id => opts[:access_key],
+                             :aws_secret_access_key => opts[:secret_key],
+                             :region => opts[:region])
+    response = @sqs.list_queues({'QueueNamePrefix' => opts[:queue]})
+    @queue_url = response[:body]['QueueUrls'].first
+  end
+
+  def tick
+    response = @sqs.get_queue_attributes(@queue_url, 'All')
+    ['ApproximateNumberOfMessages', 'ApproximateNumberOfMessagesNotVisible'].each do |attr|
+      msg = {
+        metric: response[:body]['Attributes'][attr],
+        service: "#{opts[:queue]} #{attr}",
+        state: 'ok'
+      }
+      report msg
+    end
+  end
+end
+
+Riemann::Tools::AWS.run

data/bin/riemann-aws-status

@@ -0,0 +1,63 @@
+#!/usr/bin/env ruby
+
+require 'riemann/tools'
+
+$0 = __FILE__  # Let's not expose our AWS keys in the process list
+
+class Riemann::Tools::AWS
+  include Riemann::Tools
+  require 'fog'
+  require 'date'
+
+  opt :access_key, "AWS access key", :type => String
+  opt :secret_key, "Secret access key", :type => String
+  opt :region, "AWS region", :type => String, :default => 'eu-west-1'
+
+  opt :retirement_critical, "Number of days before retirement. Defaults to 2", :default => 2
+  opt :event_warning, "Number of days before event. Defaults to nil (i.e. when the event appears)", :default => nil
+
+  def initialize
+    @compute = Fog::Compute.new(:aws_access_key_id => opts[:access_key],
+                                :aws_secret_access_key => opts[:secret_key],
+                                :region => opts[:region],
+                                :provider => 'AWS')
+  end
+
+  def tick
+    instance_status = @compute.describe_instance_status.body["instanceStatusSet"]
+    status = instance_status.inject({}) do |acc,i|
+      acc[i.delete("instanceId")] = i
+      acc
+    end
+
+    hosts = @compute.servers.select { |s| s.state == "running" }
+      inject([status, {}]) do |(status, acc), host|
+        acc[host.private_dns_name] = status.delete(host.id); [status, acc]
+      end[1]
+
+    hosts.each do |host, status|
+      status['eventsSet'].each do |event|
+        before, after = ['notBefore', 'notAfter'].map { |k| Date.parse event[k].to_s if event[k] }
+
+        ev = {:host => host,
+              :service => "aws_instance_status",
+              :description => "#{event['code']}\n\nstart #{event['notBefore']}\nend #{event['notAfter']}\n\n#{event['description']}",
+              :state => "ok",
+              :ttl => 300}
+
+        ev2 = if (event['code'] == 'instance-retirement') and
+          Date.today >= before-opts[:retirement_critical]
+          {:state => "critical"}
+        elsif opts[:event_warning] and Date.today >= before-opts[:event_warning]
+          {:state => "warning"}
+        else
+          {:state => "warning"}
+        end
+
+        report ev.merge(ev2)
+      end
+    end
+  end
+end
+
+Riemann::Tools::AWS.run

data/bin/riemann-elb-metrics

@@ -0,0 +1,160 @@
+#!/usr/bin/env ruby
+
+require 'riemann/tools'
+
+$0 = __FILE__
+
+class Riemann::Tools::ELBMetrics
+  include Riemann::Tools
+  require 'fog'
+  require 'time'
+
+  opt :fog_credentials_file, "Fog credentials file", :type => String
+  opt :fog_credential, "Fog credentials to use", :type => String
+  opt :aws_access, "AWS Access Key", :type => String
+  opt :aws_secret, "AWS Secret Key", :type => String
+  opt :aws_region, "AWS Region", :type => String, :default => "eu-west-1"
+  opt :aws_azs, "List of AZs to aggregate against", :type => :strings, :default => [ "all_az" ]
+  opt :elbs, "List of ELBs to pull metrics from", :type => :strings, :required => true
+
+  def standard_metrics
+    # ELB metric types, from:
+    # http://docs.aws.amazon.com/AmazonCloudWatch/latest/DeveloperGuide/CW_Support_For_AWS.html#elb-metricscollected
+    metric_options = {
+      "Latency" => {
+        "Unit" => "Seconds",
+        "Statistics" => ["Maximum", "Minimum", "Average" ]
+      },
+      "RequestCount" => {
+        "Unit" => "Count",
+        "Statistics" => [ "Sum" ]
+      },
+      "HealthyHostCount" => {
+        "Units" => "Count",
+        "Statistics" => [ "Minimum", "Maximum", "Average" ]
+      },
+      "UnHealthyHostCount" => {
+        "Units" => "Count",
+        "Statistics" => [ "Minimum", "Maximum", "Average" ]
+      },
+      "HTTPCode_ELB_4XX" => {
+        "Units" => "Count",
+        "Statistics" => [ "Sum" ]
+      },
+      "HTTPCode_ELB_5XX" => {
+        "Units" => "Count",
+        "Statistics" => [ "Sum" ]
+      },
+      "HTTPCode_Backend_2XX" => {
+        "Units" => "Count",
+        "Statistics" => [ "Sum" ]
+      },
+      "HTTPCode_Backend_3XX" => {
+        "Units" => "Count",
+        "Statistics" => [ "Sum" ]
+      },
+      "HTTPCode_Backend_4XX" => {
+        "Units" => "Count",
+        "Statistics" => [ "Sum" ]
+      },
+      "HTTPCode_Backend_5XX" => {
+        "Units" => "Count",
+        "Statistics" => [ "Sum" ]
+      }
+    }
+
+    metric_options
+  end
+
+  def base_metrics
+    # get last 60 seconds
+    start_time = (Time.now.utc - 60).iso8601
+    end_time = Time.now.utc.iso8601
+
+    # The base query that all metrics would get
+    metric_base = {
+      "Namespace" => "AWS/ELB",
+      "StartTime" => start_time,
+      "EndTime" => end_time,
+      "Period" => 60,
+    }
+
+    metric_base
+  end
+
+
+  def tick
+    if options[:fog_credentials_file]
+      Fog.credentials_path = options[:fog_credentials_file]
+      Fog.credential = options[:fog_credential].to_sym
+      connection = Fog::AWS::CloudWatch.new
+    else 
+      if options[:aws_access] && options[:aws_secret]
+        connection = Fog::AWS::CloudWatch.new({
+          :aws_access_key_id => options[:aws_access],
+          :aws_secret_access_key => options[:aws_secret],
+          :region => options[:aws_region]
+        })
+      else
+        connection = Fog::AWS::CloudWatch.new({
+          :use_iam_profile => true,
+          :region => options[:aws_region]
+        })        
+      end
+    end
+
+    options[:elbs].each do |lb|
+
+      metric_options = standard_metrics
+      metric_base_options = base_metrics
+
+      options[:aws_azs].each do |az|
+        metric_options.keys.sort.each do |metric_type|
+          merged_options = metric_base_options.merge(metric_options[metric_type])
+          merged_options["MetricName"] = metric_type
+          if az == "all_az"
+            merged_options["Dimensions"] = [ { "Name" => "LoadBalancerName", "Value" => lb } ]
+          else
+            merged_options["Dimensions"] = [
+              { "Name" => "LoadBalancerName", "Value" => lb },
+              { "Name" => "AvailabilityZone" , "Value" => az}
+            ]
+          end
+
+          result = connection.get_metric_statistics(merged_options)
+
+          # "If no response codes in the category 2XX-5XX range are sent to clients within 
+          # the given time period, values for these metrics will not be recorded in CloudWatch"
+          #next if result.body["GetMetricStatisticsResult"]["Datapoints"].empty? && metric_type =~ /[2345]XX/
+          #
+          # BUG:
+          # Metrics are reported every 60 seconds, but sometimes there isn't one there yet.
+          # We can skip that, or do something else?
+          next if result.body["GetMetricStatisticsResult"]["Datapoints"].empty?
+
+          # We should only ever have a single data point
+          result.body["GetMetricStatisticsResult"]["Datapoints"][0].keys.sort.each do |stat_type|
+            next if stat_type == "Unit"
+            next if stat_type == "Timestamp"
+
+            unit = result.body["GetMetricStatisticsResult"]["Datapoints"][0]["Unit"]
+            metric = result.body["GetMetricStatisticsResult"]["Datapoints"][0][stat_type]
+            event = Hash.new
+            event = {
+              host: lb,
+              service: "elb.#{az}.#{metric_type}.#{stat_type}",
+              ttl: 60,
+              description: "#{lb} #{metric_type} #{stat_type} (#{unit})",
+              tags: [ "production", "elb_metrics" ],
+              metric: metric
+            }
+
+            report(event)
+          end
+        end
+      end
+    end
+  end
+end
+
+Riemann::Tools::ELBMetrics.run

metadata

@@ -0,0 +1,96 @@
+--- !ruby/object:Gem::Specification
+name: riemann-aws
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Kyle Kingsbury
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-01-17 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: riemann-tools
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.2.7
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.2.7
+- !ruby/object:Gem::Dependency
+  name: fog
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.4.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.4.0
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
+email: aphyr@aphyr.com
+executables:
+- riemann-aws-billing
+- riemann-aws-rds-status
+- riemann-aws-sqs-status
+- riemann-aws-status
+- riemann-elb-metrics
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE
+- README.md
+- bin/riemann-aws-billing
+- bin/riemann-aws-rds-status
+- bin/riemann-aws-sqs-status
+- bin/riemann-aws-status
+- bin/riemann-elb-metrics
+homepage: https://github.com/riemann/riemann-aws
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 1.8.7
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: riemann-aws
+rubygems_version: 2.4.5
+signing_key: 
+specification_version: 4
+summary: Submits aws stats to riemann.
+test_files: []