riemann-aws 0.1.2 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 53369cc4951c585ad4512b0f5cb8787fc62afc30
-  data.tar.gz: 1b8d9e2dce92e426dd0f3776caef75b3b2ab25de
+  metadata.gz: 199a0f07d158b41eb67f84586294c9814d5b5681
+  data.tar.gz: 0d7f7526f9dac21f0b9197c610861bc4a2f82490
 SHA512:
-  metadata.gz: 8833446b2ac85047da34dbdaa2d2a4e2f8e1fe798b0ec6ad682739dd8af15ef1d7948c551e2a0568a03ef01208b4134b8d263af6eb685ef91612952fd2bf9dce
-  data.tar.gz: 2bd11a72ed228865e285db7bc0ea05e4eeb47633f4a5adac002f35856f99b96c2363217f66dbe91381dea8a92142a0ccb2c142f92765e5b4c4155b87b88f0026
+  metadata.gz: fb5e093683424c54ebb2c52183a9770e40346cfadfbcdf566b2535e5acd575a6fc1f7c53119248810697568b9461dfb8f0e7c0ad9c869d2e9e1dfebbc5feb6bc
+  data.tar.gz: 0bceced5e54574f926ffd8449196f2bafafb083d7803361e330f513167ea73289344da2dd43fb568cd33319efb794d1ab84d2371450d0cb67f827cfea9e3181b

data/README.md

@@ -23,7 +23,7 @@ riemann-aws-billing --help
 riemann-aws-rds-status --help
 ```
 
-### Riemann AWS SQS Satus
+### Riemann AWS SQS Status
 
 ```
 riemann-aws-sqs-status --help
@@ -40,3 +40,15 @@ riemann-aws-status --help
 ```
 riemann-elb-metrics --help
 ```
+
+### Riemann S3 Status by asking Cloudwatch
+
+```
+riemann-s3-status --help
+```
+
+### Riemann S3 Count by listing objects
+
+```
+riemann-s3-list --help
+```

data/bin/riemann-aws-billing

@@ -25,7 +25,15 @@ class Riemann::Tools::AWSBilling
       Fog.credential = opts[:fog_credential].to_sym
       @cloudwatch = Fog::AWS::CloudWatch.new
     else
-      @cloudwatch = Fog::AWS::CloudWatch.new(:aws_secret_access_key => opts[:secret_key], :aws_access_key_id => opts[:access_key])
+      if opts.has_key?('secret_key') and opts.has_key?('access_key')
+        creds = {
+          :aws_secret_access_key => opts[:secret_key],
+          :aws_access_key_id => opts[:access_key]
+        }
+      else
+        creds = { :use_iam_profile => true }
+      end
+      @cloudwatch = Fog::AWS::CloudWatch.new(creds)
       @start_time = (Time.now.utc - opts[:time_start]).iso8601
       @end_time = (Time.now.utc - opts[:time_end]).iso8601
     end

data/bin/riemann-aws-rds-status

@@ -17,9 +17,16 @@ class Riemann::Tools::AWS
   opt :dbinstance_identifier, "DBInstanceIdentifier", :type => String
   def initialize
     abort "FATAL: specify a DB instance name, see --help for usage" unless opts[:dbinstance_identifier]
-    @cloudwatch = Fog::AWS::CloudWatch.new(:aws_access_key_id => opts[:access_key],
-                                           :aws_secret_access_key => opts[:secret_key],
-                                           :region => opts[:region])
+    if opts[:access_key] and opts[:secret_key]
+      creds = {
+        :aws_access_key_id => opts[:access_key],
+        :aws_secret_access_key => opts[:secret_key]
+      }
+    else
+      creds = { :use_iam_profile => true }
+    end
+    creds['region'] = opts[:region]
+    @cloudwatch = Fog::AWS::CloudWatch.new(creds)
   end
 
   def tick
@@ -27,7 +34,6 @@ class Riemann::Tools::AWS
     ['DatabaseConnections', 'FreeableMemory', 'FreeStorageSpace', 'NetworkReceiveThroughput', 'NetworkTransmitThroughput', 'ReadThroughput', 'CPUUtilization'].each do |metric|
       result = @cloudwatch.get_metric_statistics({"Namespace" => 'AWS/RDS', "MetricName" => "#{metric}", "Statistics" => 'Average', "Dimensions" => [{"Name" => "DBInstanceIdentifier", "Value" => "#{opts[:dbinstance_identifier]}"}], "StartTime" => (time-120).to_time.iso8601, "EndTime" => time.to_time.iso8601, "Period" => 60})
       metricsResult = result.data[:body]['GetMetricStatisticsResult']
-      puts JSON.dump(metricsResult)
       if (metricsResult['Datapoints'].length>0)
         datapoint = metricsResult['Datapoints'][0]
         ev = {:metric => datapoint['Average'],
@@ -44,4 +50,4 @@ class Riemann::Tools::AWS
   end
 end
 
-Riemann::Tools::AWS.run
+Riemann::Tools::AWS.run

data/bin/riemann-aws-sqs-status

@@ -13,9 +13,16 @@ class Riemann::Tools::AWS
   opt :region, "AWS region", :type => String, :default => 'us-east-1'
   opt :queue, "SQS Queue name", :type => String
   def initialize
-    @sqs = Fog::AWS::SQS.new(:aws_access_key_id => opts[:access_key],
-                             :aws_secret_access_key => opts[:secret_key],
-                             :region => opts[:region])
+    if opts.has_key?('access_key') and opts.has_key?('secret_key')
+      creds = {
+        :aws_access_key_id => opts[:access_key],
+        :aws_secret_access_key => opts[:secret_key]
+      }
+    else
+      creds = { :use_iam_profile => true }
+    end
+    creds['region'] = opts[:region]
+    @sqs = Fog::AWS::SQS.new(creds)
     response = @sqs.list_queues({'QueueNamePrefix' => opts[:queue]})
     @queue_url = response[:body]['QueueUrls'].first
   end

data/bin/riemann-aws-status

@@ -17,10 +17,17 @@ class Riemann::Tools::AWS
   opt :event_warning, "Number of days before event. Defaults to nil (i.e. when the event appears)", :default => nil
 
   def initialize
-    @compute = Fog::Compute.new(:aws_access_key_id => opts[:access_key],
-                                :aws_secret_access_key => opts[:secret_key],
-                                :region => opts[:region],
-                                :provider => 'AWS')
+    if opts.has_key?('secret_key') and opts.has_key?('access_key')
+      creds = {
+        :aws_secret_access_key => opts[:secret_key],
+        :aws_access_key_id => opts[:access_key]
+      }
+    else
+      creds = { :use_iam_profile => true }
+    end
+    creds['region'] = opts[:region]
+    creds['provider'] = 'AWS'
+    @compute = Fog::Compute.new(creds)
   end
 
   def tick

data/bin/riemann-s3-list

@@ -0,0 +1,77 @@
+#!/usr/bin/env ruby
+
+require 'riemann/tools'
+
+$0 = __FILE__
+
+class Riemann::Tools::S3Metrics
+  include Riemann::Tools
+  require 'fog'
+  require 'time'
+
+  opt :fog_credentials_file, "Fog credentials file", :type => String
+  opt :fog_credential, "Fog credentials to use", :type => String
+  opt :aws_access, "AWS Access Key", :type => String
+  opt :aws_secret, "AWS Secret Key", :type => String
+  opt :aws_region, "AWS Region", :type => String, :default => "eu-west-1"
+  opt :buckets, "Buckets to pull metrics from, multi=true", :type => String, :multi => true, :required => true
+  opt :max_objects, "Max number of objects to list before stopping to save bandwidth", :default => -1
+
+
+  def tick
+    if options[:fog_credentials_file]
+      Fog.credentials_path = options[:fog_credentials_file]
+      Fog.credential = options[:fog_credential].to_sym
+      connection = Fog::Storage.new
+    else
+      if options[:aws_access] && options[:aws_secret]
+        connection = Fog::Storage.new({
+                                          :provider => "AWS",
+                                          :aws_access_key_id => options[:aws_access],
+                                          :aws_secret_access_key => options[:aws_secret],
+                                          :region => options[:aws_region]
+                                      })
+      else
+        connection = Fog::Storage.new({
+                                          :provider => "AWS",
+                                          :use_iam_profile => true,
+                                          :region => options[:aws_region]
+                                      })
+      end
+    end
+
+    options[:buckets].each do |bucket|
+
+      directory = connection.directories.get(bucket)
+      count = 0
+      directory.files.each do |file|
+        count = count +1
+        if (options[:max_objects]>0 && count>options[:max_objects])
+          break
+        end
+      end
+      if (options[:max_objects]>0 && count>options[:max_objects])
+        event = event(bucket, "objectCount", count, "count was bigger than threshold #{options[:max_objects]}", "warning")
+        report(event)
+      else
+        event = event(bucket, "objectCount", count, "All objects counted, threshold=#{options[:max_objects]}", "ok")
+        report(event)
+      end
+    end
+  end
+
+  private
+  def event(bucket, label, metric, description, severity)
+    event = {
+        host: "bucket_#{bucket}",
+        service: "s3.#{label}",
+        ttl: 300,
+        description: "#{bucket} #{description}",
+        tags: ["s3_metrics"],
+        metric: metric,
+        state: severity
+    }
+  end
+end
+
+Riemann::Tools::S3Metrics.run

data/bin/riemann-s3-status

@@ -0,0 +1,98 @@
+#!/usr/bin/env ruby
+
+require 'riemann/tools'
+
+$0 = __FILE__
+
+class Riemann::Tools::S3Metrics
+  include Riemann::Tools
+  require 'fog'
+  require 'time'
+
+  opt :fog_credentials_file, "Fog credentials file", :type => String
+  opt :fog_credential, "Fog credentials to use", :type => String
+  opt :aws_access, "AWS Access Key", :type => String
+  opt :aws_secret, "AWS Secret Key", :type => String
+  opt :aws_region, "AWS Region", :type => String, :default => "eu-west-1"
+  opt :buckets, "Buckets to pull metrics from, multi=true", :type => String, :multi => true, :required => true
+  opt :statistic, "Statistic to retrieve, multi=true, e.g. --statistic=Average --statistic=Maximum", :type => String, :multi => true, :required => true
+
+
+  def base_metrics
+    # get last 60 seconds
+    start_time = (Time.now.utc - 3600 * 24 * 1).iso8601
+    end_time = Time.now.utc.iso8601
+
+    # The base query that all metrics would get
+    metric_base = {
+        "Namespace" => "AWS/S3",
+        "StartTime" => start_time,
+        "EndTime" => end_time,
+        "Period" => 3600,
+        "MetricName" => "NumberOfObjects",
+    }
+
+    metric_base
+  end
+
+
+  def tick
+    if options[:fog_credentials_file]
+      Fog.credentials_path = options[:fog_credentials_file]
+      Fog.credential = options[:fog_credential].to_sym
+      connection = Fog::AWS::CloudWatch.new
+    else
+      if options[:aws_access] && options[:aws_secret]
+        connection = Fog::AWS::CloudWatch.new({
+                                                  :aws_access_key_id => options[:aws_access],
+                                                  :aws_secret_access_key => options[:aws_secret],
+                                                  :region => options[:aws_region]
+                                              })
+      else
+        connection = Fog::AWS::CloudWatch.new({
+                                                  :use_iam_profile => true,
+                                                  :region => options[:aws_region]
+                                              })
+      end
+    end
+
+    options[:statistic].each do |statistic|
+      options[:buckets].each do |bucket|
+
+        metric_base_options = base_metrics
+        metric_base_options["Statistics"] = statistic
+        metric_base_options["Dimensions"] = [
+            {"Name" => "BucketName", "Value" => bucket},
+            {"Name" => "StorageType", "Value" => "AllStorageTypes"}]
+
+        result = connection.get_metric_statistics(metric_base_options)
+        if result.body["GetMetricStatisticsResult"]["Datapoints"].empty?
+          next
+        end
+          result.body["GetMetricStatisticsResult"]["Datapoints"][0].keys.sort.each do |stat_type|
+          next if stat_type == "Unit"
+          next if stat_type == "Timestamp"
+
+          unit = result.body["GetMetricStatisticsResult"]["Datapoints"][0]["Unit"]
+          metric = result.body["GetMetricStatisticsResult"]["Datapoints"][0][stat_type]
+          event = event(bucket, result.body["GetMetricStatisticsResult"]["Label"], stat_type, unit, metric)
+          report(event)
+        end
+      end
+    end
+  end
+
+  private
+  def event(bucket, label, metric_type, stat_type, metric, unit=nil)
+    event = {
+        host: "bucket_#{bucket}",
+        service: "s3.#{label}.#{metric_type}.#{stat_type}",
+        ttl: 300,
+        description: "#{bucket} #{metric_type} #{stat_type} (#{unit})",
+        tags: ["s3_metrics"],
+        metric: metric
+    }
+  end
+end
+
+Riemann::Tools::S3Metrics.run

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: riemann-aws
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.3
 platform: ruby
 authors:
 - Kyle Kingsbury
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-03-15 00:00:00.000000000 Z
+date: 2017-04-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: riemann-tools
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.2.8
+        version: 0.2.11
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.2.8
+        version: 0.2.11
 - !ruby/object:Gem::Dependency
   name: fog
   requirement: !ruby/object:Gem::Requirement
@@ -60,6 +60,8 @@ executables:
 - riemann-aws-sqs-status
 - riemann-aws-status
 - riemann-elb-metrics
+- riemann-s3-list
+- riemann-s3-status
 extensions: []
 extra_rdoc_files: []
 files:
@@ -70,6 +72,8 @@ files:
 - bin/riemann-aws-sqs-status
 - bin/riemann-aws-status
 - bin/riemann-elb-metrics
+- bin/riemann-s3-list
+- bin/riemann-s3-status
 homepage: https://github.com/riemann/riemann-tools
 licenses:
 - MIT
@@ -90,7 +94,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: riemann-aws
-rubygems_version: 2.4.5
+rubygems_version: 2.5.2
 signing_key: 
 specification_version: 4
 summary: Submits AWS stats to riemann.