riemann-tools 0.2.13 → 1.1.0

data/riemann-tools.gemspec

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require_relative 'lib/riemann/tools/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'riemann-tools'
+  spec.version       = Riemann::Tools::VERSION
+  spec.authors       = ['Kyle Kingsbury']
+  spec.email         = ['aphyr@aphyr.com']
+
+  spec.summary       = 'Utilities which submit events to Riemann.'
+  spec.description   = 'Collection of utilities which submit events to Riemann,'
+  spec.homepage      = 'https://github.com/aphyr/riemann-tools'
+  spec.license       = 'MIT'
+  spec.required_ruby_version = Gem::Requirement.new('>= 2.6.0')
+
+  spec.metadata['allowed_push_host'] = 'https://rubygems.org/'
+
+  spec.metadata['homepage_uri']    = spec.homepage
+  spec.metadata['source_code_uri'] = spec.homepage
+  spec.metadata['changelog_uri']   = spec.homepage
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files = Dir.chdir(File.expand_path(__dir__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  end
+  spec.bindir        = 'bin'
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.add_runtime_dependency 'json', '>= 1.8'
+  spec.add_runtime_dependency 'optimist', '~> 3.0', '>= 3.0.0'
+  spec.add_runtime_dependency 'riemann-client', '~> 1.0'
+
+  spec.add_development_dependency 'github_changelog_generator'
+  spec.add_development_dependency 'rake'
+  spec.add_development_dependency 'rspec'
+  spec.add_development_dependency 'rubocop'
+  spec.add_development_dependency 'rubocop-rake'
+  spec.add_development_dependency 'rubocop-rspec'
+end

data/tools/riemann-aws/LICENSE

@@ -0,0 +1,21 @@
+The MIT License
+
+Copyright (c) 2011 Kyle Kingsbury
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/tools/riemann-aws/README.md

@@ -0,0 +1,54 @@
+# Riemann AWS tools
+
+Gathers various AWS services statistics and submits them to Riemann.
+
+## Getting started
+
+```
+gem install riemann-aws
+```
+
+## AWS Commands
+
+
+### Riemann AWS Billing
+
+```
+riemann-aws-billing --help
+```
+
+### Riemann AWS RDS Status
+
+```
+riemann-aws-rds-status --help
+```
+
+### Riemann AWS SQS Status
+
+```
+riemann-aws-sqs-status --help
+```
+
+### Riemann AWS Status
+
+```
+riemann-aws-status --help
+```
+
+### Riemann ELB Metrics
+
+```
+riemann-elb-metrics --help
+```
+
+### Riemann S3 Status by asking Cloudwatch
+
+```
+riemann-s3-status --help
+```
+
+### Riemann S3 Count by listing objects
+
+```
+riemann-s3-list --help
+```

data/tools/riemann-aws/Rakefile

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require 'rubygems'
+require 'rubygems/package_task'
+require 'rdoc/task'
+require 'find'
+
+# Don't include resource forks in tarballs on Mac OS X.
+ENV['COPY_EXTENDED_ATTRIBUTES_DISABLE'] = 'true'
+ENV['COPYFILE_DISABLE'] = 'true'
+
+# Gemspec
+gemspec = Gem::Specification.new do |s|
+  s.rubyforge_project = 'riemann-aws'
+
+  s.name = 'riemann-aws'
+  s.version = '0.1.4'
+  s.author = 'Kyle Kingsbury'
+  s.email = 'aphyr@aphyr.com'
+  s.homepage = 'https://github.com/riemann/riemann-tools'
+  s.platform = Gem::Platform::RUBY
+  s.summary = 'Submits AWS stats to riemann.'
+  s.license = 'MIT'
+
+  s.add_dependency 'riemann-tools', '>= 0.2.13'
+  s.add_dependency 'fog', '>= 1.4.0'
+  s.add_dependency 'json'
+
+  s.files = FileList['bin/*', 'LICENSE', 'README.md'].to_a
+  s.executables |= Dir.entries('bin/')
+  s.has_rdoc = false
+
+  s.required_ruby_version = '>= 1.8.7'
+end
+
+Gem::PackageTask.new gemspec do |p|
+end

data/tools/riemann-aws/bin/riemann-aws-billing

@@ -0,0 +1,93 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+Process.setproctitle($PROGRAM_NAME)
+
+require 'riemann/tools'
+
+$0 = __FILE__
+
+module Riemann
+  module Tools
+    class AWSBilling
+      include Riemann::Tools
+      require 'fog'
+
+      opt :fog_credentials_file, 'Fog credentials file', type: String
+      opt :fog_credential, 'Fog credentials to use', type: String
+
+      opt :access_key, 'AWS access key', type: String
+      opt :secret_key, 'Secret access key', type: String
+      opt :services, 'AWS services: AmazonEC2  AmazonS3  AWSDataTransfer', type: :strings, multi: true,
+                                                                           default: %w[AmazonEC2 AmazonS3 AWSDataTransfer]
+
+      opt :time_start, 'Start time in seconds of the metrics period (2hrs ago default)', type: Integer,
+                                                                                         default: 7200
+      opt :time_end, 'End time in seconds of the metrics period ', type: Integer, default: 60
+
+      def initialize
+        if options[:fog_credentials_file]
+          Fog.credentials_path = opts[:fog_credentials_file]
+          Fog.credential = opts[:fog_credential].to_sym
+          @cloudwatch = Fog::AWS::CloudWatch.new
+        else
+          creds = if opts.key?('secret_key') && opts.key?('access_key')
+                    {
+                      aws_secret_access_key: opts[:secret_key],
+                      aws_access_key_id: opts[:access_key],
+                    }
+                  else
+                    { use_iam_profile: true }
+                  end
+          @cloudwatch = Fog::AWS::CloudWatch.new(creds)
+        end
+        @start_time = (Time.now.utc - opts[:time_start]).iso8601
+        @end_time = (Time.now.utc - opts[:time_end]).iso8601
+      end
+
+      def tick
+        opts[:services].each do |service|
+          data = @cloudwatch.get_metric_statistics({
+                                                     'Statistics' => ['Maximum'],
+                                                     'StartTime'  => @start_time,
+                                                     'EndTime'    => @end_time,
+                                                     'Period'     => 3600,
+                                                     'Unit'       => 'None',
+                                                     'MetricName' => 'EstimatedCharges',
+                                                     'Namespace'  => 'AWS/Billing',
+                                                     'Dimensions' => [
+                                                       {
+                                                         'Name'  => 'ServiceName',
+                                                         'Value' => service,
+                                                       },
+                                                       {
+                                                         'Name'  => 'Currency',
+                                                         'Value' => 'USD',
+                                                       },
+                                                     ],
+                                                   }).body['GetMetricStatisticsResult']['Datapoints']
+
+          data.each do |metrics|
+            name = "AWScloudwatch.Billing.#{service}"
+            value = metrics['Maximum']
+            timestamp = metrics['Timestamp'].to_i
+
+            event = {
+              host: nil,
+              service: name,
+              time: timestamp,
+              description: "AWS Estimate Charges for #{service}",
+              tags: ['aws_billing'],
+              state: 'ok',
+              metric: value,
+            }
+
+            report event
+          end
+        end
+      end
+    end
+  end
+end
+
+Riemann::Tools::AWSBilling.run

data/tools/riemann-aws/bin/riemann-aws-rds-status

@@ -0,0 +1,68 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+Process.setproctitle($PROGRAM_NAME)
+
+require 'riemann/tools'
+
+$0 = __FILE__ # Let's not expose our AWS keys in the process list
+
+module Riemann
+  module Tools
+    class AWS
+      include Riemann::Tools
+      require 'fog'
+      require 'date'
+      require 'time'
+      require 'json'
+
+      opt :access_key, 'AWS access key', type: String
+      opt :secret_key, 'Secret access key', type: String
+      opt :region, 'AWS region', type: String, default: 'eu-west-1'
+      opt :dbinstance_identifier, 'DBInstanceIdentifier', type: String
+      def initialize
+        abort 'FATAL: specify a DB instance name, see --help for usage' unless opts[:dbinstance_identifier]
+        creds = if opts[:access_key] && opts[:secret_key]
+                  {
+                    aws_access_key_id: opts[:access_key],
+                    aws_secret_access_key: opts[:secret_key],
+                  }
+                else
+                  { use_iam_profile: true }
+                end
+        creds['region'] = opts[:region]
+        @cloudwatch = Fog::AWS::CloudWatch.new(creds)
+      end
+
+      def tick
+        time = Time.new
+        %w[DatabaseConnections FreeableMemory FreeStorageSpace NetworkReceiveThroughput
+           NetworkTransmitThroughput ReadThroughput CPUUtilization].each do |metric|
+          result = @cloudwatch.get_metric_statistics(
+            'Namespace'  => 'AWS/RDS',
+            'MetricName' => metric.to_s,
+            'Statistics' => 'Average',
+            'Dimensions' => [{ 'Name' => 'DBInstanceIdentifier', 'Value' => opts[:dbinstance_identifier].to_s }],
+            'StartTime'  => (time - 120).to_time.iso8601,
+            'EndTime'    => time.to_time.iso8601, 'Period' => 60,
+          )
+          metrics_result = result.data[:body]['GetMetricStatisticsResult']
+          next unless metrics_result['Datapoints'].length.positive?
+
+          datapoint = metrics_result['Datapoints'][0]
+          ev = {
+            metric: datapoint['Average'],
+            service: "#{opts[:dbinstance_identifier]}.#{metric} (#{datapoint['Unit']})",
+            description: JSON.dump(metrics_result),
+            state: 'ok',
+            ttl: 300,
+          }
+
+          report ev
+        end
+      end
+    end
+  end
+end
+
+Riemann::Tools::AWS.run

data/tools/riemann-aws/bin/riemann-aws-sqs-status

@@ -0,0 +1,50 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+Process.setproctitle($PROGRAM_NAME)
+
+require 'riemann/tools'
+
+$0 = __FILE__ # Let's not expose our AWS keys in the process list
+
+module Riemann
+  module Tools
+    class AWS
+      include Riemann::Tools
+      require 'fog'
+
+      opt :access_key, 'AWS access key', type: String
+      opt :secret_key, 'Secret access key', type: String
+      opt :region, 'AWS region', type: String, default: 'us-east-1'
+      opt :queue, 'SQS Queue name', type: String
+      def initialize
+        creds = if opts.key?('access_key') && opts.key?('secret_key')
+                  {
+                    aws_access_key_id: opts[:access_key],
+                    aws_secret_access_key: opts[:secret_key],
+                  }
+                else
+                  { use_iam_profile: true }
+                end
+        creds['region'] = opts[:region]
+        @sqs = Fog::AWS::SQS.new(creds)
+        response = @sqs.list_queues({ 'QueueNamePrefix' => opts[:queue] })
+        @queue_url = response[:body]['QueueUrls'].first
+      end
+
+      def tick
+        response = @sqs.get_queue_attributes(@queue_url, 'All')
+        %w[ApproximateNumberOfMessages ApproximateNumberOfMessagesNotVisible].each do |attr|
+          msg = {
+            metric: response[:body]['Attributes'][attr],
+            service: "#{opts[:queue]} #{attr}",
+            state: 'ok',
+          }
+          report msg
+        end
+      end
+    end
+  end
+end
+
+Riemann::Tools::AWS.run

data/tools/riemann-aws/bin/riemann-aws-status

@@ -0,0 +1,83 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+Process.setproctitle($PROGRAM_NAME)
+
+require 'riemann/tools'
+
+$0 = __FILE__ # Let's not expose our AWS keys in the process list
+
+module Riemann
+  module Tools
+    class AWS
+      include Riemann::Tools
+      require 'fog'
+      require 'date'
+
+      opt :access_key, 'AWS access key', type: String
+      opt :secret_key, 'Secret access key', type: String
+      opt :region, 'AWS region', type: String, default: 'eu-west-1'
+
+      opt :retirement_critical, 'Number of days before retirement. Defaults to 2', default: 2
+      opt :event_warning, 'Number of days before event. Defaults to nil (i.e. when the event appears)', default: nil
+
+      def initialize
+        creds = if opts.key?('secret_key') && opts.key?('access_key')
+                  {
+                    aws_secret_access_key: opts[:secret_key],
+                    aws_access_key_id: opts[:access_key],
+                  }
+                else
+                  { use_iam_profile: true }
+                end
+        creds['region'] = opts[:region]
+        creds['provider'] = 'AWS'
+        @compute = Fog::Compute.new(creds)
+      end
+
+      def tick
+        # XXX: needs work:
+        #   * instance_status and status seems unused
+        #   * where is inject comming from
+        # It looks line the only needed line for the 2 next paragraph is the one that assigns hosts.
+        instance_status = @compute.describe_instance_status.body['instanceStatusSet']
+        status = instance_status.each_with_object({}) do |i, acc|
+          acc[i.delete('instanceId')] = i
+        end
+
+        hosts = @compute.servers.select { |s| s.state == 'running' }
+        inject([status, {}]) do |(astatus, acc), host|
+          acc[host.private_dns_name] = astatus.delete(host.id)
+          [astatus, acc]
+        end[1]
+
+        hosts.each do |host, host_status|
+          host_status['eventsSet'].each do |event|
+            before, _after = %w[notBefore notAfter].map { |k| Date.parse event[k].to_s if event[k] }
+
+            ev = {
+              host: host,
+              service: 'aws_instance_status',
+              description: "#{event['code']}\n\nstart #{event['notBefore']}\nend #{event['notAfter']}\n\n#{event['description']}",
+              state: 'ok',
+              ttl: 300,
+            }
+
+            ev2 = if (event['code'] == 'instance-retirement') &&
+                     (Date.today >= before - opts[:retirement_critical])
+                    { state: 'critical' }
+                  elsif opts[:event_warning] && (Date.today >= before - opts[:event_warning])
+                    { state: 'warning' }
+                  else
+                    { state: 'warning' }
+                  end
+
+            report ev.merge(ev2)
+          end
+        end
+      end
+    end
+  end
+end
+
+Riemann::Tools::AWS.run

data/tools/riemann-aws/bin/riemann-elb-metrics

@@ -0,0 +1,168 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+Process.setproctitle($PROGRAM_NAME)
+
+require 'riemann/tools'
+
+$0 = __FILE__
+
+module Riemann
+  module Tools
+    class ELBMetrics
+      include Riemann::Tools
+      require 'fog'
+      require 'time'
+
+      opt :fog_credentials_file, 'Fog credentials file', type: String
+      opt :fog_credential, 'Fog credentials to use', type: String
+      opt :aws_access, 'AWS Access Key', type: String
+      opt :aws_secret, 'AWS Secret Key', type: String
+      opt :aws_region, 'AWS Region', type: String, default: 'eu-west-1'
+      opt :aws_azs, 'List of AZs to aggregate against', type: :strings, default: ['all_az']
+      opt :elbs, 'List of ELBs to pull metrics from', type: :strings, required: true
+
+      def standard_metrics
+        # ELB metric types, from:
+        # http://docs.aws.amazon.com/AmazonCloudWatch/latest/DeveloperGuide/CW_Support_For_AWS.html#elb-metricscollected
+        {
+          'Latency'              => {
+            'Unit'       => 'Seconds',
+            'Statistics' => %w[Maximum Minimum Average],
+          },
+          'RequestCount'         => {
+            'Unit'       => 'Count',
+            'Statistics' => ['Sum'],
+          },
+          'HealthyHostCount'     => {
+            'Units'      => 'Count',
+            'Statistics' => %w[Minimum Maximum Average],
+          },
+          'UnHealthyHostCount'   => {
+            'Units'      => 'Count',
+            'Statistics' => %w[Minimum Maximum Average],
+          },
+          'HTTPCode_ELB_4XX'     => {
+            'Units'      => 'Count',
+            'Statistics' => ['Sum'],
+          },
+          'HTTPCode_ELB_5XX'     => {
+            'Units'      => 'Count',
+            'Statistics' => ['Sum'],
+          },
+          'HTTPCode_Backend_2XX' => {
+            'Units'      => 'Count',
+            'Statistics' => ['Sum'],
+          },
+          'HTTPCode_Backend_3XX' => {
+            'Units'      => 'Count',
+            'Statistics' => ['Sum'],
+          },
+          'HTTPCode_Backend_4XX' => {
+            'Units'      => 'Count',
+            'Statistics' => ['Sum'],
+          },
+          'HTTPCode_Backend_5XX' => {
+            'Units'      => 'Count',
+            'Statistics' => ['Sum'],
+          },
+        }
+      end
+
+      def base_metrics
+        # get last 60 seconds
+        start_time = (Time.now.utc - 60).iso8601
+        end_time = Time.now.utc.iso8601
+
+        # The base query that all metrics would get
+        {
+          'Namespace' => 'AWS/ELB',
+          'StartTime' => start_time,
+          'EndTime'   => end_time,
+          'Period'    => 60,
+        }
+      end
+
+      def tick
+        if options[:fog_credentials_file]
+          Fog.credentials_path = options[:fog_credentials_file]
+          Fog.credential = options[:fog_credential].to_sym
+          connection = Fog::AWS::CloudWatch.new
+        else
+          connection = if options[:aws_access] && options[:aws_secret]
+                         Fog::AWS::CloudWatch.new({
+                                                    aws_access_key_id: options[:aws_access],
+                                                    aws_secret_access_key: options[:aws_secret],
+                                                    region: options[:aws_region],
+                                                  })
+                       else
+                         Fog::AWS::CloudWatch.new({
+                                                    use_iam_profile: true,
+                                                    region: options[:aws_region],
+                                                  })
+                       end
+        end
+
+        options[:elbs].each do |lb|
+          metric_options = standard_metrics
+          metric_base_options = base_metrics
+
+          options[:aws_azs].each do |az|
+            metric_options.keys.sort.each do |metric_type|
+              merged_options = metric_base_options.merge(metric_options[metric_type])
+              merged_options['MetricName'] = metric_type
+              merged_options['Dimensions'] = if az == 'all_az'
+                                               [{ 'Name' => 'LoadBalancerName', 'Value' => lb }]
+                                             else
+                                               [
+                                                 { 'Name' => 'LoadBalancerName', 'Value' => lb },
+                                                 { 'Name' => 'AvailabilityZone', 'Value' => az },
+                                               ]
+                                             end
+
+              result = connection.get_metric_statistics(merged_options)
+
+              # "If no response codes in the category 2XX-5XX range are sent to clients within
+              # the given time period, values for these metrics will not be recorded in CloudWatch"
+              # next if result.body["GetMetricStatisticsResult"]["Datapoints"].empty? && metric_type =~ /[2345]XX/
+              #
+              if result.body['GetMetricStatisticsResult']['Datapoints'].empty?
+                standard_metrics[metric_type]['Statistics'].each do |stat_type|
+                  event = event(lb, az, metric_type, stat_type, 0.0)
+                  report(event)
+                end
+                next
+              end
+
+              # We should only ever have a single data point
+              result.body['GetMetricStatisticsResult']['Datapoints'][0].keys.sort.each do |stat_type|
+                next if stat_type == 'Unit'
+                next if stat_type == 'Timestamp'
+
+                unit = result.body['GetMetricStatisticsResult']['Datapoints'][0]['Unit']
+                metric = result.body['GetMetricStatisticsResult']['Datapoints'][0][stat_type]
+                event = event(lb, az, metric_type, stat_type, metric, unit)
+                report(event)
+              end
+            end
+          end
+        end
+      end
+
+      private
+
+      def event(lb, az, metric_type, stat_type, metric, unit = nil)
+        {
+          host: lb,
+          service: "elb.#{az}.#{metric_type}.#{stat_type}",
+          ttl: 60,
+          description: "#{lb} #{metric_type} #{stat_type} (#{unit})",
+          tags: ['elb_metrics'],
+          metric: metric,
+        }
+      end
+    end
+  end
+end
+
+Riemann::Tools::ELBMetrics.run