riemann-tools 1.1.0 → 1.3.0

data/tools/riemann-aws/lib/riemann/tools/aws/s3_list.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+require 'riemann/tools'
+
+module Riemann
+  module Tools
+    module Aws
+      class S3List
+        include Riemann::Tools
+        require 'fog/aws'
+        require 'time'
+
+        opt :fog_credentials_file, 'Fog credentials file', type: String
+        opt :fog_credential, 'Fog credentials to use', type: String
+        opt :access_key, 'AWS Access Key', type: String
+        opt :secret_key, 'AWS Secret Key', type: String
+        opt :region, 'AWS Region', type: String, default: 'eu-west-1'
+        opt :buckets, 'Buckets to pull metrics from, multi=true, can have a prefix like mybucket/prefix', type: String,
+                                                                                                          multi: true, required: true
+        opt :max_objects, 'Max number of objects to list before stopping to save bandwidth', default: -1
+
+        def tick
+          if options[:fog_credentials_file]
+            Fog.credentials_path = options[:fog_credentials_file]
+            Fog.credential = options[:fog_credential].to_sym
+            connection = Fog::Storage.new
+          else
+            connection = if options[:access_key] && options[:secret_key]
+                           Fog::Storage.new({
+                                              provider: 'AWS',
+                                              aws_access_key_id: options[:access_key],
+                                              aws_secret_access_key: options[:secret_key],
+                                              region: options[:region],
+                                            })
+                         else
+                           Fog::Storage.new({
+                                              provider: 'AWS',
+                                              use_iam_profile: true,
+                                              region: options[:region],
+                                            })
+                         end
+          end
+
+          options[:buckets].each do |url|
+            split = url.split('/')
+            bucket = split[0]
+            prefix = ''
+            prefix = url[(split[0].length + 1)..] if split[1]
+            count = 0
+            connection.directories.get(bucket, prefix: prefix).files.map do |_file|
+              count += 1
+              break if options[:max_objects].positive? && count > options[:max_objects]
+            end
+            event = if options[:max_objects].positive? && count > options[:max_objects]
+                      event(
+                        url, 'objectCount', count, "count was bigger than threshold #{options[:max_objects]}",
+                        'warning',
+                      )
+                    else
+                      event(url, 'objectCount', count, "All objects counted, threshold=#{options[:max_objects]}", 'ok')
+                    end
+            report(event)
+          end
+        end
+
+        private
+
+        def event(bucket, label, metric, description, severity)
+          {
+            host: "bucket_#{bucket}",
+            service: "s3.#{label}",
+            ttl: 300,
+            description: "#{bucket} #{description}",
+            tags: ['s3_metrics'],
+            metric: metric,
+            state: severity,
+          }
+        end
+      end
+    end
+  end
+end

data/tools/riemann-aws/lib/riemann/tools/aws/s3_status.rb

@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+require 'riemann/tools'
+
+module Riemann
+  module Tools
+    module Aws
+      class S3Status
+        include Riemann::Tools
+        require 'fog/aws'
+        require 'time'
+
+        opt :fog_credentials_file, 'Fog credentials file', type: String
+        opt :fog_credential, 'Fog credentials to use', type: String
+        opt :access_key, 'AWS Access Key', type: String
+        opt :secret_key, 'AWS Secret Key', type: String
+        opt :region, 'AWS Region', type: String, default: 'eu-west-1'
+        opt :buckets, 'Buckets to pull metrics from, multi=true', type: String, multi: true, required: true
+        opt :statistic, 'Statistic to retrieve, multi=true, e.g. --statistic=Average --statistic=Maximum', type: String,
+                                                                                                           multi: true, required: true
+
+        def base_metrics
+          # get last 60 seconds
+          start_time = (Time.now.utc - 3600 * 24 * 1).iso8601
+          end_time = Time.now.utc.iso8601
+
+          # The base query that all metrics would get
+          {
+            'Namespace'  => 'AWS/S3',
+            'StartTime'  => start_time,
+            'EndTime'    => end_time,
+            'Period'     => 3600,
+            'MetricName' => 'NumberOfObjects',
+          }
+        end
+
+        def tick
+          if options[:fog_credentials_file]
+            Fog.credentials_path = options[:fog_credentials_file]
+            Fog.credential = options[:fog_credential].to_sym
+            connection = Fog::AWS::CloudWatch.new
+          else
+            connection = if options[:access_key] && options[:secret_key]
+                           Fog::AWS::CloudWatch.new({
+                                                      aws_access_key_id: options[:access_key],
+                                                      aws_secret_access_key: options[:secret_key],
+                                                      region: options[:region],
+                                                    })
+                         else
+                           Fog::AWS::CloudWatch.new({
+                                                      use_iam_profile: true,
+                                                      region: options[:region],
+                                                    })
+                         end
+          end
+
+          options[:statistic].each do |statistic|
+            options[:buckets].each do |bucket|
+              metric_base_options = base_metrics
+              metric_base_options['Statistics'] = statistic
+              metric_base_options['Dimensions'] = [
+                { 'Name' => 'BucketName', 'Value' => bucket },
+                { 'Name' => 'StorageType', 'Value' => 'AllStorageTypes' },
+              ]
+
+              result = connection.get_metric_statistics(metric_base_options)
+              next if result.body['GetMetricStatisticsResult']['Datapoints'].empty?
+
+              result.body['GetMetricStatisticsResult']['Datapoints'][0].keys.sort.each do |stat_type|
+                next if stat_type == 'Unit'
+                next if stat_type == 'Timestamp'
+
+                unit = result.body['GetMetricStatisticsResult']['Datapoints'][0]['Unit']
+                metric = result.body['GetMetricStatisticsResult']['Datapoints'][0][stat_type]
+                event = event(bucket, result.body['GetMetricStatisticsResult']['Label'], stat_type, unit, metric)
+                report(event)
+              end
+            end
+          end
+        end
+
+        private
+
+        def event(bucket, label, metric_type, stat_type, metric, unit = nil)
+          {
+            host: "bucket_#{bucket}",
+            service: "s3.#{label}.#{metric_type}.#{stat_type}",
+            ttl: 300,
+            description: "#{bucket} #{metric_type} #{stat_type} (#{unit})",
+            tags: ['s3_metrics'],
+            metric: metric,
+          }
+        end
+      end
+    end
+  end
+end

data/tools/riemann-aws/lib/riemann/tools/aws/sqs_status.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+require 'riemann/tools'
+
+module Riemann
+  module Tools
+    module Aws
+      class SqsStatus
+        include Riemann::Tools
+        require 'fog/aws'
+
+        opt :access_key, 'AWS access key', type: String
+        opt :secret_key, 'Secret access key', type: String
+        opt :region, 'AWS region', type: String, default: 'us-east-1'
+        opt :queue, 'SQS Queue name', type: String
+        def initialize
+          creds = if opts.key?('access_key') && opts.key?('secret_key')
+                    {
+                      aws_access_key_id: opts[:access_key],
+                      aws_secret_access_key: opts[:secret_key],
+                    }
+                  else
+                    { use_iam_profile: true }
+                  end
+          creds['region'] = opts[:region]
+          @sqs = Fog::AWS::SQS.new(creds)
+          response = @sqs.list_queues({ 'QueueNamePrefix' => opts[:queue] })
+          @queue_url = response[:body]['QueueUrls'].first
+        end
+
+        def tick
+          response = @sqs.get_queue_attributes(@queue_url, 'All')
+          %w[ApproximateNumberOfMessages ApproximateNumberOfMessagesNotVisible].each do |attr|
+            msg = {
+              metric: response[:body]['Attributes'][attr],
+              service: "#{opts[:queue]} #{attr}",
+              state: 'ok',
+            }
+            report msg
+          end
+        end
+      end
+    end
+  end
+end

data/tools/riemann-aws/lib/riemann/tools/aws/status.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+require 'riemann/tools'
+
+module Riemann
+  module Tools
+    module Aws
+      class Status
+        include Riemann::Tools
+        require 'fog/aws'
+        require 'date'
+
+        opt :fog_credentials_file, 'Fog credentials file', type: String
+        opt :fog_credential, 'Fog credentials to use', type: String
+        opt :access_key, 'AWS access key', type: String
+        opt :secret_key, 'Secret access key', type: String
+        opt :region, 'AWS region', type: String, default: 'eu-west-1'
+
+        opt :retirement_critical, 'Number of days before retirement. Defaults to 2', default: 2
+        opt :event_warning, 'Number of days before event. Defaults to nil (i.e. when the event appears)', default: nil
+
+        def initialize
+          if options[:fog_credentials_file]
+            Fog.credentials_path = options[:fog_credentials_file]
+            Fog.credential = options[:fog_credential].to_sym
+            @compute = Fog::AWS::Compute.new
+          else
+            @compute = if options[:access_key] && options[:secret_key]
+                         Fog::AWS::Compute.new({
+                                                 access_key_key_id: options[:access_key],
+                                                 secret_key_access_key: options[:secret_key],
+                                                 region: options[:region],
+                                               })
+                       else
+                         Fog::AWS::Compute.new({
+                                                 use_iam_profile: true,
+                                                 region: options[:region],
+                                               })
+                       end
+          end
+        end
+
+        def tick
+          hosts = @compute.servers.select { |s| s.state == 'running' }
+
+          hosts.each do |host, host_status|
+            host_status['eventsSet'].each do |event|
+              before, _after = %w[notBefore notAfter].map { |k| Date.parse event[k].to_s if event[k] }
+
+              ev = {
+                host: host,
+                service: 'aws_instance_status',
+                description: "#{event['code']}\n\nstart #{event['notBefore']}\nend #{event['notAfter']}\n\n#{event['description']}",
+                state: 'ok',
+                ttl: 300,
+              }
+
+              ev2 = if (event['code'] == 'instance-retirement') &&
+                       (Date.today >= before - opts[:retirement_critical])
+                      { state: 'critical' }
+                    elsif opts[:event_warning] && (Date.today >= before - opts[:event_warning])
+                      { state: 'warning' }
+                    else
+                      { state: 'warning' }
+                    end
+
+              report ev.merge(ev2)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/tools/riemann-chronos/Rakefile

@@ -11,10 +11,8 @@ ENV['COPYFILE_DISABLE'] = 'true'
 
 # Gemspec
 gemspec = Gem::Specification.new do |s|
-  s.rubyforge_project = 'riemann-chronos'
-
   s.name = 'riemann-chronos'
-  s.version = '0.1.1'
+  s.version = '0.1.3'
   s.author = 'Peter Ericson'
   s.email = 'peter.ericson@cba.com.au'
   s.homepage = 'https://github.com/riemann/riemann-tools'
@@ -22,15 +20,14 @@ gemspec = Gem::Specification.new do |s|
   s.summary = 'Submits Chronos stats to riemann.'
   s.license = 'MIT'
 
-  s.add_dependency 'riemann-tools', '>= 0.2.13'
-  s.add_dependency 'faraday', '>= 0.8.5'
-  s.add_dependency 'json'
+  s.add_runtime_dependency 'riemann-tools', '~> 1.0', '>= 1.1.1'
+  s.add_runtime_dependency 'faraday', '~> 2.3', '>= 2.3.0'
+  s.add_runtime_dependency 'json', '~> 2.6', '>=2.6.2'
 
-  s.files = FileList['bin/*', 'LICENSE', 'README.md'].to_a
+  s.files = FileList['bin/*', 'lib/**/*.rb', 'LICENSE', 'README.md'].to_a
   s.executables |= Dir.entries('bin/')
-  s.has_rdoc = false
 
-  s.required_ruby_version = '>= 1.8.7'
+  s.required_ruby_version = Gem::Requirement.new('>= 2.6.0')
 end
 
 Gem::PackageTask.new gemspec do |p|

data/tools/riemann-chronos/bin/riemann-chronos

@@ -3,159 +3,6 @@
 
 Process.setproctitle($PROGRAM_NAME)
 
-require 'riemann/tools'
+require 'riemann/tools/chronos'
 
-module Riemann
-  module Tools
-    class Chronos
-      include Riemann::Tools
-
-      require 'faraday'
-      require 'json'
-      require 'uri'
-
-      opt :read_timeout, 'Faraday read timeout', type: :int, default: 2
-      opt :open_timeout, 'Faraday open timeout', type: :int, default: 1
-      opt :path_prefix,
-          'Chronos path prefix for proxied installations e.g. "chronos" for target http://localhost/chronos/metrics', default: '/'
-      opt :chronos_host, 'Chronos host', default: 'localhost'
-      opt :chronos_port, 'Chronos port', type: :int, default: 4400
-
-      def initialize
-        options[:interval] = 60
-        options[:ttl] = 120
-      end
-
-      # Handles HTTP connections and GET requests safely
-      def safe_get(uri)
-        # Handle connection timeouts
-        response = nil
-        begin
-          connection = Faraday.new(uri)
-          response = connection.get do |req|
-            req.options[:timeout] = options[:read_timeout]
-            req.options[:open_timeout] = options[:open_timeout]
-          end
-        rescue StandardError => e
-          report(
-            host: uri.host,
-            service: 'chronos health',
-            state: 'critical',
-            description: "HTTP connection error: #{e.class} - #{e.message}",
-          )
-        end
-        response
-      end
-
-      def health_url
-        path_prefix = options[:path_prefix]
-        path_prefix[0] = '' if path_prefix[0] == '/'
-        path_prefix[path_prefix.length - 1] = '' if path_prefix[path_prefix.length - 1] == '/'
-        "http://#{options[:chronos_host]}:#{options[:chronos_port]}#{path_prefix.length.positive? ? '/' : ''}#{path_prefix}/metrics"
-      end
-
-      def jobs_url
-        path_prefix = options[:path_prefix]
-        path_prefix[0] = '' if path_prefix[0] == '/'
-        path_prefix[path_prefix.length - 1] = '' if path_prefix[path_prefix.length - 1] == '/'
-        "http://#{options[:chronos_host]}:#{options[:chronos_port]}#{path_prefix.length.positive? ? '/' : ''}#{path_prefix}/scheduler/jobs"
-      end
-
-      def tick
-        tick_health
-        tick_jobs
-      end
-
-      def tick_health
-        uri = URI(health_url)
-        response = safe_get(uri)
-
-        return if response.nil?
-
-        if response.status != 200
-          report(
-            host: uri.host,
-            service: 'chronos health',
-            state: 'critical',
-            description: "HTTP connection error: #{response.status} - #{response.body}",
-          )
-        else
-          # Assuming that a 200 will give json
-          json = JSON.parse(response.body)
-
-          report(
-            host: uri.host,
-            service: 'chronos health',
-            state: 'ok',
-          )
-
-          json.each_pair do |t, d|
-            next unless d.respond_to? :each_pair
-
-            d.each_pair do |service, counters|
-              report(
-                host: uri.host,
-                service: "chronos_metric #{t} #{service}",
-                metric: 1,
-                tags: ['metric_name'],
-                ttl: 600,
-              )
-              next unless counters.respond_to? :each_pair
-
-              counters.each_pair do |k, v|
-                next unless v.is_a? Numeric
-
-                report(
-                  host: uri.host,
-                  service: "chronos #{service} #{k}",
-                  metric: v,
-                  tags: ['metric', t.to_s],
-                  ttl: 600,
-                )
-              end
-            end
-          end
-        end
-      end
-
-      def tick_jobs
-        uri = URI(jobs_url)
-        response = safe_get(uri)
-
-        return if response.nil?
-
-        if response.status != 200
-          report(
-            host: uri.host,
-            service: 'chronos health',
-            state: 'critical',
-            description: "HTTP connection error: #{response.status} - #{response.body}",
-          )
-        else
-          # Assuming that a 200 will give json
-          json = JSON.parse(response.body)
-
-          report(
-            host: uri.host,
-            service: 'chronos health',
-            state: 'ok',
-          )
-
-          json.each do |job|
-            job.each_pair do |k, v|
-              next unless v.is_a? Numeric
-
-              report(
-                host: uri.host,
-                service: "chronos job #{job['name']} #{k}",
-                metric: v,
-                ttl: 120,
-              )
-            end
-          end
-        end
-      end
-    end
-  end
-end
 Riemann::Tools::Chronos.run

data/tools/riemann-chronos/lib/riemann/tools/chronos.rb

@@ -0,0 +1,157 @@
+# frozen_string_literal: true
+
+require 'riemann/tools'
+
+module Riemann
+  module Tools
+    class Chronos
+      include Riemann::Tools
+
+      require 'faraday'
+      require 'json'
+      require 'uri'
+
+      opt :read_timeout, 'Faraday read timeout', type: :int, default: 2
+      opt :open_timeout, 'Faraday open timeout', type: :int, default: 1
+      opt :path_prefix,
+          'Chronos path prefix for proxied installations e.g. "chronos" for target http://localhost/chronos/metrics', default: '/'.dup
+      opt :chronos_host, 'Chronos host', default: 'localhost'
+      opt :chronos_port, 'Chronos port', type: :int, default: 4400
+
+      def initialize
+        options[:interval] = 60
+        options[:ttl] = 120
+      end
+
+      # Handles HTTP connections and GET requests safely
+      def safe_get(uri)
+        # Handle connection timeouts
+        response = nil
+        begin
+          connection = Faraday.new(uri)
+          response = connection.get do |req|
+            req.options[:timeout] = options[:read_timeout]
+            req.options[:open_timeout] = options[:open_timeout]
+          end
+        rescue StandardError => e
+          report(
+            host: uri.host,
+            service: 'chronos health',
+            state: 'critical',
+            description: "HTTP connection error: #{e.class} - #{e.message}",
+          )
+        end
+        response
+      end
+
+      def health_url
+        path_prefix = options[:path_prefix]
+        path_prefix[0] = '' if path_prefix[0] == '/'
+        path_prefix[path_prefix.length - 1] = '' if path_prefix[path_prefix.length - 1] == '/'
+        "http://#{options[:chronos_host]}:#{options[:chronos_port]}#{path_prefix.length.positive? ? '/' : ''}#{path_prefix}/metrics"
+      end
+
+      def jobs_url
+        path_prefix = options[:path_prefix]
+        path_prefix[0] = '' if path_prefix[0] == '/'
+        path_prefix[path_prefix.length - 1] = '' if path_prefix[path_prefix.length - 1] == '/'
+        "http://#{options[:chronos_host]}:#{options[:chronos_port]}#{path_prefix.length.positive? ? '/' : ''}#{path_prefix}/scheduler/jobs"
+      end
+
+      def tick
+        tick_health
+        tick_jobs
+      end
+
+      def tick_health
+        uri = URI(health_url)
+        response = safe_get(uri)
+
+        return if response.nil?
+
+        if response.status != 200
+          report(
+            host: uri.host,
+            service: 'chronos health',
+            state: 'critical',
+            description: "HTTP connection error: #{response.status} - #{response.body}",
+          )
+        else
+          # Assuming that a 200 will give json
+          json = JSON.parse(response.body)
+
+          report(
+            host: uri.host,
+            service: 'chronos health',
+            state: 'ok',
+          )
+
+          json.each_pair do |t, d|
+            next unless d.respond_to? :each_pair
+
+            d.each_pair do |service, counters|
+              report(
+                host: uri.host,
+                service: "chronos_metric #{t} #{service}",
+                metric: 1,
+                tags: ['metric_name'],
+                ttl: 600,
+              )
+              next unless counters.respond_to? :each_pair
+
+              counters.each_pair do |k, v|
+                next unless v.is_a? Numeric
+
+                report(
+                  host: uri.host,
+                  service: "chronos #{service} #{k}",
+                  metric: v,
+                  tags: ['metric', t.to_s],
+                  ttl: 600,
+                )
+              end
+            end
+          end
+        end
+      end
+
+      def tick_jobs
+        uri = URI(jobs_url)
+        response = safe_get(uri)
+
+        return if response.nil?
+
+        if response.status != 200
+          report(
+            host: uri.host,
+            service: 'chronos health',
+            state: 'critical',
+            description: "HTTP connection error: #{response.status} - #{response.body}",
+          )
+        else
+          # Assuming that a 200 will give json
+          json = JSON.parse(response.body)
+
+          report(
+            host: uri.host,
+            service: 'chronos health',
+            state: 'ok',
+          )
+
+          json.each do |job|
+            job.each_pair do |k, v|
+              next unless v.is_a? Numeric
+
+              report(
+                host: uri.host,
+                service: "chronos job #{job['name']} #{k}",
+                metric: v,
+                ttl: 120,
+              )
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/tools/riemann-docker/Rakefile

@@ -11,10 +11,8 @@ ENV['COPYFILE_DISABLE'] = 'true'
 
 # Gemspec
 gemspec = Gem::Specification.new do |s|
-  s.rubyforge_project = 'riemann-docker'
-
   s.name = 'riemann-docker'
-  s.version = '0.1.3'
+  s.version = '0.1.5'
   s.author = 'Shani Elharrar'
   s.email = ''
   s.homepage = 'https://github.com/riemann/riemann-tools'
@@ -22,14 +20,13 @@ gemspec = Gem::Specification.new do |s|
   s.summary = 'Submits Docker container stats to riemann.'
   s.license = 'MIT'
 
-  s.add_dependency 'riemann-tools', '>= 0.2.13'
-  s.add_dependency 'docker-api', '>= 1.22.0'
+  s.add_runtime_dependency 'riemann-tools', '~> 1.0', '>= 1.1.1'
+  s.add_runtime_dependency 'docker-api', '~> 1.22', '>= 1.22.0'
 
-  s.files = FileList['bin/*', 'LICENSE', 'README.md'].to_a
+  s.files = FileList['bin/*', 'lib/**/*.rb', 'LICENSE', 'README.md'].to_a
   s.executables |= Dir.entries('bin/')
-  s.has_rdoc = false
 
-  s.required_ruby_version = '>= 1.8.7'
+  s.required_ruby_version = Gem::Requirement.new('>= 2.6.0')
 end
 
 Gem::PackageTask.new gemspec do |p|