riemann-tools 0.2.14 → 1.0.0

data/tools/riemann-aws/bin/riemann-aws-status

@@ -0,0 +1,71 @@
+#!/usr/bin/env ruby
+Process.setproctitle($0)
+
+require 'riemann/tools'
+
+$0 = __FILE__  # Let's not expose our AWS keys in the process list
+
+class Riemann::Tools::AWS
+  include Riemann::Tools
+  require 'fog'
+  require 'date'
+
+  opt :access_key, "AWS access key", :type => String
+  opt :secret_key, "Secret access key", :type => String
+  opt :region, "AWS region", :type => String, :default => 'eu-west-1'
+
+  opt :retirement_critical, "Number of days before retirement. Defaults to 2", :default => 2
+  opt :event_warning, "Number of days before event. Defaults to nil (i.e. when the event appears)", :default => nil
+
+  def initialize
+    if opts.has_key?('secret_key') and opts.has_key?('access_key')
+      creds = {
+        :aws_secret_access_key => opts[:secret_key],
+        :aws_access_key_id => opts[:access_key]
+      }
+    else
+      creds = { :use_iam_profile => true }
+    end
+    creds['region'] = opts[:region]
+    creds['provider'] = 'AWS'
+    @compute = Fog::Compute.new(creds)
+  end
+
+  def tick
+    instance_status = @compute.describe_instance_status.body["instanceStatusSet"]
+    status = instance_status.inject({}) do |acc,i|
+      acc[i.delete("instanceId")] = i
+      acc
+    end
+
+    hosts = @compute.servers.select { |s| s.state == "running" }
+      inject([status, {}]) do |(status, acc), host|
+        acc[host.private_dns_name] = status.delete(host.id); [status, acc]
+      end[1]
+
+    hosts.each do |host, status|
+      status['eventsSet'].each do |event|
+        before, after = ['notBefore', 'notAfter'].map { |k| Date.parse event[k].to_s if event[k] }
+
+        ev = {:host => host,
+              :service => "aws_instance_status",
+              :description => "#{event['code']}\n\nstart #{event['notBefore']}\nend #{event['notAfter']}\n\n#{event['description']}",
+              :state => "ok",
+              :ttl => 300}
+
+        ev2 = if (event['code'] == 'instance-retirement') and
+          Date.today >= before-opts[:retirement_critical]
+          {:state => "critical"}
+        elsif opts[:event_warning] and Date.today >= before-opts[:event_warning]
+          {:state => "warning"}
+        else
+          {:state => "warning"}
+        end
+
+        report ev.merge(ev2)
+      end
+    end
+  end
+end
+
+Riemann::Tools::AWS.run

data/tools/riemann-aws/bin/riemann-elb-metrics

@@ -0,0 +1,167 @@
+#!/usr/bin/env ruby
+Process.setproctitle($0)
+
+require 'riemann/tools'
+
+$0 = __FILE__
+
+class Riemann::Tools::ELBMetrics
+  include Riemann::Tools
+  require 'fog'
+  require 'time'
+
+  opt :fog_credentials_file, "Fog credentials file", :type => String
+  opt :fog_credential, "Fog credentials to use", :type => String
+  opt :aws_access, "AWS Access Key", :type => String
+  opt :aws_secret, "AWS Secret Key", :type => String
+  opt :aws_region, "AWS Region", :type => String, :default => "eu-west-1"
+  opt :aws_azs, "List of AZs to aggregate against", :type => :strings, :default => [ "all_az" ]
+  opt :elbs, "List of ELBs to pull metrics from", :type => :strings, :required => true
+
+  def standard_metrics
+    # ELB metric types, from:
+    # http://docs.aws.amazon.com/AmazonCloudWatch/latest/DeveloperGuide/CW_Support_For_AWS.html#elb-metricscollected
+    metric_options = {
+      "Latency" => {
+        "Unit" => "Seconds",
+        "Statistics" => ["Maximum", "Minimum", "Average" ]
+      },
+      "RequestCount" => {
+        "Unit" => "Count",
+        "Statistics" => [ "Sum" ]
+      },
+      "HealthyHostCount" => {
+        "Units" => "Count",
+        "Statistics" => [ "Minimum", "Maximum", "Average" ]
+      },
+      "UnHealthyHostCount" => {
+        "Units" => "Count",
+        "Statistics" => [ "Minimum", "Maximum", "Average" ]
+      },
+      "HTTPCode_ELB_4XX" => {
+        "Units" => "Count",
+        "Statistics" => [ "Sum" ]
+      },
+      "HTTPCode_ELB_5XX" => {
+        "Units" => "Count",
+        "Statistics" => [ "Sum" ]
+      },
+      "HTTPCode_Backend_2XX" => {
+        "Units" => "Count",
+        "Statistics" => [ "Sum" ]
+      },
+      "HTTPCode_Backend_3XX" => {
+        "Units" => "Count",
+        "Statistics" => [ "Sum" ]
+      },
+      "HTTPCode_Backend_4XX" => {
+        "Units" => "Count",
+        "Statistics" => [ "Sum" ]
+      },
+      "HTTPCode_Backend_5XX" => {
+        "Units" => "Count",
+        "Statistics" => [ "Sum" ]
+      }
+    }
+
+    metric_options
+  end
+
+  def base_metrics
+    # get last 60 seconds
+    start_time = (Time.now.utc - 60).iso8601
+    end_time = Time.now.utc.iso8601
+
+    # The base query that all metrics would get
+    metric_base = {
+      "Namespace" => "AWS/ELB",
+      "StartTime" => start_time,
+      "EndTime" => end_time,
+      "Period" => 60,
+    }
+
+    metric_base
+  end
+
+
+  def tick
+    if options[:fog_credentials_file]
+      Fog.credentials_path = options[:fog_credentials_file]
+      Fog.credential = options[:fog_credential].to_sym
+      connection = Fog::AWS::CloudWatch.new
+    else 
+      if options[:aws_access] && options[:aws_secret]
+        connection = Fog::AWS::CloudWatch.new({
+          :aws_access_key_id => options[:aws_access],
+          :aws_secret_access_key => options[:aws_secret],
+          :region => options[:aws_region]
+        })
+      else
+        connection = Fog::AWS::CloudWatch.new({
+          :use_iam_profile => true,
+          :region => options[:aws_region]
+        })        
+      end
+    end
+
+    options[:elbs].each do |lb|
+
+      metric_options = standard_metrics
+      metric_base_options = base_metrics
+
+      options[:aws_azs].each do |az|
+        metric_options.keys.sort.each do |metric_type|
+          merged_options = metric_base_options.merge(metric_options[metric_type])
+          merged_options["MetricName"] = metric_type
+          if az == "all_az"
+            merged_options["Dimensions"] = [ { "Name" => "LoadBalancerName", "Value" => lb } ]
+          else
+            merged_options["Dimensions"] = [
+              { "Name" => "LoadBalancerName", "Value" => lb },
+              { "Name" => "AvailabilityZone" , "Value" => az}
+            ]
+          end
+
+          result = connection.get_metric_statistics(merged_options)
+
+          # "If no response codes in the category 2XX-5XX range are sent to clients within 
+          # the given time period, values for these metrics will not be recorded in CloudWatch"
+          #next if result.body["GetMetricStatisticsResult"]["Datapoints"].empty? && metric_type =~ /[2345]XX/
+          #
+          if result.body["GetMetricStatisticsResult"]["Datapoints"].empty?
+            standard_metrics[metric_type]['Statistics'].each do |stat_type|
+              event = event(lb, az, metric_type, stat_type, 0.0)
+              report(event)
+            end
+            next
+          end
+
+          # We should only ever have a single data point
+          result.body["GetMetricStatisticsResult"]["Datapoints"][0].keys.sort.each do |stat_type|
+            next if stat_type == "Unit"
+            next if stat_type == "Timestamp"
+
+            unit = result.body["GetMetricStatisticsResult"]["Datapoints"][0]["Unit"]
+            metric = result.body["GetMetricStatisticsResult"]["Datapoints"][0][stat_type]
+            event = event(lb, az, metric_type, stat_type, metric, unit)
+            report(event)
+          end
+        end
+      end
+    end
+  end
+
+  private
+  def event(lb, az, metric_type, stat_type, metric, unit=nil)
+    event = {
+      host: lb,
+      service: "elb.#{az}.#{metric_type}.#{stat_type}",
+      ttl: 60,
+      description: "#{lb} #{metric_type} #{stat_type} (#{unit})",
+      tags: ["elb_metrics"],
+      metric: metric
+    }
+  end
+end
+
+Riemann::Tools::ELBMetrics.run

data/tools/riemann-aws/bin/riemann-s3-list

@@ -0,0 +1,82 @@
+#!/usr/bin/env ruby
+Process.setproctitle($0)
+
+require 'riemann/tools'
+
+$0 = __FILE__
+
+class Riemann::Tools::S3Metrics
+  include Riemann::Tools
+  require 'fog'
+  require 'time'
+
+  opt :fog_credentials_file, "Fog credentials file", :type => String
+  opt :fog_credential, "Fog credentials to use", :type => String
+  opt :aws_access, "AWS Access Key", :type => String
+  opt :aws_secret, "AWS Secret Key", :type => String
+  opt :aws_region, "AWS Region", :type => String, :default => "eu-west-1"
+  opt :buckets, "Buckets to pull metrics from, multi=true, can have a prefix like mybucket/prefix", :type => String, :multi => true, :required => true
+  opt :max_objects, "Max number of objects to list before stopping to save bandwidth", :default => -1
+
+
+  def tick
+    if options[:fog_credentials_file]
+      Fog.credentials_path = options[:fog_credentials_file]
+      Fog.credential = options[:fog_credential].to_sym
+      connection = Fog::Storage.new
+    else
+      if options[:aws_access] && options[:aws_secret]
+        connection = Fog::Storage.new({
+                                          :provider => "AWS",
+                                          :aws_access_key_id => options[:aws_access],
+                                          :aws_secret_access_key => options[:aws_secret],
+                                          :region => options[:aws_region]
+                                      })
+      else
+        connection = Fog::Storage.new({
+                                          :provider => "AWS",
+                                          :use_iam_profile => true,
+                                          :region => options[:aws_region]
+                                      })
+      end
+    end
+
+    options[:buckets].each do |url|
+      split = url.split('/')
+      bucket = split[0]
+      prefix = ""
+      if (split[1])
+        prefix = url[(split[0].length+1)..-1]
+      end
+      count = 0
+      connection.directories.get(bucket, prefix: prefix).files.map do |file|
+        count = count +1
+        if (options[:max_objects]>0 && count>options[:max_objects])
+          break
+        end
+      end
+      if (options[:max_objects]>0 && count>options[:max_objects])
+        event = event(url, "objectCount", count, "count was bigger than threshold #{options[:max_objects]}", "warning")
+        report(event)
+      else
+        event = event(url, "objectCount", count, "All objects counted, threshold=#{options[:max_objects]}", "ok")
+        report(event)
+      end
+    end
+  end
+
+  private
+  def event(bucket, label, metric, description, severity)
+    event = {
+        host: "bucket_#{bucket}",
+        service: "s3.#{label}",
+        ttl: 300,
+        description: "#{bucket} #{description}",
+        tags: ["s3_metrics"],
+        metric: metric,
+        state: severity
+    }
+  end
+end
+
+Riemann::Tools::S3Metrics.run

data/tools/riemann-aws/bin/riemann-s3-status

@@ -0,0 +1,99 @@
+#!/usr/bin/env ruby
+Process.setproctitle($0)
+
+require 'riemann/tools'
+
+$0 = __FILE__
+
+class Riemann::Tools::S3Metrics
+  include Riemann::Tools
+  require 'fog'
+  require 'time'
+
+  opt :fog_credentials_file, "Fog credentials file", :type => String
+  opt :fog_credential, "Fog credentials to use", :type => String
+  opt :aws_access, "AWS Access Key", :type => String
+  opt :aws_secret, "AWS Secret Key", :type => String
+  opt :aws_region, "AWS Region", :type => String, :default => "eu-west-1"
+  opt :buckets, "Buckets to pull metrics from, multi=true", :type => String, :multi => true, :required => true
+  opt :statistic, "Statistic to retrieve, multi=true, e.g. --statistic=Average --statistic=Maximum", :type => String, :multi => true, :required => true
+
+
+  def base_metrics
+    # get last 60 seconds
+    start_time = (Time.now.utc - 3600 * 24 * 1).iso8601
+    end_time = Time.now.utc.iso8601
+
+    # The base query that all metrics would get
+    metric_base = {
+        "Namespace" => "AWS/S3",
+        "StartTime" => start_time,
+        "EndTime" => end_time,
+        "Period" => 3600,
+        "MetricName" => "NumberOfObjects",
+    }
+
+    metric_base
+  end
+
+
+  def tick
+    if options[:fog_credentials_file]
+      Fog.credentials_path = options[:fog_credentials_file]
+      Fog.credential = options[:fog_credential].to_sym
+      connection = Fog::AWS::CloudWatch.new
+    else
+      if options[:aws_access] && options[:aws_secret]
+        connection = Fog::AWS::CloudWatch.new({
+                                                  :aws_access_key_id => options[:aws_access],
+                                                  :aws_secret_access_key => options[:aws_secret],
+                                                  :region => options[:aws_region]
+                                              })
+      else
+        connection = Fog::AWS::CloudWatch.new({
+                                                  :use_iam_profile => true,
+                                                  :region => options[:aws_region]
+                                              })
+      end
+    end
+
+    options[:statistic].each do |statistic|
+      options[:buckets].each do |bucket|
+
+        metric_base_options = base_metrics
+        metric_base_options["Statistics"] = statistic
+        metric_base_options["Dimensions"] = [
+            {"Name" => "BucketName", "Value" => bucket},
+            {"Name" => "StorageType", "Value" => "AllStorageTypes"}]
+
+        result = connection.get_metric_statistics(metric_base_options)
+        if result.body["GetMetricStatisticsResult"]["Datapoints"].empty?
+          next
+        end
+          result.body["GetMetricStatisticsResult"]["Datapoints"][0].keys.sort.each do |stat_type|
+          next if stat_type == "Unit"
+          next if stat_type == "Timestamp"
+
+          unit = result.body["GetMetricStatisticsResult"]["Datapoints"][0]["Unit"]
+          metric = result.body["GetMetricStatisticsResult"]["Datapoints"][0][stat_type]
+          event = event(bucket, result.body["GetMetricStatisticsResult"]["Label"], stat_type, unit, metric)
+          report(event)
+        end
+      end
+    end
+  end
+
+  private
+  def event(bucket, label, metric_type, stat_type, metric, unit=nil)
+    event = {
+        host: "bucket_#{bucket}",
+        service: "s3.#{label}.#{metric_type}.#{stat_type}",
+        ttl: 300,
+        description: "#{bucket} #{metric_type} #{stat_type} (#{unit})",
+        tags: ["s3_metrics"],
+        metric: metric
+    }
+  end
+end
+
+Riemann::Tools::S3Metrics.run

data/tools/riemann-chronos/LICENSE

@@ -0,0 +1,21 @@
+The MIT License
+
+Copyright (c) 2011 Kyle Kingsbury
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/tools/riemann-chronos/README.md

@@ -0,0 +1,10 @@
+# Riemann Chronos
+
+Gathers Chronos metrics and sends them to Riemann.
+
+# Getting started
+
+```
+gem install riemann-chronos
+riemann-chronos --help
+```

data/tools/riemann-chronos/Rakefile.rb

@@ -0,0 +1,35 @@
+require 'rubygems'
+require 'rubygems/package_task'
+require 'rdoc/task'
+require 'find'
+
+# Don't include resource forks in tarballs on Mac OS X.
+ENV['COPY_EXTENDED_ATTRIBUTES_DISABLE'] = 'true'
+ENV['COPYFILE_DISABLE'] = 'true'
+
+# Gemspec
+gemspec = Gem::Specification.new do |s|
+  s.rubyforge_project = 'riemann-chronos'
+
+  s.name = 'riemann-chronos'
+  s.version = '0.1.1'
+  s.author = 'Peter Ericson'
+  s.email = 'peter.ericson@cba.com.au'
+  s.homepage = 'https://github.com/riemann/riemann-tools'
+  s.platform = Gem::Platform::RUBY
+  s.summary = 'Submits Chronos stats to riemann.'
+  s.license = 'MIT'
+
+  s.add_dependency 'riemann-tools', '>= 0.2.13'
+  s.add_dependency 'faraday', '>= 0.8.5'
+  s.add_dependency 'json'
+
+  s.files = FileList['bin/*', 'LICENSE', 'README.md'].to_a
+  s.executables |= Dir.entries('bin/')
+  s.has_rdoc = false
+
+  s.required_ruby_version = '>= 1.8.7'
+end
+
+Gem::PackageTask.new gemspec do |p|
+end

data/tools/riemann-chronos/bin/riemann-chronos

@@ -0,0 +1,144 @@
+#!/usr/bin/env ruby
+Process.setproctitle($0)
+
+require 'riemann/tools'
+
+class Riemann::Tools::Chronos
+  include Riemann::Tools
+
+  require 'faraday'
+  require 'json'
+  require 'uri'
+
+  opt :read_timeout, 'Faraday read timeout', type: :int, default: 2
+  opt :open_timeout, 'Faraday open timeout', type: :int, default: 1
+  opt :path_prefix, 'Chronos path prefix for proxied installations e.g. "chronos" for target http://localhost/chronos/metrics', default: "/"
+  opt :chronos_host, 'Chronos host', default: "localhost"
+  opt :chronos_port, 'Chronos port', type: :int, default: 4400
+
+  def initialize
+    options[:interval] = 60
+    options[:ttl] = 120
+  end
+
+  # Handles HTTP connections and GET requests safely
+  def safe_get(uri)
+      # Handle connection timeouts
+      response = nil
+      begin
+        connection = Faraday.new(uri)
+        response = connection.get do |req|
+          req.options[:timeout] = options[:read_timeout]
+          req.options[:open_timeout] = options[:open_timeout]
+        end
+      rescue => e
+        report(:host => uri.host,
+          :service => "chronos health",
+          :state => "critical",
+          :description => "HTTP connection error: #{e.class} - #{e.message}"
+        )
+      end
+      response
+  end
+
+  def health_url
+    path_prefix = options[:path_prefix]
+    path_prefix[0] = '' if path_prefix[0]=='/'
+    path_prefix[path_prefix.length-1] = '' if path_prefix[path_prefix.length-1]=='/'
+    "http://#{options[:chronos_host]}:#{options[:chronos_port]}#{path_prefix.length>0?'/':''}#{path_prefix}/metrics"
+  end
+
+  def jobs_url
+    path_prefix = options[:path_prefix]
+    path_prefix[0] = '' if path_prefix[0]=='/'
+    path_prefix[path_prefix.length-1] = '' if path_prefix[path_prefix.length-1]=='/'
+    "http://#{options[:chronos_host]}:#{options[:chronos_port]}#{path_prefix.length>0?'/':''}#{path_prefix}/scheduler/jobs"
+  end
+
+  def tick
+    tick_health
+    tick_jobs
+  end
+
+  def tick_health
+    uri = URI(health_url)
+    response = safe_get(uri)
+
+    return if response.nil?
+
+    if response.status != 200
+        report(:host => uri.host,
+          :service => "chronos health",
+          :state => "critical",
+          :description => "HTTP connection error: #{response.status} - #{response.body}"
+        )
+    else
+      # Assuming that a 200 will give json
+      json = JSON.parse(response.body)
+
+      report(:host => uri.host,
+             :service => "chronos health",
+             :state => "ok")
+
+      json.each_pair do |t, d|
+        if d.respond_to? :each_pair
+          d.each_pair do |service, counters|
+            report(:host => uri.host,
+                   :service => "chronos_metric #{t} #{service}",
+                   :metric => 1,
+                   :tags => ["metric_name"],
+                   :ttl => 600
+            )
+            if counters.respond_to? :each_pair
+              counters.each_pair do |k, v|
+                if v.is_a? Numeric
+                  report(:host => uri.host,
+                         :service => "chronos #{service} #{k}",
+                         :metric => v,
+                         :tags => ["metric", "#{t}"],
+                         :ttl => 600
+                  )
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+
+  def tick_jobs
+    uri = URI(jobs_url)
+    response = safe_get(uri)
+
+    return if response.nil?
+
+    if response.status != 200
+        report(:host => uri.host,
+          :service => "chronos health",
+          :state => "critical",
+          :description => "HTTP connection error: #{response.status} - #{response.body}"
+        )
+    else
+      # Assuming that a 200 will give json
+      json = JSON.parse(response.body)
+
+      report(:host => uri.host,
+             :service => "chronos health",
+             :state => "ok")
+
+      json.each do |job|
+        job.each_pair do |k, v|
+          if v.is_a? Numeric
+            report(:host => uri.host,
+                   :service => "chronos job #{job["name"]} #{k}",
+                   :metric => v,
+                   :ttl => 120
+            )
+          end
+        end
+      end
+    end
+  end
+end
+Riemann::Tools::Chronos.run

data/tools/riemann-docker/LICENSE

@@ -0,0 +1,21 @@
+The MIT License
+
+Copyright (c) 2011 Kyle Kingsbury
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/tools/riemann-docker/README.md

@@ -0,0 +1,10 @@
+# Riemann Docker
+
+Gathers Docker container metrics and sends them to Riemann.
+
+# Getting started
+
+```
+gem install riemann-docker
+riemann-docker --help
+```