sensu-plugins-kafka 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 65754e8c1e0baa13b3cdf0899acf18e43f0a4324
+  data.tar.gz: 049a7b8157a9d8a65c7a9ce565b4aab4841996f6
+SHA512:
+  metadata.gz: 14f47ddcc363ae87861c322b6ed558b7967e9b1959889a8e316cb94395ad65d5afde86c04b0a649fc00d8a1eeff56b8020c3275df162823ea9ad6da2ac9629ae
+  data.tar.gz: 0e48d28fdef98c2f3ab7bf852f2809091253f9b253bf2ce85a3037a886dd13ce58cf294c4a585914750340e2921bdb09805d2a2fc885841487b7a6fdfa46c8ae

data/CHANGELOG.md

@@ -0,0 +1,8 @@
+#Change Log
+This project adheres to [Semantic Versioning](http://semver.org/).
+
+This CHANGELOG follows the format listed at [Keep A Changelog](http://keepachangelog.com/)
+
+## Unreleased
+### Added
+- initial release

data/LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2015 Sensu-Plugins
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,21 @@
+## Sensu-Plugins-Kafka
+
+[![Build Status](https://travis-ci.org/obazoud/sensu-plugins-kafka.svg?branch=master)](https://travis-ci.org/obazoud/sensu-plugins-kafka)
+
+## Functionality
+
+**check-consumer-lag.rb**
+
+**metrics-consumer.rb**
+
+## Files
+
+## Usage
+
+## Installation
+
+[Installation and Setup](http://sensu-plugins.io/docs/installation_instructions.html)
+
+Kafka installation is required where checks will be made.
+
+Note:  In addition to the standard installation requirements the installation of this gem will require compiling the nokogiri gem.  Due to this you'll need certain developmemnt packages on your system.  On Ubuntu systems install build-essential, libxml2-dev and zlib1g-dev.  On CentOS install gcc and zlib-devel.

data/bin/check-consumer-lag.rb

@@ -0,0 +1,181 @@
+#! /usr/bin/env ruby
+#
+# check-consumer-lag
+#
+# DESCRIPTION:
+#   This plugin checks the lag of your kafka's consumers.
+#
+# OUTPUT:
+#   plain-text
+#
+# PLATFORMS:
+#   Linux
+#
+# DEPENDENCIES:
+#   gem: sensu-plugin
+#
+# USAGE:
+#   ./check-consumer-lag
+#
+# NOTES:
+#
+# LICENSE:
+#   Olivier Bazoud
+#   Released under the same terms as Sensu (the MIT license); see LICENSE
+#   for details.
+#
+
+require 'sensu-plugin/check/cli'
+
+class ConsumerLagCheck < Sensu::Plugin::Check::CLI
+  option :group,
+         description: 'Consumer group',
+         short:       '-g NAME',
+         long:        '--group NAME',
+         required:    true
+
+  option :kafka_home,
+         description: 'Kafka home',
+         short:       '-k NAME',
+         long:        '--kafka-home NAME',
+         default:     '/opt/kafka'
+
+  option :topic_excludes,
+         description: 'Excludes consumer topics',
+         short:       '-e NAME',
+         long:        '--topic-excludes NAME',
+         proc:        proc { |a| a.split(',') }
+
+  option :autolist,
+         description: 'Auto list topics',
+         short:       '-a VALUE',
+         long:        '--auto-list VALUE',
+         boolean: true,
+         default: true
+
+  option :zookeeper,
+         description: 'ZooKeeper connect string',
+         short:       '-z NAME',
+         long:        '--zookeeper NAME',
+         default:     'localhost:2181'
+
+  option :warning_over,
+         description: 'Warning if metric statistics is over specified value.',
+         short:       '-W N',
+         long:        '--warning-over N'
+
+  option :critical_over,
+         description: 'Critical if metric statistics is over specified value.',
+         short:       '-C N',
+         long:        '--critical-over N'
+
+  option :warning_under,
+         description: 'Warning if metric statistics is under specified value.',
+         short:       '-w N',
+         long:        '--warning-under N'
+
+  option :critical_under,
+         description: 'Critical if metric statistics is under specified value.',
+         short:       '-c N',
+         long:        '--critical-under N'
+
+  # read the output of a command
+  # @param cmd [String] the command to read the output from
+  def read_lines(cmd)
+    IO.popen(cmd + ' 2>&1') do |child|
+      child.read.split("\n")
+    end
+  end
+
+  # create a hash from the output of each line of a command
+  # @param line [String]
+  # @param cols
+  def line_to_hash(line, *cols)
+    Hash[cols.zip(line.strip.split(/\s+/, cols.size))]
+  end
+
+  # run command and return a hash from the output
+  # @param cms [String]
+  def run_offset(cmd)
+    read_lines(cmd).drop(1).map do |line|
+      line_to_hash(line, :group, :topic, :pid, :offset, :logsize, :lag, :owner)
+    end
+  end
+
+  # run command and return a hash from the output
+  # @param cms [String]
+  def run_topics(cmd)
+    topics = []
+    read_lines(cmd).map do |line|
+      if !line.include?('__consumer_offsets') && !line.include?('marked for deletion')
+        topics.push(line)
+      end
+    end
+    topics
+  end
+
+  def run
+    kafka_run_class = "#{config[:kafka_home]}/bin/kafka-run-class.sh"
+    unknown "Can not find #{kafka_run_class}" unless File.exist?(kafka_run_class)
+
+    topics_to_read = []
+    if config[:autolist].to_s == 'true'
+      cmd_topics = "#{kafka_run_class} kafka.admin.TopicCommand --zookeeper #{config[:zookeeper]} --list"
+      topics_to_read = run_topics(cmd_topics)
+      topics_to_read.delete_if { |x| config[:topic_excludes].include?(x) } if config[:topic_excludes]
+    end
+
+    cmd_offset = "#{kafka_run_class} kafka.tools.ConsumerOffsetChecker --group #{config[:group]} --zookeeper #{config[:zookeeper]}"
+    cmd_offset += " --topic #{topics_to_read.join(',')}" unless topics_to_read.empty?
+
+    topics = run_offset(cmd_offset).group_by { |h| h[:topic] }
+
+    critical 'Could not found topics/partitions' if topics.empty?
+
+    [:offset, :logsize, :lag].each do |field|
+      topics.map do |k, v|
+        critical "Topic #{k} has partitions with #{field} < 0" unless v.select { |w| w[field].to_i < 0 }.empty?
+      end
+    end
+
+    topics.map do |k, v|
+      critical "Topic #{k} has partitions with no owner" unless v.select { |w| w[:owner] == 'none' }.empty?
+    end
+
+    lags = topics.map do |k, v|
+      Hash[k, v.inject(0) { |a, e| a + e[:lag].to_i }]
+    end
+
+    max_lag = lags.map(&:values).flatten.max
+    max_topics = lags.select { |a| a.key(max_lag) }.map(&:keys).flatten
+
+    min_lag = lags.map(&:values).flatten.min
+    min_topics = lags.select { |a| a.key(min_lag) }.map(&:keys).flatten
+
+    [:over, :under].each do |over_or_under|
+      [:critical, :warning].each do |severity|
+        threshold = config[:"#{severity}_#{over_or_under}"]
+
+        next unless threshold
+        case over_or_under
+        when :over
+          if max_lag > threshold.to_i
+            msg = "Topics `#{max_topics}` for the group `#{config[:group]}` lag: #{max_lag} (>= #{threshold})"
+            send severity, msg
+          end
+        when :under
+          if min_lag < threshold.to_i
+            msg =  "Topics `#{min_topics}` for the group `#{config[:group]}` lag: #{min_lag} (<= #{threshold})"
+            send severity, msg
+          end
+        end
+      end
+    end
+
+    ok "Group `#{config[:group]}`'s lag is ok (#{min_lag}/#{max_lag})"
+
+  rescue => e
+    puts "Error: exception: #{e} - #{e.backtrace}"
+    critical
+  end
+end

data/bin/check-topics.rb

@@ -0,0 +1,92 @@
+#! /usr/bin/env ruby
+#
+# check-topics
+#
+# DESCRIPTION:
+#   This plugin checks topics properties.
+#
+# OUTPUT:
+#   plain-text
+#
+# PLATFORMS:
+#   Linux
+#
+# DEPENDENCIES:
+#   gem: sensu-plugin
+#
+# USAGE:
+#   ./check-consumer-lag
+#
+# NOTES:
+#
+# LICENSE:
+#   Olivier Bazoud
+#   Released under the same terms as Sensu (the MIT license); see LICENSE
+#   for details.
+#
+
+require 'sensu-plugin/check/cli'
+
+class TopicCheck < Sensu::Plugin::Check::CLI
+  option :kafka_home,
+         description: 'Kafka home',
+         short:       '-k NAME',
+         long:        '--kafka-home NAME',
+         default:     '/opt/kafka'
+
+  option :zookeeper,
+         description: 'ZooKeeper connect string',
+         short:       '-z NAME',
+         long:        '--zookeeper NAME',
+         default:     'localhost:2181'
+
+  option :properites,
+         description: 'Replication factor for this topic',
+         short:       '-R N',
+         long:        '--replication-factor N'
+
+  # read the output of a command
+  # @param cmd [String] the command to read the output from
+  def read_lines(cmd)
+    IO.popen(cmd + ' 2>&1') do |child|
+      child.read.split("\n")
+    end
+  end
+
+  # create a hash from the output of each line of a command
+  # @param line [String]
+  # @param cols
+  def line_to_hash(line, *cols)
+    Hash[cols.zip(line.strip.split(/\s+/, cols.size))]
+  end
+
+  # run command and return a hash from the output
+  # @param cms [String]
+  def run_cmd(cmd)
+    read_lines(cmd).drop(1).map do |line|
+      line_to_hash(line, :_, :topic, :_, :partition, :_, :leader, :_, :replicas, :_, :isr)
+    end
+  end
+
+  def run
+    kafka_topics = "#{config[:kafka_home]}/bin/kafka-topics.sh"
+    unknown "Can not find #{kafka_topics}" unless File.exist?(kafka_topics)
+
+    cmd_details = "#{kafka_topics} --describe --zookeeper #{config[:zookeeper]} | grep -v ReplicationFactor"
+    results = run_cmd(cmd_details)
+
+    topics = results.group_by { |h| h[:topic] }
+
+    # {:_=>"Isr:", :topic=>"__consumer_offsets", :partition=>"49", :leader=>"172314554", :replicas=>"172314554,172314557,172314558", :isr=>"172314554,172314557,172314558"}
+
+    topics.each do |_, topic|
+      topic.inject(0) { |a, e| [a, e[:replicas].length].max }
+    end
+    ok
+
+  rescue => e
+    puts "Error: #{e}"
+    puts "Error: #{e.backtrace}"
+    critical "Error: #{e}"
+  end
+end

data/bin/metrics-consumer.rb

@@ -0,0 +1,112 @@
+#! /usr/bin/env ruby
+#
+# metrics-consumer
+#
+# DESCRIPTION:
+#   Gets consumers's offset, logsize and lag metrics from Kafka/Zookeeper and puts them in Graphite for longer term storage
+#
+# OUTPUT:
+#   metric-data
+#
+# PLATFORMS:
+#   Linux
+#
+# DEPENDENCIES:
+#   gem: sensu-plugin
+#
+# USAGE:
+#   #YELLOW
+#
+#
+# LICENSE:
+#   Copyright 2015 Olivier Bazoud
+#   Released under the same terms as Sensu (the MIT license); see LICENSE
+#   for details.
+#
+
+require 'sensu-plugin/metric/cli'
+
+class ConsumerOffsetMetrics < Sensu::Plugin::Metric::CLI::Graphite
+  option :scheme,
+         description: 'Metric naming scheme, text to prepend to metric',
+         short: '-s SCHEME',
+         long: '--scheme SCHEME',
+         default: 'sensu.kafka.consumers'
+
+  option :group,
+         description: 'Consumer group',
+         short:       '-g NAME',
+         long:        '--group NAME',
+         required:    true
+
+  option :kafka_home,
+         description: 'Kafka home',
+         short:       '-k NAME',
+         long:        '--kafka-home NAME',
+         default:     '/opt/kafka'
+
+  option :topic,
+         description: 'Comma-separated list of consumer topics',
+         short:       '-t NAME',
+         long:        '--topic NAME'
+
+  option :topic_excludes,
+         description: 'Excludes consumer topics',
+         short:       '-e NAME',
+         long:        '--topic-excludes NAME',
+         proc:        proc { |a| a.split(',') }
+
+  option :zookeeper,
+         description: 'ZooKeeper connect string',
+         short:       '-z NAME',
+         long:        '--zookeeper NAME',
+         default:     'localhost:2181'
+
+  # read the output of a command
+  # @param cmd [String] the command to read the output from
+  def read_lines(cmd)
+    IO.popen(cmd + ' 2>&1') do |child|
+      child.read.split("\n")
+    end
+  end
+
+  # create a hash from the output of each line of a command
+  # @param line [String]
+  # @param cols
+  def line_to_hash(line, *cols)
+    Hash[cols.zip(line.strip.split(/\s+/, cols.size))]
+  end
+
+  # run command and return a hash from the output
+  # @param cms [String]
+  def run_cmd(cmd)
+    read_lines(cmd).drop(1).map do |line|
+      line_to_hash(line, :group, :topic, :pid, :offset, :logsize, :lag, :owner)
+    end
+  end
+
+  def run
+    begin
+      kafka_run_class = "#{config[:kafka_home]}/bin/kafka-run-class.sh"
+      unknown "Can not find #{kafka_run_class}" unless File.exist?(kafka_run_class)
+
+      cmd = "#{kafka_run_class} kafka.tools.ConsumerOffsetChecker --group #{config[:group]} --zookeeper #{config[:zookeeper]}"
+      cmd += " --topic #{config[:topic]}" if config[:topic]
+
+      results = run_cmd(cmd)
+
+      [:offset, :logsize, :lag].each do |field|
+        sum_by_group = results.group_by { |h| h[:topic] }.map do |k, v|
+          Hash[k, v.inject(0) { |a, e| a + e[field].to_i }]
+        end
+        sum_by_group.delete_if { |x| config[:topic_excludes].include?(x.keys[0]) } if config[:topic_excludes]
+        sum_by_group.each do |x|
+          output "#{config[:scheme]}.#{config[:group]}.#{x.keys[0]}.#{field}", x.values[0]
+        end
+      end
+    rescue => e
+      critical "Error: exception: #{e}"
+    end
+    ok
+  end
+end

data/lib/sensu-plugins-kafka.rb

@@ -0,0 +1,2 @@
+
+require 'sensu-plugins-kafka/version'

data/lib/sensu-plugins-kafka/version.rb

@@ -0,0 +1,13 @@
+require 'json'
+
+# encoding: utf-8
+module SensuPluginsKafka
+  # This defines the version of the gem
+  module Version
+    MAJOR = 0
+    MINOR = 1
+    PATCH = 0
+
+    VER_STRING = [MAJOR, MINOR, PATCH].compact.join('.')
+  end
+end

metadata

@@ -0,0 +1,201 @@
+--- !ruby/object:Gem::Specification
+name: sensu-plugins-kafka
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Sensu-Plugins and contributors
+autorequire: 
+bindir: bin
+cert_chain:
+- certs/sensu-plugins.pem
+date: 2017-02-04 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: sensu-plugin
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: codeclimate-test-reporter
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.4'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.4'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.32.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.32.1
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.1'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: github-markup
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: redcarpet
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
+- !ruby/object:Gem::Dependency
+  name: yard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.8'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.8'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.10'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.10'
+description: Sensu plugins for Kafka
+email: "<sensu-users@googlegroups.com>"
+executables:
+- metrics-consumer.rb
+- check-consumer-lag.rb
+- check-topics.rb
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- LICENSE
+- README.md
+- bin/check-consumer-lag.rb
+- bin/check-topics.rb
+- bin/metrics-consumer.rb
+- lib/sensu-plugins-kafka.rb
+- lib/sensu-plugins-kafka/version.rb
+homepage: https://github.com/sensu-plugins/sensu-plugins-kafka
+licenses:
+- MIT
+metadata:
+  maintainer: sensu-plugin
+  development_status: active
+  production_status: unstable - testing recommended
+  release_draft: 'false'
+  release_prerelease: 'false'
+post_install_message: You can use the embedded Ruby by setting EMBEDDED_RUBY=true
+  in /etc/default/sensu
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.1.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.8
+signing_key: 
+specification_version: 4
+summary: Sensu plugins for Kafka
+test_files: []