sensu-plugins-kafka 0.8.1 → 0.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 238d5d13127c16a53fd30c5a4dae4a05471547cf
-  data.tar.gz: 0d81d88567c829276fb5a9a11db4c9eae162f196
+  metadata.gz: 0a503a49eba3fdde83ea058f05aa528e880e88d3
+  data.tar.gz: 987863996d1810bb470709f75141a9f6e5c6d752
 SHA512:
-  metadata.gz: 41831c5fc35254acbf40b9cee04c8bf0735a9be6062f2a94f94b9552d8f85cd6fcfaa5356147ac369d7452965d7c1cba3c9a9dc2ea9478ee083a837f16cccc9c
-  data.tar.gz: 2eb96ee56a1888894765e8d28614bd185919a1542abd5ae028092506afc86b018cca1909041d3ac28dfbfd7a44d486f6563125f2a4d6b15878361e507cf11ae6
+  metadata.gz: b48c13220791bec9f7d77e9502865bb208c91cc1767fb70788baa13a4329d05bb17449979ab76c39ded7e5f92666900945c86d59e04cd0e1927f0315d921aabc
+  data.tar.gz: 04d858babaebd2080e08adc7e0d34a879a384ea3899bea6b6e1c8e374e5dda15340df03606d7575da410342dc61325fa8e8b8d6df5673c73d1294a2a89a9de2d

data/CHANGELOG.md

@@ -10,17 +10,22 @@ This CHANGELOG follows the format listed at [Keep A Changelog](http://keepachang
 ### Fixed
 ### Changed
 
-## [0.8.1]
+## [0.9.0] - 2017-02-16
+
+### Breaking Changes
+- Refactoring metrics-consumer-lag to remove shell/jvm execution
+
+## [0.8.1] - 2017-02-16
 
 ### Fixed
 - check-consumer-lag: remove celluloid (temporary), fix data structure, fix partition_owner function
 
-## [0.8.0]
+## [0.8.0] - 2017-02-15
 
 ### Breaking Changes
 - Refactoring check-consumer-lag to remove shell/jvm execution
 
-## [0.7.0]
+## [0.7.0] - 2017-02-10
 
 ### Added
 - Add option to check topic's replicas

data/README.md

@@ -20,8 +20,8 @@
 
 [Installation and Setup](http://sensu-plugins.io/docs/installation_instructions.html)
 
-Kafka installation is required where checks will be made.
-
-Kafka version: 0.8.2.x
+Tested on:
+* Zookeeper version: 3.4.6
+* Kafka version: 0.8.2.x
 
 Note:  In addition to the standard installation requirements the installation of this gem will require compiling the nokogiri gem.  Due to this you'll need certain developmemnt packages on your system.  On Ubuntu systems install build-essential, libxml2-dev and zlib1g-dev.  On CentOS install gcc and zlib-devel.

data/bin/metrics-consumer.rb

@@ -26,6 +26,10 @@
 
 require 'sensu-plugin/metric/cli'
 
+require 'json'
+require 'poseidon'
+require 'zookeeper'
+
 class ConsumerOffsetMetrics < Sensu::Plugin::Metric::CLI::Graphite
   option :scheme,
          description: 'Metric naming scheme, text to prepend to metric',
@@ -39,12 +43,6 @@ class ConsumerOffsetMetrics < Sensu::Plugin::Metric::CLI::Graphite
          long:        '--group NAME',
          required:    true
 
-  option :kafka_home,
-         description: 'Kafka home',
-         short:       '-k NAME',
-         long:        '--kafka-home NAME',
-         default:     '/opt/kafka'
-
   option :topic,
          description: 'Comma-separated list of consumer topics',
          short:       '-t NAME',
@@ -62,51 +60,46 @@ class ConsumerOffsetMetrics < Sensu::Plugin::Metric::CLI::Graphite
          long:        '--zookeeper NAME',
          default:     'localhost:2181'
 
-  # read the output of a command
-  # @param cmd [String] the command to read the output from
-  def read_lines(cmd)
-    IO.popen(cmd + ' 2>&1') do |child|
-      child.read.split("\n")
-    end
-  end
+  def run
+    z = Zookeeper.new(config[:zookeeper])
 
-  # create a hash from the output of each line of a command
-  # @param line [String]
-  # @param cols
-  def line_to_hash(line, *cols)
-    Hash[cols.zip(line.strip.split(/\s+/, cols.size))]
-  end
+    group = config[:group]
+    topics = kafka_topics(z, group)
+
+    critical 'Could not found topics' if topics.empty?
+
+    consumers = {}
+    topics.each do |topic|
+      consumers[topic] = []
+
+      topics_partitions(z, topic).each do |partition|
+        leader = leader_broker(z, topic, partition)
+        consumer = Poseidon::PartitionConsumer.new('CheckConsumerLag', leader['host'], leader['port'], topic, partition, :latest_offset)
+        logsize = consumer.next_offset
 
-  # run command and return a hash from the output
-  # @param cmd [String]
-  def run_cmd(cmd)
-    read_lines(cmd).drop(1).map do |line|
-      line_to_hash(line, :group, :topic, :pid, :offset, :logsize, :lag, :owner)
+        offset = consumer_offset(z, group, topic, partition)
+
+        lag = logsize - offset
+        consumers[topic].push(partition: partition, logsize: logsize, offset: offset, lag: lag)
+      end
     end
-  end
 
-  def run
-    begin
-      kafka_run_class = "#{config[:kafka_home]}/bin/kafka-run-class.sh"
-      unknown "Can not find #{kafka_run_class}" unless File.exist?(kafka_run_class)
-
-      cmd = "#{kafka_run_class} kafka.tools.ConsumerOffsetChecker --group #{config[:group]} --zookeeper #{config[:zookeeper]}"
-      cmd += " --topic #{config[:topic]}" if config[:topic]
-
-      results = run_cmd(cmd)
-
-      [:offset, :logsize, :lag].each do |field|
-        sum_by_group = results.group_by { |h| h[:topic] }.map do |k, v|
-          Hash[k, v.inject(0) { |a, e| a + e[field].to_i }]
-        end
-        sum_by_group.delete_if { |x| config[:topic_excludes].include?(x.keys[0]) } if config[:topic_excludes]
-        sum_by_group.each do |x|
-          output "#{config[:scheme]}.#{config[:group]}.#{x.keys[0]}.#{field}", x.values[0]
-        end
+    [:offset, :logsize, :lag].each do |field|
+      consumers.each do |k, v|
+        critical "Topic #{k} has #{field} < 0 '#{v[field]}'" unless v.select { |w| w[field].to_i < 0 }.empty?
+      end
+    end
+
+    [:offset, :logsize, :lag].each do |field|
+      sum_by_group = consumers.map do |k, v|
+        Hash[k, v.inject(0) { |a, e| a + e[field].to_i }]
+      end
+      sum_by_group.each do |x|
+        output "#{config[:scheme]}.#{config[:group]}.#{x.keys[0]}.#{field}", x.values[0]
       end
-    rescue => e
-      critical "Error: exception: #{e}"
     end
     ok
+  rescue => e
+    critical "Error: exception: #{e}"
   end
 end

data/lib/sensu-plugins-kafka/version.rb

@@ -5,8 +5,8 @@ module SensuPluginsKafka
   # This defines the version of the gem
   module Version
     MAJOR = 0
-    MINOR = 8
-    PATCH = 1
+    MINOR = 9
+    PATCH = 0
 
     VER_STRING = [MAJOR, MINOR, PATCH].compact.join('.')
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: sensu-plugins-kafka
 version: !ruby/object:Gem::Version
-  version: 0.8.1
+  version: 0.9.0
 platform: ruby
 authors:
 - Sensu-Plugins and contributors