librato-storm-kafka 0.0.1

data/.gitignore

@@ -0,0 +1,3 @@
+*.gem
+.bundle
+pkg/*

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in kafka-storm-monitor.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,43 @@
+PATH
+  remote: .
+  specs:
+    librato-storm-kafka (0.0.1)
+      faraday_middleware
+      kafka-rb
+      librato-metrics
+      multi_xml
+      trollop
+      yajl-ruby
+      zookeeper
+
+GEM
+  remote: http://rubygems.org/
+  specs:
+    aggregate (0.2.2)
+    faraday (0.8.4)
+      multipart-post (~> 1.1)
+    faraday_middleware (0.9.0)
+      faraday (>= 0.7.4, < 0.9)
+    kafka-rb (0.0.11)
+    librato-metrics (0.7.4)
+      aggregate (~> 0.2.2)
+      faraday (~> 0.7)
+      multi_json
+    little-plugger (1.1.3)
+    logging (1.7.2)
+      little-plugger (>= 1.1.3)
+    multi_json (1.3.7)
+    multi_xml (0.5.1)
+    multipart-post (1.1.5)
+    rake (10.0.2)
+    trollop (2.0)
+    yajl-ruby (1.1.0)
+    zookeeper (1.3.0)
+      logging (~> 1.7.2)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  librato-storm-kafka!
+  rake

data/README.md

@@ -0,0 +1,72 @@
+# librato-storm-kafka
+
+Automatically detects Kafka spouts from a Storm topology from
+ZooKeeper, determines the Spout consumer offsets, connects to the
+Kafka hosts/partitions and reads current JMX/offset stats, and
+publishes topic metrics to [Librato
+Metrics](https://metrics.librato.com). The metrics published from this
+gem allow the user to graph consumer lag.
+
+Intended to be run periodically, for example, from cron.
+
+### Install
+
+```
+$ gem install librato-storm-kafka
+```
+
+### Requirements
+
+* Storm 0.8.1+ topology
+* storm-contrib dynamic-brokers branch
+* Kafka 0.7.1+ with MX4J
+
+Firewall requirements:
+
+* Access to ZK servers
+* Access to MX4J server on Kafka
+* Access to Kafka on regular port (9092)
+
+### Run
+
+Run with minimal options:
+
+```
+$ librato-storm-kafka  --email <librato email>
+                       --token <librato token>
+                       --zk-servers <comma sperated list of ZK servers>
+                       --zk-prefix <Storm prefix in ZK>
+```
+
+### Options
+
+* `email`: Email associated with your Librato Metrics account.
+* `token`: API token for your Librato Metrics accounts.
+* `prefix`: Prefix to use for metric names. Defaults to
+            **kafkastorm.**.
+* `mx4j-port`: Port to connect to Kafka's MX4J port on (default 8082)
+* `zk-servers`: Comma-separated list of ZK Servers.
+* `zk-port`: ZK port (default 2181)
+* `zk-prefix`: Prefix of Storm spout configs in ZK (What you pass to `SpoutConfig`)
+* `floor-in-secs`: By default all measurements are posted with
+                   the current time as the measure_time. This option,
+                   specified as a value in seconds, will floor the
+                   time by this value. For example, 300 would floor
+                   all measure times to the 5 minute mark.
+
+
+## Contributing to librato-storm-kafka
+
+* Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet.
+* Check out the issue tracker to make sure someone already hasn't requested it and/or contributed it.
+* Fork the project.
+* Start a feature/bugfix branch.
+* Commit and push until you are happy with your contribution.
+* Make sure to add tests for it. This is important so I don't break it in a future version unintentionally.
+* Please try not to mess with the Rakefile, version, or history. If you want to have your own version, or is otherwise necessary, that is fine, but please isolate to its own commit so I can cherry-pick around it.
+
+## Copyright
+
+Copyright (c) 2012 Mike Heffner. See LICENSE.txt for
+further details.
+

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/bin/librato-storm-kafka

@@ -0,0 +1,247 @@
+#!/usr/bin/env ruby
+
+require 'trollop'
+require 'zookeeper'
+require 'yajl/json_gem'
+require 'faraday_middleware'
+require 'kafka'
+
+require 'librato/metrics'
+
+$:.unshift File.join(File.dirname(__FILE__), '../lib')
+
+require 'librato-storm-kafka'
+
+parser = Trollop::Parser.new do
+  version "librato-storm-kafka version %s" %
+    [Librato::Storm::Kafka::VERSION]
+
+  opt :api, "Change the API endpoint", {
+    :type => :string, :default => "metrics-api.librato.com"
+  }
+
+  opt :email, "Librato Metrics Email", {
+    :type => :string, :default => ENV['LIBRATO_METRICS_EMAIL']
+  }
+
+  opt :token, "Librato Metrics API Token", {
+    :type => :string, :default => ENV['LIBRATO_METRICS_TOKEN']
+  }
+
+  opt :prefix, "Metric name prefix", {
+    :type => :string, :default => "kafkastorm."
+  }
+
+  opt :floor_in_secs, "floor() measure times to this interval", {
+    :type => :int
+  }
+
+  opt :mx4j_port, "Port to connect to MX4J on", {
+    :default => 8082
+  }
+
+  opt :zk_servers, "ZooKeeper servers (comma separated)", {
+    :type => :string
+  }
+
+  opt :zk_port, "Port to connect to ZK on", {
+    :default => 2181
+  }
+
+  opt :zk_prefix, "Prefix path in ZK to find consumers", {
+    :default => "/kafkastorm"
+  }
+end
+
+opts = Trollop::with_standard_exception_handling(parser) do
+  if ARGV.length == 0
+    raise Trollop::HelpNeeded
+  end
+
+  opts = parser.parse ARGV
+  %w{ email token zk_servers }.each do |f|
+    unless opts[f.to_sym]
+      $stderr.puts "Error: Must specify option --%s." % [f]
+      puts
+      raise Trollop::HelpNeeded
+    end
+  end
+  opts
+end
+
+$mt = Time.now.tv_sec
+if opts[:floor_in_secs]
+  $mt = ($mt / opts[:floor_in_secs]) * opts[:floor_in_secs]
+end
+
+endpoint = "https://#{opts[:api]}"
+$client = Librato::Metrics::Client.new
+$client.api_endpoint = endpoint
+$client.authenticate opts[:email], opts[:token]
+
+zk_connect_str = opts[:zk_servers].split(",").
+  map {|s| "#{s}:#{opts[:zk_port]}"}.join(",")
+
+$z = Zookeeper.new(zk_connect_str)
+
+def submit(q)
+  begin
+    q.submit
+  rescue => err
+    $stderr.puts "Failed to submit stats to Librato Metrics: %s" %
+      [err.message]
+    exit 1
+  end
+end
+
+def with_children(path, &blk)
+  kids = $z.get_children(:path => path)
+  if kids[:rc] != 0
+    $stderr.puts "Unable to list children at path: #{path}"
+    exit 1
+  end
+  kids[:children].each {|k| yield(k, "#{path}/#{k}") }
+end
+
+def http(host)
+  Faraday.new("http://#{host}:8082") do |conn|
+    conn.response :xml
+
+    conn.adapter Faraday.default_adapter
+  end
+end
+
+def bean_lookup(host, beanname)
+  begin
+    http(host).get("/mbean") do |req|
+      req.params = { :objectname => beanname, :template  => 'identity' }
+      req.options[:timeout] = 5
+      req.options[:open_timeout] = 3
+    end
+  rescue Faraday::Error::TimeoutError
+    raise Errno::ETIMEDOUT
+  rescue Faraday::Error::ConnectionFailed
+    raise Errno::EHOSTUNREACH
+  end
+end
+
+def partition_bean_lookup(host, partname, partidx)
+  bean_lookup(host, "kafka:type=kafka.logs.#{partname}-#{partidx}") 
+end
+
+def jmxinfo_to_attrs(jmxinfo)
+  attrs = {}
+  jmxinfo.body['MBean']['Attribute'].each do |attr|
+    next if attr['isnull'] != 'false'
+    if ['long', 'int'].include?(attr['type'])
+      attrs[attr['name']] = Integer(attr['value'])
+    elsif attr['type'] =~ /CompositeData/
+      # XXX: Hack out the contents of the composite type. This should
+      # really be expanded in the XML
+      fields = attr['value'].gsub(/^.*,contents={/, '').gsub(/}\)$/, '')
+        .gsub(/[=,]/, '@').split("@")
+      attrs[attr['name']] = Hash[*fields.map {|u| u.strip}]
+    else
+      attrs[attr['name']] = attr['value']
+    end
+  end
+  attrs
+end
+
+def monitor_partition(submit_queue, hosts, opts, part)
+  jmxinfo = partition_bean_lookup(part['broker']['host'],
+                                  part['topic'], part['partition'])
+
+  hosts[part['broker']['host']] = 1
+
+  attrs = jmxinfo_to_attrs(jmxinfo)
+
+  #puts "part: #{part.inspect}"
+  #puts "attrs: #{attrs.inspect}"
+
+  consumer = Kafka::Consumer.new({ :host => part['broker']['host'],
+                                   :topic => part['topic'],
+                                   :partition => part['partition']})
+  endoffset = consumer.fetch_latest_offset
+  #puts "%s:%s:%d: latest offset: %d" % [part['broker']['host'], part['topic'], part['partition'], endoffset]
+
+  q = $client.new_queue({ :prefix => "#{opts[:prefix]}topics.#{part['topic']}",
+                          :source => "#{part['broker']['host']}_#{part['partition']}"})
+
+  q.add :bytes_vol => {:value => endoffset, :type => :counter}
+  #endsize = [attrs['CurrentOffset'], attrs['Size']].max
+  diff = endoffset > part['offset'] ? endoffset - part['offset'] : 0
+  q.add :pending_bytes => {:value => diff}
+
+
+  if attrs['NumAppendedMessages'] > 0
+    bytes_per_msg = Float(endoffset) / Float(attrs['NumAppendedMessages'])
+    q.add :bytes_per_msg => {:value => bytes_per_msg}
+    q.add :pending_msgs => {:value => Float(diff) / bytes_per_msg} if bytes_per_msg > 0.0
+    q.add :msg_vol => {:value => attrs['NumAppendedMessages'], :type => :counter}
+  end
+   
+  submit_queue.merge!(q)
+end
+
+partitions = 0
+hosts = {}
+submit_queue = $client.new_queue(:measure_time => $mt)
+with_children(opts[:zk_prefix]) do |spout, fullsp|
+  with_children(fullsp) do |part, fullpart|
+    next if part == 'user'
+
+    partinfo = $z.get(:path => fullpart)
+    if partinfo[:rc] != 0
+      $stderr.puts "Failed to lookup partition: #{fullpart}"
+      exit 1
+    end
+
+    body = JSON.parse(partinfo[:data])
+    begin
+      monitor_partition(submit_queue, hosts, opts, body)
+      partitions += 1
+    rescue Errno::ETIMEDOUT, Errno::EHOSTUNREACH
+    end
+  end
+end
+
+submit_queue.add "#{opts[:prefix]}active_partitions" => {:value => partitions}
+
+# Now check host-level options
+active_hosts = 0
+hosts.each_pair do |host, v|
+  q = $client.new_queue({ :prefix => "#{opts[:prefix]}hosts",
+                          :source => "#{host}"})
+  begin
+    jmxinfo = bean_lookup(host, "kafka:type=kafka.BrokerAllTopicStat")
+    attrs = jmxinfo_to_attrs(jmxinfo)
+    q.add :bytes_in => {:value => attrs['BytesIn'], :type => :counter}
+    q.add :bytes_out => {:value => attrs['BytesOut'], :type => :counter}
+    q.add :failed_fetch_request => {:value => attrs['FailedFetchRequest'], :type => :counter}
+    q.add :failed_produce_request => {:value => attrs['FailedProduceRequest'], :type => :counter}
+    q.add :messages_in => {:value => attrs['MessagesIn'], :type => :counter}
+
+    jmxinfo = bean_lookup(host, "java.lang:type=Memory")
+    attrs = jmxinfo_to_attrs(jmxinfo)
+    attrs.each_pair do |name, values|
+      next unless name =~ /HeapMemory/
+      values.each_pair do |attrn, attrv|
+        q.add "memory.#{name.downcase}.#{attrn.downcase}" => {:value => Integer(attrv)}
+      end
+    end
+
+    active_hosts += 1
+  rescue Errno::ETIMEDOUT, Errno::EHOSTUNREACH
+    next
+  end
+  submit_queue.merge!(q)
+end
+
+submit_queue.add "#{opts[:prefix]}active_hosts" => {:value => active_hosts}
+
+submit_queue.submit
+
+# Local Variables:
+# mode: ruby
+# End:

data/lib/librato-storm-kafka.rb

@@ -0,0 +1,8 @@
+require "librato-storm-kafka/version"
+
+module Librato
+  module Storm
+    module Kafka
+    end
+  end
+end

data/lib/librato-storm-kafka/version.rb

@@ -0,0 +1,7 @@
+module Librato
+  module Storm
+    module Kafka
+      VERSION = "0.0.1"
+    end
+  end
+end

data/librato-storm-kafka.gemspec

@@ -0,0 +1,34 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "librato-storm-kafka/version"
+
+Gem::Specification.new do |s|
+  s.name        = "librato-storm-kafka"
+  s.version     = Librato::Storm::Kafka::VERSION
+  s.authors     = ["Mike Heffner"]
+  s.email       = ["mike@librato.com"]
+  s.homepage    = ""
+  s.summary     = %q{Monitor storm-kafka topics}
+  s.description = %q{Collects metrics from a storm-kafka setup and publishes them to Librato}
+
+  s.rubyforge_project = "librato-storm-kafka"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  s.add_runtime_dependency "zookeeper"
+  s.add_runtime_dependency "librato-metrics"
+  s.add_runtime_dependency "trollop"
+  s.add_runtime_dependency "yajl-ruby"
+  s.add_runtime_dependency "faraday_middleware"
+  s.add_runtime_dependency "multi_xml"
+  s.add_runtime_dependency "kafka-rb"
+
+  s.add_development_dependency "rake"
+
+  # specify any dependencies here; for example:
+  # s.add_development_dependency "rspec"
+  # s.add_runtime_dependency "rest-client"
+end

metadata

@@ -0,0 +1,189 @@
+--- !ruby/object:Gem::Specification
+name: librato-storm-kafka
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Mike Heffner
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-11-22 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: zookeeper
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: librato-metrics
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: trollop
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: yajl-ruby
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: faraday_middleware
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: multi_xml
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: kafka-rb
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Collects metrics from a storm-kafka setup and publishes them to Librato
+email:
+- mike@librato.com
+executables:
+- librato-storm-kafka
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- Gemfile.lock
+- README.md
+- Rakefile
+- bin/librato-storm-kafka
+- lib/librato-storm-kafka.rb
+- lib/librato-storm-kafka/version.rb
+- librato-storm-kafka.gemspec
+homepage: ''
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: -917739231179684100
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: -917739231179684100
+requirements: []
+rubyforge_project: librato-storm-kafka
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: Monitor storm-kafka topics
+test_files: []