hermann 0.20.1 → 0.23.0.232

checksums.yaml

@@ -1,7 +1,15 @@
---- 
-SHA1: 
-  metadata.gz: 4fbd82155fea1d32129ee67adcccc3fa420aa320
-  data.tar.gz: cbaa50e297bdceda35c83e6397441d32b9783978
-SHA512: 
-  metadata.gz: 6405cf451dce7e66abaa18e698ead0bfca6e70c0e67736ab378b8bfba0a89eb0368bd4c94e22e79c502ae197f201d0fd51d5c8451776fe671df7dcebd0dccbc9
-  data.tar.gz: 3f92eff84eddb9dbeed39b1cda46b30669431f87566127119660383c128d2af9485302d8f21aed5b5b5bf2582e1ff42bebfdc6cedfaf4cdc421250071921798a
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    Y2UyMjNlM2Q5NTg5NzJlY2MxZWYzNTdiNjA3ZWFlZGVlMWE2MDEwZg==
+  data.tar.gz: !binary |-
+    Mzc0Zjg5OTQ5Zjk3OTJlNzk3YjhjNjdhNWZmMGQwNjdjYmY0MmQ0Yw==
+SHA512:
+  metadata.gz: !binary |-
+    ZWFhYWY5OGZjZmYwOTQ5YzIyM2ExYjQzNDJhMTkyOGRiNzQyOTUwZTdiMzRm
+    NzhlMTI5YWM1ZTkzYzdiMGVlZjUyMGE4ODgyNzI0YWE5MWVhMTMyNzUzYjdj
+    YjEzN2QxNDAyNTgzZGQ1ZDIyZTZmZTBiY2U5ZjFiYjA0Zjk2ZDY=
+  data.tar.gz: !binary |-
+    OWY5ZWY5NTZjMzE3NzNmOTk3OTYyNTc1YmU3YmVlYjg3MmM5OTBlMGM5NDI2
+    MmJmOGI0M2M5NzMxOWQ4YTI4ZTYyNWYwMTgzMTEwYzA4ZDYzODMyOTU5YmVj
+    NjFmNzNmZjhhZjI3MzE0NGUxNTAwOTI1OWJkNjE1NjIwMDUwNTA=

data/lib/hermann.rb

@@ -31,5 +31,5 @@ module Hermann
 end
 
 if Hermann.jruby?
- require 'hermann/java'
+  require 'hermann/java'
 end

data/lib/hermann/consumer.rb

@@ -10,32 +10,27 @@ module Hermann
   # Hermann::Consumer provides a simple consumer API which is only safe to be
   # executed in a single thread
   class Consumer
-    attr_reader :topic, :brokers, :partition, :internal
+    attr_reader :topic, :internal
 
 
     # Instantiate Consumer
     #
     # @params [String] kafka topic
+    # @params [Hash] options for Consumer
+    # @option opts [String]  :brokers    (for MRI) Comma separated list of brokers
+    # @option opts [Integer] :partition  (for MRI) The kafka partition
+    # @option opts [Integer] :zookeepers (for jruby) list of zookeeper servers
+    # @option opts [Integer] :group_id   (for jruby) client group_id
     #
-    # @params [String] group ID
-    #
-    # @params [String] comma separated zookeeper list
-    #
-    # @params [Hash] options for consumer
-    # @option opts [String] :brokers   (for MRI) Comma separated list of brokers
-    # @option opts [String] :partition (for MRI) The kafka partition
-    # @option opts [Fixnum]  :sleep_time (Jruby) Time to sleep between consume retries, defaults to 1sec
-    # @option opts [Boolean] :do_retry (Jruby) Retry consume attempts if exceptions are thrown, defaults to true
-    def initialize(topic, groupId, zookeepers, opts={})
+    def initialize(topic, opts = {})
       @topic = topic
-      @brokers = brokers
-      @partition = partition
-
       if Hermann.jruby?
-        @internal = Hermann::Provider::JavaSimpleConsumer.new(zookeepers, groupId, topic, opts)
+        zookeepers, group_id = require_values_at(opts, :zookeepers, :group_id)
+
+        @internal = Hermann::Provider::JavaSimpleConsumer.new(zookeepers, group_id, topic, opts)
       else
-        brokers   = opts.delete(:brokers)
-        partition = opts.delete(:partition)
+        brokers, partition = require_values_at(opts, :brokers, :partition)
+
         @internal = Hermann::Lib::Consumer.new(topic, brokers, partition)
       end
     end
@@ -53,5 +48,12 @@ module Hermann
         #no op
       end
     end
+
+    def require_values_at(opts, *args)
+      args.map do |a|
+        raise "Please provide :#{a} option!" unless opts[a]
+        opts.delete(a)
+      end
+    end
   end
 end

data/lib/hermann/discovery/zookeeper.rb

@@ -1,21 +1,30 @@
 require 'hermann'
-require 'zk'
+
+if RUBY_PLATFORM == 'java'
+  require 'java'
+end
+
 require 'json'
 require 'hermann/errors'
 
 module Hermann
   module Discovery
-
-
     # Communicates with Zookeeper to discover kafka broker ids
     #
     class Zookeeper
-      attr_reader :zookeepers
+      attr_reader :zookeepers, :impl
 
       BROKERS_PATH = "/brokers/ids".freeze
-
       def initialize(zookeepers)
         @zookeepers = zookeepers
+        @impl =  nil
+        if CuratorImpl.usable?
+          @impl = CuratorImpl.new(zookeepers)
+        elsif ZkGemImpl.usable?
+          @impl = ZkGemImpl.new(zookeepers)
+        else
+          raise Hermann::Errors::GeneralError, "Could not find a usable Zookeeper implementation, please make sure either the `zk` gem is installed or Curator is on the classpath"
+        end
       end
 
       # Gets comma separated string of brokers
@@ -27,10 +36,8 @@ module Hermann
       # @return [Array] List of brokers from ZK
       # @raises [NoBrokersError] if could not discover brokers thru zookeeper
       def get_brokers(timeout=0)
-        brokers = []
-        ZK.open(zookeepers, {:timeout => timeout}) do |zk|
-          brokers = fetch_brokers(zk)
-        end
+        brokers = impl.brokers(timeout).map { |b| format_broker_from_znode(b) }
+
         if brokers.empty?
           raise Hermann::Errors::NoBrokersError
         end
@@ -39,6 +46,43 @@ module Hermann
 
       private
 
+      # Formats the node data into string
+      #
+      # @param [String] node data
+      #
+      # @return [String] formatted node data or empty string if error
+      def format_broker_from_znode(znode)
+        hash = JSON.parse(znode)
+        host = hash['host']
+        port = hash['port']
+        host && port ? "#{host}:#{port}" : nil
+      rescue JSON::ParserError
+        nil
+      end
+
+      # The ZkGemImpl class is an implementation of simple broker discovery
+      # using the `zk` gem if it is available
+      class ZkGemImpl
+        def self.usable?
+          begin
+            require 'zk'
+            return true
+          rescue LoadError
+            return false
+          end
+        end
+
+        def initialize(zks)
+          @zookeepers = zks
+        end
+
+        def brokers(timeout=0)
+          ZK.open(@zookeepers, {:timeout => timeout}) do |zk|
+            return fetch_brokers(zk)
+          end
+        end
+
+        private
         # Gets an Array of broker strings
         #
         # @param [ZK::Client] zookeeper client
@@ -49,9 +93,9 @@ module Hermann
           zk.children(BROKERS_PATH).each do |id|
             node = fetch_znode(zk, id)
             next if node.nil? # whatever error could happen from ZK#get
-            brokers << format_broker_from_znode(node)
+            brokers << node
           end
-          brokers.compact
+          return brokers
         end
 
         # Gets node from zookeeper
@@ -65,20 +109,44 @@ module Hermann
         rescue ZK::Exceptions::NoNode
           nil
         end
+      end
 
-        # Formats the node data into string
-        #
-        # @param [String] node data
-        #
-        # @return [String] formatted node data or empty string if error
-        def format_broker_from_znode(znode)
-          hash = JSON.parse(znode)
-          host = hash['host']
-          port = hash['port']
-          host && port ? "#{host}:#{port}" : nil
-        rescue JSON::ParserError
-          nil
+      # The CuratorImpl is an implementation of simple broker discovery using
+      # Apache Curator libraries, if they are made available on the classpath
+      # for the process running Hermann::Discovery::Zookeeper.
+      #
+      # For a number of reasons this is preferred over the `zk` gem, namely
+      # being a much more simple and mature Zookeeper client interface
+      class CuratorImpl
+        def self.usable?
+          begin
+            Java::OrgApacheCuratorFramework::CuratorFrameworkFactory
+            return true
+          rescue NameError
+            return false
+          end
         end
+
+        def initialize(zks)
+          retry_policy = Java::OrgApacheCuratorRetry::ExponentialBackoffRetry.new(1000, 3)
+          @curator = Java::OrgApacheCuratorFramework::CuratorFrameworkFactory.newClient(zks, retry_policy)
+        end
+
+        # Timeout is discarded, only later versions of Curator support
+        # blockUntilConnected which would take the timeout variable
+        def brokers(timeout=0)
+          unless @curator.started?
+            @curator.start
+          end
+
+          brokers = []
+          @curator.children.for_path(BROKERS_PATH).each do |id|
+            path = "#{BROKERS_PATH}/#{id}"
+            brokers << @curator.data.for_path(path).to_s
+          end
+          return brokers
+        end
+      end
     end
   end
 end

data/lib/hermann/producer.rb

@@ -50,6 +50,7 @@ module Hermann
     # @param [Object] value A single object to push
     # @param [Hash] opts to pass to push method
     # @option opts [String] :topic The topic to push messages to
+    #                       :partition_key The string to partition by
     #
     # @return [Hermann::Result] A future-like object which will store the
     #   result from the broker
@@ -62,7 +63,8 @@ module Hermann
       end
 
       if Hermann.jruby?
-        result = @internal.push_single(value, topic, nil)
+        key = opts.has_key?(:partition_key) ? opts[:partition_key].to_java : nil
+        result = @internal.push_single(value, topic, key)
         unless result.nil?
           @children << result
         end

data/lib/hermann/provider/java_producer.rb

@@ -1,6 +1,7 @@
 require 'hermann'
 require 'concurrent'
 require 'json'
+require 'hermann/errors'
 
 module Hermann
   module Provider
@@ -43,9 +44,9 @@ module Hermann
       # @return +Concurrent::Promise+ Representa a promise to send the
       #   data to the kafka broker.  Upon execution the Promise's status
       #   will be set
-      def push_single(msg, topic, unused)
+      def push_single(msg, topic, key)
         Concurrent::Promise.execute {
-          data = ProducerUtil::KeyedMessage.new(topic, msg.to_java_bytes)
+          data = ProducerUtil::KeyedMessage.new(topic, nil, key, msg.to_java_bytes)
           begin
             @producer.send(data)
           rescue Java::KafkaCommon::FailedToSendMessageException => jexc

data/lib/hermann/provider/java_simple_consumer.rb

@@ -12,11 +12,18 @@ module Hermann
       NUM_THREADS = 1
 
       #default zookeeper connection options
-      DEFAULTS = {
-                   'zookeeper.session.timeout.ms' => '400',
-                   'zookeeper.sync.time.ms'       => '200',
-                   'auto.commit.interval.ms'      => '1000'
-                  }.freeze
+      DEFAULTS_HERMANN_OPTS = {
+        'zookeeper.session.timeout.ms' => '400',
+        'zookeeper.sync.time.ms'       => '200',
+        'auto.commit.interval.ms'      => '1000',
+      }.freeze
+
+      DEFAULT_CONSUMER_OPTIONS = {
+        :do_retry         => true,
+        :max_retries      => 3,
+        :backoff_time_sec => 1,
+        :logger           => nil
+      }.freeze
 
       # Instantiate JavaSimpleConsumer
       #
@@ -27,14 +34,22 @@ module Hermann
       # @params [String] Kafka topic
       #
       # @params [Hash] kafka options for consumer
-      # @option opts [Fixnum] :sleep_time Time to sleep between consume retries, defaults to 1sec
+      # @option opts [Fixnum]  :backoff_time_sec Time to sleep between consume retries, defaults to 1sec
       # @option opts [Boolean] :do_retry Retry consume attempts if exceptions are thrown, defaults to true
+      # @option opts [Fixnum]  :max_retries Number of max_retries to retry #consume when it throws an exception
+      # @option opts [Logger]  :logger Pass in a Logger
+      # @option opts [Other]   other opts from kafka
       def initialize(zookeepers, groupId, topic, opts={})
-        config       = create_config(zookeepers, groupId)
-        @consumer    = ConsumerUtil::Consumer.createJavaConsumerConnector(config)
-        @topic       = topic
-        @sleep_time  = opts.delete(:sleep_time) || 1
-        @do_retry    = opts.delete(:do_retry)   || true
+        @topic            = topic
+        options           = DEFAULT_CONSUMER_OPTIONS.merge(opts)
+        @backoff_time_sec = options.delete(:backoff_time_sec)
+        @do_retry         = options.delete(:do_retry)
+        @max_retries      = options.delete(:max_retries)
+        @logger           = options.delete(:logger)
+        # deleting options above so that they do not get sent to
+        # the create_config method
+        config            = create_config(zookeepers, groupId, options)
+        @consumer         = ConsumerUtil::Consumer.createJavaConsumerConnector(config)
       end
 
       # Shuts down the various threads created by createMessageStreams
@@ -63,14 +78,13 @@ module Hermann
             message = it.next.message
             yield String.from_java_bytes(message)
           end
-        rescue Exception => e
-          puts "#{self.class.name}#consume exception: #{e.class.name}"
-          puts "Msg: #{e.message}"
-          puts e.backtrace.join("\n")
-          if retry?
-            sleep @sleep_time
+        rescue => e
+          if retry? && @max_retries > 0
+            sleep @backoff_time_sec
+            @max_retries -= 1
             retry
           else
+            log_exception(e)
             raise e
           end
         end
@@ -81,6 +95,13 @@ module Hermann
           @do_retry
         end
 
+        def log_exception(e)
+          return unless @logger
+          @logger.error("#{self.class.name}#consume exception: #{e.class.name}")
+          @logger.error("Msg: #{e.message}")
+          @logger.error(e.backtrace.join("\n"))
+        end
+
         # Gets the message stream of the topic. Creates message streams for
         # a topic and the number of threads requested.  In this case the default
         # number of threads is NUM_THREADS.
@@ -106,7 +127,7 @@ module Hermann
         # @raises [RuntimeError] if options does not contain key value strings
         def create_config(zookeepers, groupId, opts={})
           config     = connect_opts(zookeepers, groupId)
-          options    = DEFAULTS.merge(config).merge(opts)
+          options    = DEFAULTS_HERMANN_OPTS.merge(config).merge(opts)
           properties = Hermann.package_properties(options)
           ConsumerUtil::ConsumerConfig.new(properties)
         end

data/lib/hermann/version.rb

@@ -1,3 +1,3 @@
 module Hermann
-  VERSION = '0.20.1'
+  VERSION = '0.23.0'
 end

data/lib/hermann_jars.rb

@@ -1,15 +1,13 @@
 # this is a generated file, to avoid over-writing it just delete this comment
 require 'jar_dependencies'
 
-require_jar( 'junit', 'junit', '3.8.1' )
+require_jar( 'org.apache.zookeeper', 'zookeeper', '3.3.4' )
+require_jar( 'net.sf.jopt-simple', 'jopt-simple', '3.2' )
+require_jar( 'org.xerial.snappy', 'snappy-java', '1.0.5' )
+require_jar( 'jline', 'jline', '0.9.94' )
 require_jar( 'com.101tec', 'zkclient', '0.3' )
-require_jar( 'com.yammer.metrics', 'metrics-core', '2.2.0' )
 require_jar( 'log4j', 'log4j', '1.2.17' )
-require_jar( 'jline', 'jline', '0.9.94' )
-require_jar( 'net.sf.jopt-simple', 'jopt-simple', '3.2' )
-require_jar( 'org.apache.zookeeper', 'zookeeper', '3.3.4' )
-require_jar( 'org.mod4j.org.eclipse.xtext', 'log4j', '1.2.15' )
+require_jar( 'org.scala-lang', 'scala-library', '2.10.1' )
 require_jar( 'org.slf4j', 'slf4j-api', '1.7.2' )
+require_jar( 'com.yammer.metrics', 'metrics-core', '2.2.0' )
 require_jar( 'org.apache.kafka', 'kafka_2.10', '0.8.1.1' )
-require_jar( 'org.scala-lang', 'scala-library', '2.10.1' )
-require_jar( 'org.xerial.snappy', 'snappy-java', '1.0.5' )

metadata

@@ -1,68 +1,69 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: hermann
-version: !ruby/object:Gem::Version 
-  version: 0.20.1
+version: !ruby/object:Gem::Version
+  version: 0.23.0.232
 platform: ruby
-authors: 
+authors:
 - R. Tyler Croy
+- James Way
 - Stan Campbell
 autorequire: 
 bindir: bin
 cert_chain: []
-
-date: 2014-11-13 00:00:00 Z
-dependencies: 
-- !ruby/object:Gem::Dependency 
+date: 2015-04-27 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
   name: concurrent-ruby
-  requirement: &id001 !ruby/object:Gem::Requirement 
-    requirements: 
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
     - - ~>
-      - !ruby/object:Gem::Version 
+      - !ruby/object:Gem::Version
         version: 0.7.0
   type: :runtime
   prerelease: false
-  version_requirements: *id001
-- !ruby/object:Gem::Dependency 
-  name: zk
-  requirement: &id002 !ruby/object:Gem::Requirement 
-    requirements: 
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
     - - ~>
-      - !ruby/object:Gem::Version 
-        version: 1.9.4
-  type: :runtime
-  prerelease: false
-  version_requirements: *id002
-- !ruby/object:Gem::Dependency 
+      - !ruby/object:Gem::Version
+        version: 0.7.0
+- !ruby/object:Gem::Dependency
   name: thread_safe
-  requirement: &id003 !ruby/object:Gem::Requirement 
-    requirements: 
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
     - - ~>
-      - !ruby/object:Gem::Version 
+      - !ruby/object:Gem::Version
         version: 0.3.4
   type: :runtime
   prerelease: false
-  version_requirements: *id003
-- !ruby/object:Gem::Dependency 
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.3.4
+- !ruby/object:Gem::Dependency
   name: mini_portile
-  requirement: &id004 !ruby/object:Gem::Requirement 
-    requirements: 
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
     - - ~>
-      - !ruby/object:Gem::Version 
+      - !ruby/object:Gem::Version
         version: 0.6.0
   type: :runtime
   prerelease: false
-  version_requirements: *id004
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.6.0
 description: Ruby gem for talking to Kafka
-email: 
+email:
 - rtyler.croy@lookout.com
+- james.way@lookout.com
 - stan.campbell3@gmail.com
 executables: []
-
-extensions: 
+extensions:
 - ext/hermann/extconf.rb
 extra_rdoc_files: []
-
-files: 
+files:
 - Rakefile
 - ext/hermann/extconf.rb
 - ext/hermann/hermann_lib.c
@@ -81,31 +82,28 @@ files:
 - lib/hermann/version.rb
 - lib/hermann_jars.rb
 homepage: https://github.com/lookout/Hermann
-licenses: 
+licenses:
 - MIT
 metadata: {}
-
 post_install_message: 
 rdoc_options: []
-
-require_paths: 
+require_paths:
 - lib
 - ext/hermann
-required_ruby_version: !ruby/object:Gem::Requirement 
-  requirements: 
-  - &id005 
-    - ">="
-    - !ruby/object:Gem::Version 
-      version: "0"
-required_rubygems_version: !ruby/object:Gem::Requirement 
-  requirements: 
-  - *id005
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
 requirements: []
-
 rubyforge_project: 
-rubygems_version: 2.4.1
+rubygems_version: 2.4.6
 signing_key: 
 specification_version: 3
 summary: A Kafka consumer/producer gem supporting both MRI and JRuby
 test_files: []
-