hermann 0.20.1-java → 0.20.2-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 023ddf7f24fc065abbf63cf567200781f4f57689
-  data.tar.gz: 19a5ba3a7bbde50e1dbcde96d5da7d380fba989f
+  metadata.gz: 8c9f75e450674ab97eae2977213013ec3c22f9e6
+  data.tar.gz: 4a3fb3da2b3c6776172d9b550bfcfedaa7da9066
 SHA512:
-  metadata.gz: f96fc2c6a866c4e6fdb5b7e28f5bcc16354011ec350be4b9481e5895cba5e0899a56bd31d917f100b164333cf351560f144d290f6fbe86fc2a1b21f248a061f4
-  data.tar.gz: 5ee23b2de80fea374fbf896238c0c1e92bfda576faa9e2f9175b7d6d6c59a6c44a20775431187fa82158cc9dd8f9962c5851d69af70d29bfd29b8333865a93b4
+  metadata.gz: 3a0003509edf79b5c388d6fe22c496124613a008dc055e2889fcda5d439b825f0a65e22f860d0598597d1e999745f98df6659638cf6e27ec8b3c284ae3258a3b
+  data.tar.gz: ba0fd53110ede6966aa1a3fb460a520c4299e20bebd784f9d92cc868001f24cdbaee53b206061743fcfff528ec92dad4e86d8288999db16f36b1b01ab49ce06b

data/lib/hermann/consumer.rb

@@ -21,11 +21,9 @@ module Hermann
     #
     # @params [String] comma separated zookeeper list
     #
-    # @params [Hash] options for consumer
+    # @params [Hash] options for Consumer
     # @option opts [String] :brokers   (for MRI) Comma separated list of brokers
-    # @option opts [String] :partition (for MRI) The kafka partition
-    # @option opts [Fixnum]  :sleep_time (Jruby) Time to sleep between consume retries, defaults to 1sec
-    # @option opts [Boolean] :do_retry (Jruby) Retry consume attempts if exceptions are thrown, defaults to true
+    # @option opts [Integer] :partition (for MRI) The kafka partition
     def initialize(topic, groupId, zookeepers, opts={})
       @topic = topic
       @brokers = brokers

data/lib/hermann/discovery/zookeeper.rb

@@ -1,21 +1,30 @@
 require 'hermann'
-require 'zk'
+
+if RUBY_PLATFORM == 'java'
+  require 'java'
+end
+
 require 'json'
 require 'hermann/errors'
 
 module Hermann
   module Discovery
-
-
     # Communicates with Zookeeper to discover kafka broker ids
     #
     class Zookeeper
-      attr_reader :zookeepers
+      attr_reader :zookeepers, :impl
 
       BROKERS_PATH = "/brokers/ids".freeze
-
       def initialize(zookeepers)
         @zookeepers = zookeepers
+        @impl =  nil
+        if CuratorImpl.usable?
+          @impl = CuratorImpl.new(zookeepers)
+        elsif ZkGemImpl.usable?
+          @impl = ZkGemImpl.new(zookeepers)
+        else
+          raise Hermann::Errors::GeneralError, "Could not find a usable Zookeeper implementation, please make sure either the `zk` gem is installed or Curator is on the classpath"
+        end
       end
 
       # Gets comma separated string of brokers
@@ -27,10 +36,8 @@ module Hermann
       # @return [Array] List of brokers from ZK
       # @raises [NoBrokersError] if could not discover brokers thru zookeeper
       def get_brokers(timeout=0)
-        brokers = []
-        ZK.open(zookeepers, {:timeout => timeout}) do |zk|
-          brokers = fetch_brokers(zk)
-        end
+        brokers = impl.brokers(timeout).map { |b| format_broker_from_znode(b) }
+
         if brokers.empty?
           raise Hermann::Errors::NoBrokersError
         end
@@ -39,6 +46,43 @@ module Hermann
 
       private
 
+      # Formats the node data into string
+      #
+      # @param [String] node data
+      #
+      # @return [String] formatted node data or empty string if error
+      def format_broker_from_znode(znode)
+        hash = JSON.parse(znode)
+        host = hash['host']
+        port = hash['port']
+        host && port ? "#{host}:#{port}" : nil
+      rescue JSON::ParserError
+        nil
+      end
+
+      # The ZkGemImpl class is an implementation of simple broker discovery
+      # using the `zk` gem if it is available
+      class ZkGemImpl
+        def self.usable?
+          begin
+            require 'zk'
+            return true
+          rescue LoadError
+            return false
+          end
+        end
+
+        def initialize(zks)
+          @zookeepers = zks
+        end
+
+        def brokers(timeout=0)
+          ZK.open(@zookeepers, {:timeout => timeout}) do |zk|
+            return fetch_brokers(zk)
+          end
+        end
+
+        private
         # Gets an Array of broker strings
         #
         # @param [ZK::Client] zookeeper client
@@ -49,9 +93,9 @@ module Hermann
           zk.children(BROKERS_PATH).each do |id|
             node = fetch_znode(zk, id)
             next if node.nil? # whatever error could happen from ZK#get
-            brokers << format_broker_from_znode(node)
+            brokers << node
           end
-          brokers.compact
+          return brokers
         end
 
         # Gets node from zookeeper
@@ -65,20 +109,44 @@ module Hermann
         rescue ZK::Exceptions::NoNode
           nil
         end
+      end
 
-        # Formats the node data into string
-        #
-        # @param [String] node data
-        #
-        # @return [String] formatted node data or empty string if error
-        def format_broker_from_znode(znode)
-          hash = JSON.parse(znode)
-          host = hash['host']
-          port = hash['port']
-          host && port ? "#{host}:#{port}" : nil
-        rescue JSON::ParserError
-          nil
+      # The CuratorImpl is an implementation of simple broker discovery using
+      # Apache Curator libraries, if they are made available on the classpath
+      # for the process running Hermann::Discovery::Zookeeper.
+      #
+      # For a number of reasons this is preferred over the `zk` gem, namely
+      # being a much more simple and mature Zookeeper client interface
+      class CuratorImpl
+        def self.usable?
+          begin
+            Java::OrgApacheCuratorFramework::CuratorFrameworkFactory
+            return true
+          rescue NameError
+            return false
+          end
         end
+
+        def initialize(zks)
+          retry_policy = Java::OrgApacheCuratorRetry::ExponentialBackoffRetry.new(1000, 3)
+          @curator = Java::OrgApacheCuratorFramework::CuratorFrameworkFactory.newClient(zks, retry_policy)
+        end
+
+        # Timeout is discarded, only later versions of Curator support
+        # blockUntilConnected which would take the timeout variable
+        def brokers(timeout=0)
+          unless @curator.started?
+            @curator.start
+          end
+
+          brokers = []
+          @curator.children.for_path(BROKERS_PATH).each do |id|
+            path = "#{BROKERS_PATH}/#{id}"
+            brokers << @curator.data.for_path(path).to_s
+          end
+          return brokers
+        end
+      end
     end
   end
 end

data/lib/hermann/provider/java_producer.rb

@@ -1,6 +1,7 @@
 require 'hermann'
 require 'concurrent'
 require 'json'
+require 'hermann/errors'
 
 module Hermann
   module Provider

data/lib/hermann/provider/java_simple_consumer.rb

@@ -12,11 +12,18 @@ module Hermann
       NUM_THREADS = 1
 
       #default zookeeper connection options
-      DEFAULTS = {
-                   'zookeeper.session.timeout.ms' => '400',
-                   'zookeeper.sync.time.ms'       => '200',
-                   'auto.commit.interval.ms'      => '1000'
-                  }.freeze
+      DEFAULTS_HERMANN_OPTS = {
+        'zookeeper.session.timeout.ms' => '400',
+        'zookeeper.sync.time.ms'       => '200',
+        'auto.commit.interval.ms'      => '1000'
+      }.freeze
+
+      DEFAULT_CONSUMER_OPTIONS = {
+        :do_retry         => true,
+        :max_retries      => 3,
+        :backoff_time_sec => 1,
+        :logger           => nil
+      }.freeze
 
       # Instantiate JavaSimpleConsumer
       #
@@ -27,14 +34,20 @@ module Hermann
       # @params [String] Kafka topic
       #
       # @params [Hash] kafka options for consumer
-      # @option opts [Fixnum] :sleep_time Time to sleep between consume retries, defaults to 1sec
+      # @option opts [Fixnum]  :backoff_time_sec Time to sleep between consume retries, defaults to 1sec
       # @option opts [Boolean] :do_retry Retry consume attempts if exceptions are thrown, defaults to true
+      # @option opts [Fixnum]  :max_retries Number of max_retries to retry #consume when it throws an exception
+      # @option opts [Logger]  :logger Pass in a Logger
       def initialize(zookeepers, groupId, topic, opts={})
-        config       = create_config(zookeepers, groupId)
-        @consumer    = ConsumerUtil::Consumer.createJavaConsumerConnector(config)
-        @topic       = topic
-        @sleep_time  = opts.delete(:sleep_time) || 1
-        @do_retry    = opts.delete(:do_retry)   || true
+        config            = create_config(zookeepers, groupId)
+        @consumer         = ConsumerUtil::Consumer.createJavaConsumerConnector(config)
+        @topic            = topic
+
+        options           = DEFAULT_CONSUMER_OPTIONS.merge(opts)
+        @backoff_time_sec = options[:backoff_time_sec]
+        @do_retry         = options[:do_retry]
+        @max_retries      = options[:max_retries]
+        @logger           = options[:logger]
       end
 
       # Shuts down the various threads created by createMessageStreams
@@ -63,14 +76,13 @@ module Hermann
             message = it.next.message
             yield String.from_java_bytes(message)
           end
-        rescue Exception => e
-          puts "#{self.class.name}#consume exception: #{e.class.name}"
-          puts "Msg: #{e.message}"
-          puts e.backtrace.join("\n")
-          if retry?
-            sleep @sleep_time
+        rescue => e
+          if retry? && @max_retries > 0
+            sleep @backoff_time_sec
+            @max_retries -= 1
             retry
           else
+            log_exception(e)
             raise e
           end
         end
@@ -81,6 +93,13 @@ module Hermann
           @do_retry
         end
 
+        def log_exception(e)
+          return unless @logger
+          @logger.error("#{self.class.name}#consume exception: #{e.class.name}")
+          @logger.error("Msg: #{e.message}")
+          @logger.error(e.backtrace.join("\n"))
+        end
+
         # Gets the message stream of the topic. Creates message streams for
         # a topic and the number of threads requested.  In this case the default
         # number of threads is NUM_THREADS.
@@ -106,7 +125,7 @@ module Hermann
         # @raises [RuntimeError] if options does not contain key value strings
         def create_config(zookeepers, groupId, opts={})
           config     = connect_opts(zookeepers, groupId)
-          options    = DEFAULTS.merge(config).merge(opts)
+          options    = DEFAULTS_HERMANN_OPTS.merge(config).merge(opts)
           properties = Hermann.package_properties(options)
           ConsumerUtil::ConsumerConfig.new(properties)
         end

data/lib/hermann/version.rb

@@ -1,3 +1,3 @@
 module Hermann
-  VERSION = '0.20.1'
+  VERSION = '0.20.2'
 end

metadata

@@ -1,15 +1,16 @@
 --- !ruby/object:Gem::Specification
 name: hermann
 version: !ruby/object:Gem::Version
-  version: 0.20.1
+  version: 0.20.2
 platform: java
 authors:
 - R. Tyler Croy
+- James Way
 - Stan Campbell
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2014-11-13 00:00:00.000000000 Z
+date: 2015-02-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -25,20 +26,6 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: 0.7.0
-- !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: 1.9.4
-  name: zk
-  prerelease: false
-  type: :runtime
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: 1.9.4
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -67,37 +54,10 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: 0.1.2
-- !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: 3.1.1.0
-  name: ruby-maven
-  prerelease: false
-  type: :development
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: 3.1.1.0
-- !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  name: rake
-  prerelease: false
-  type: :development
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: '0'
 description: Ruby gem for talking to Kafka
 email:
 - rtyler.croy@lookout.com
+- james.way@lookout.com
 - stan.campbell3@gmail.com
 executables: []
 extensions: []
@@ -147,7 +107,7 @@ requirements:
 - jar log4j:log4j, 1.2.17
 - jar com.101tec:zkclient, 0.3
 rubyforge_project:
-rubygems_version: 2.2.2
+rubygems_version: 2.4.5
 signing_key:
 specification_version: 3
 summary: A Kafka consumer/producer gem supporting both MRI and JRuby