hermann 0.18.1 → 0.19.0

checksums.yaml

@@ -1,7 +1,7 @@
 --- 
 SHA1: 
-  metadata.gz: f84207f3ca150090784e3e241945bb9b84dad953
-  data.tar.gz: d7b9f0e0d83906d8eda31f6f88551f7bf7cd8100
+  metadata.gz: df28bdfb07a0ed99d0bd3b2e60bf0a40a078f359
+  data.tar.gz: e5a9810c7e622be6d7aef9f131d7b436ded85771
 SHA512: 
-  metadata.gz: bcba75e278f588a9c0f208d8317e0e775ea3f8a656b798a27b1b70e3d36fc7b1865985d224e002bc788230c645996098ebc217285c6ac23762f27f499ee0e293
-  data.tar.gz: 0031e641ec11e16c08db2bbabe56c048279f59e0e45b1f2dd782af2908a3fe8bc6d7e44e7371dbbc55a03f6b2d7c539e77f0e485111dd28b296c9899c028fff2
+  metadata.gz: 734f5dd11fc62af1e8777a6a07bf135af0974f6aac12a493318b63dc7b8268c597b1927daf4410d305524b80f4c6a8a05bcf953529ddc59e72a5d08d9ad2d19f
+  data.tar.gz: a4d0de27069f7638c61a66eba2746e0e47a544c9428efa37b737d0471a4fd04ffc1502a325a53177027c74f341994ff821379ea4ad2d79e71814de8ff35405bd

data/ext/hermann/hermann_lib.c

@@ -379,13 +379,12 @@ void consumer_consume_loop(HermannInstanceConfig* consumerConfig) {
 }
 
 /**
- * Hermann::Consumer.consume
- *
- * Begin listening on the configured topic for messages.  msg_consume will be called on each message received.
+ * Hermann::Lib::Consumer.consume
  *
  * @param   VALUE   self	the Ruby object for this consumer
+ * @param   VALUE   topic	the Ruby string representing a topic to consume
  */
-static VALUE consumer_consume(VALUE self) {
+static VALUE consumer_consume(VALUE self, VALUE topic) {
 
 	HermannInstanceConfig* consumerConfig;
 
@@ -1012,7 +1011,7 @@ void Init_hermann_lib() {
 	rb_define_method(c_consumer, "initialize_copy", consumer_init_copy, 1);
 
 	/* Consumer has method 'consume' */
-	rb_define_method( c_consumer, "consume", consumer_consume, 0 );
+	rb_define_method( c_consumer, "consume", consumer_consume, 1);
 
 	/* ---- Define the producer class ---- */
 	c_producer = rb_define_class_under(lib_module, "Producer", rb_cObject);

data/lib/hermann.rb

@@ -2,19 +2,34 @@ module Hermann
   def self.jruby?
     return RUBY_PLATFORM == "java"
   end
-
-  if self.jruby?
-    require 'java'
-    require 'hermann_jars'
-
-    module JavaUtil
-      include_package 'java.util'
+  # Validates that the args are non-blank strings
+  #
+  # @param [Object] key to validate
+  #
+  # @param [Object] val to validate
+  #
+  # @raises [ConfigurationErorr] if either values are empty
+  def self.validate_property!(key, val)
+    if key.to_s.empty? || val.to_s.empty?
+      raise Hermann::Errors::ConfigurationError
     end
-    module ProducerUtil
-      include_package 'kafka.producer'
-    end
-    module JavaApiUtil
-      include_package 'kafka.javaapi.producer'
+  end
+
+  # Packages options into Java Properties object
+  #
+  # @params [Hash] hash of options to package
+  #
+  # @return [Properties] packaged java properties
+  def self.package_properties(options)
+    properties = JavaUtil::Properties.new
+    options.each do |key, val|
+      Hermann.validate_property!(key, val)
+      properties.put(key, val)
     end
+    properties
   end
 end
+
+if Hermann.jruby?
+ require 'hermann/java'
+end

data/lib/hermann/consumer.rb

@@ -1,24 +1,57 @@
 require 'hermann'
 
-unless Hermann.jruby?
+if Hermann.jruby?
+  require 'hermann/provider/java_simple_consumer'
+else
   require 'hermann_lib'
 end
 
 module Hermann
+  # Hermann::Consumer provides a simple consumer API which is only safe to be
+  # executed in a single thread
   class Consumer
     attr_reader :topic, :brokers, :partition, :internal
 
-    def initialize(topic, brokers, partition)
+
+    # Instantiate Consumer
+    #
+    # @params [String] kafka topic
+    #
+    # @params [String] group ID
+    #
+    # @params [String] comma separated zookeeper list
+    #
+    # @params [Hash] options for consumer
+    # @option opts [String] :brokers   (for MRI) Comma separated list of brokers
+    # @option opts [String] :partition (for MRI) The kafka partition
+    # @option opts [Fixnum]  :sleep_time (Jruby) Time to sleep between consume retries, defaults to 1sec
+    # @option opts [Boolean] :do_retry (Jruby) Retry consume attempts if exceptions are thrown, defaults to true
+    def initialize(topic, groupId, zookeepers, opts={})
       @topic = topic
       @brokers = brokers
       @partition = partition
-      unless Hermann.jruby?
+
+      if Hermann.jruby?
+        @internal = Hermann::Provider::JavaSimpleConsumer.new(zookeepers, groupId, topic, opts)
+      else
+        brokers   = opts.delete(:brokers)
+        partition = opts.delete(:partition)
         @internal = Hermann::Lib::Consumer.new(topic, brokers, partition)
       end
     end
 
-    def consume(&block)
-      @internal.consume(&block)
+    # Delegates the consume method to internal consumer classes
+    def consume(topic=nil, &block)
+      @internal.consume(topic, &block)
+    end
+
+    # Delegates the shutdown of kafka messages threads to internal consumer classes
+    def shutdown
+      if Hermann.jruby?
+        @internal.shutdown
+      else
+        #no op
+      end
     end
   end
 end

data/lib/hermann/errors.rb

@@ -2,7 +2,20 @@
 module Hermann
   module Errors
     # Error for connectivity problems with the Kafka brokers
-    class ConnectivityError; end;
+    class ConnectivityError < StandardError
+      attr_reader :java_exception
+
+      # Initialize a connectivity error
+      #
+      # @param [String] message Exception's message
+      # @param [Hash[ options
+      # @option options [Java::Lang::RuntimeException] :java_exception An
+      #   underlying Java exception
+      def initialize(message, options={})
+        super(message)
+        @java_exception = options[:java_exception]
+      end
+    end
 
     # For passing incorrect config and options to kafka
     class ConfigurationError < StandardError; end

data/lib/hermann/java.rb

@@ -0,0 +1,20 @@
+module Hermann
+  require 'java'
+  require 'hermann_jars'
+
+  module JavaUtil
+    include_package 'java.util'
+  end
+  module ProducerUtil
+    include_package 'kafka.producer'
+  end
+  module ConsumerUtil
+    include_package "kafka.consumer"
+  end
+  module JavaApiUtil
+    include_package 'kafka.javaapi.producer'
+  end
+  module KafkaUtil
+    include_package "kafka.util"
+  end
+end

data/lib/hermann/producer.rb

@@ -1,7 +1,8 @@
+require 'thread_safe'
+
 require 'hermann'
 require 'hermann/result'
 
-
 if RUBY_PLATFORM == "java"
   require 'hermann/provider/java_producer'
 else
@@ -18,15 +19,15 @@ module Hermann
     # @param [Array] brokers An array of "host:port" strings for the brokers
     def initialize(topic, brokers, opts={})
       @topic = topic
-      @brokers = brokers
-      if RUBY_PLATFORM == "java"
+      @brokers = ThreadSafe::Array.new(brokers)
+      if Hermann.jruby?
         @internal = Hermann::Provider::JavaProducer.new(brokers, opts)
       else
-        @internal = Hermann::Lib::Producer.new(brokers)
+        @internal = Hermann::Lib::Producer.new(brokers.join(','))
       end
       # We're tracking children so we can make sure that at Producer exit we
       # make a reasonable attempt to clean up outstanding result objects
-      @children = []
+      @children = ThreadSafe::Array.new
     end
 
     # @return [Boolean] True if our underlying producer object thinks it's
@@ -60,7 +61,7 @@ module Hermann
         return value.map { |e| self.push(e, opts) }
       end
 
-      if RUBY_PLATFORM == "java"
+      if Hermann.jruby?
         result = @internal.push_single(value, topic, nil)
         unless result.nil?
           @children << result
@@ -69,6 +70,9 @@ module Hermann
         # called correctly and we don't leak memory
         reap_children
       else
+        # Ticking reactor to make sure that we don't inadvertantly let the
+        # librdkafka callback queue overflow
+        tick_reactor
         result = create_result
         @internal.push_single(value, topic, result)
       end
@@ -118,7 +122,6 @@ module Hermann
       return (total_children - children.size)
     end
 
-
     private
 
     def rounded_timeout(timeout)

data/lib/hermann/provider/java_producer.rb

@@ -11,6 +11,13 @@ module Hermann
     class JavaProducer
       attr_accessor :producer
 
+      #default kafka Producer options
+      DEFAULTS = {
+                    'serializer.class'      => 'kafka.serializer.StringEncoder',
+                    'partitioner.class'     => 'kafka.producer.DefaultPartitioner',
+                    'request.required.acks' => '1',
+                    'message.send.max.retries' => '0'
+                  }.freeze
 
       # Instantiate JavaProducer
       #
@@ -25,17 +32,10 @@ module Hermann
       # JavaProducer.new('0:9092', {'request.required.acks' => '1'})
       #
       def initialize(brokers, opts={})
-        properties  = create_properties(brokers, opts)
-        config      = create_config(properties)
+        config      = create_config(brokers, opts)
         @producer   = JavaApiUtil::Producer.new(config)
       end
 
-      DEFAULTS = {
-                    'serializer.class'      => 'kafka.serializer.StringEncoder',
-                    'partitioner.class'     => 'kafka.producer.DefaultPartitioner',
-                    'request.required.acks' => '1'
-                  }.freeze
-
       # Push a value onto the Kafka topic passed to this +Producer+
       #
       # @param [Object] value A single object to push
@@ -47,7 +47,12 @@ module Hermann
       def push_single(msg, topic, unused)
         Concurrent::Promise.execute {
           data = ProducerUtil::KeyedMessage.new(topic, msg)
-          @producer.send(data)
+          begin
+            @producer.send(data)
+          rescue Java::KafkaCommon::FailedToSendMessageException => jexc
+            raise Hermann::Errors::ConnectivityError.new(jexc.message,
+                                                         :java_exception => jexc)
+          end
         }
       end
 
@@ -67,39 +72,21 @@ module Hermann
       end
 
       private
-
         # Creates a ProducerConfig object
         #
-        # @param [Properties] object with broker properties
-        #
-        # @return [ProducerConfig] - packaged config for +Producer+
-        def create_config(properties)
-          ProducerUtil::ProducerConfig.new(properties)
-        end
-
-        # Creates Properties Object
+        # @param [String] comma separated list of brokers
         #
         # @param [Hash] brokers passed into this function
         # @option args [String] :brokers - string of brokers
         #
-        # @return [Properties] properties object for creating +ProducerConfig+
+        # @return [ProducerConfig] - packaged config for +Producer+
         #
         # @raises [RuntimeError] if options does not contain key value strings
-        def create_properties(brokers, opts={})
-          brokers = { 'metadata.broker.list' => brokers }
-          options = DEFAULTS.merge(brokers).merge(opts)
-          properties = JavaUtil::Properties.new
-          options.each do |key, val|
-            validate_property!(key, val)
-            properties.put(key, val)
-          end
-          properties
-        end
-
-        def validate_property!(key, val)
-          if key.to_s.empty? || val.to_s.empty?
-            raise Hermann::Errors::ConfigurationError, "Invalid Broker Properties"
-          end
+        def create_config(brokers, opts={})
+          brokers    = { 'metadata.broker.list' => brokers }
+          options    = DEFAULTS.merge(brokers).merge(opts)
+          properties = Hermann.package_properties(options)
+          ProducerUtil::ProducerConfig.new(properties)
         end
     end
   end

data/lib/hermann/provider/java_simple_consumer.rb

@@ -0,0 +1,122 @@
+require 'hermann'
+require 'hermann/errors'
+
+module Hermann
+  module Provider
+
+    # Implements a java based consumer.  The #consumer method loops infinitely,
+    # the hasNext() method blocks until a message is available.
+    class JavaSimpleConsumer
+      attr_accessor :consumer, :topic, :zookeeper
+
+      NUM_THREADS = 1
+
+      #default zookeeper connection options
+      DEFAULTS = {
+                   'zookeeper.session.timeout.ms' => '400',
+                   'zookeeper.sync.time.ms'       => '200',
+                   'auto.commit.interval.ms'      => '1000'
+                  }.freeze
+
+      # Instantiate JavaSimpleConsumer
+      #
+      # @params [String] list of zookeepers
+      #
+      # @params [String] Group ID
+      #
+      # @params [String] Kafka topic
+      #
+      # @params [Hash] kafka options for consumer
+      # @option opts [Fixnum] :sleep_time Time to sleep between consume retries, defaults to 1sec
+      # @option opts [Boolean] :do_retry Retry consume attempts if exceptions are thrown, defaults to true
+      def initialize(zookeepers, groupId, topic, opts={})
+        config       = create_config(zookeepers, groupId)
+        @consumer    = ConsumerUtil::Consumer.createJavaConsumerConnector(config)
+        @topic       = topic
+        @sleep_time  = opts.delete(:sleep_time) || 1
+        @do_retry    = opts.delete(:do_retry)   || true
+      end
+
+      # Shuts down the various threads created by createMessageStreams
+      # This can be called after the thread executing consume has exited
+      # to clean up.
+      def shutdown
+        @consumer.shutdown
+      end
+
+      # Starts infinite loop to consume messages. hasNext() blocks until a
+      # message is available at which point it is yielded to the block
+      #
+      # @params [String] optional topic to override initialized topic
+      #
+      # ==== Examples
+      #
+      # consumer.consume do |message|
+      #   puts "Received: #{message}"
+      # end
+      #
+      def consume(topic=nil)
+        begin
+          stream = get_stream(topic)
+          it = stream.iterator
+          while it.hasNext do
+            yield it.next.message.to_s
+          end
+        rescue Exception => e
+          puts "#{self.class.name}#consume exception: #{e.class.name}"
+          puts "Msg: #{e.message}"
+          puts e.backtrace.join("\n")
+          if retry?
+            sleep @sleep_time
+            retry
+          else
+            raise e
+          end
+        end
+      end
+
+      private
+        def retry?
+          @do_retry
+        end
+
+        # Gets the message stream of the topic. Creates message streams for
+        # a topic and the number of threads requested.  In this case the default
+        # number of threads is NUM_THREADS.
+        #
+        # @params [String] optional topic to override initialized topic
+        def get_stream(topic)
+          current_topic = topic || @topic
+          @topicCountMap = JavaUtil::HashMap.new
+          @value         = NUM_THREADS.to_java Java::int
+          @topicCountMap.put("#{current_topic}", @value)
+          consumerMap = @consumer.createMessageStreams(@topicCountMap)
+          consumerMap[current_topic].first
+        end
+
+        # Creates a ConsumerConfig object
+        #
+        # @param [String] zookeepers list
+        #
+        # @param [String] group ID
+        #
+        # @return [ConsumerConfig] - packaged config for +Consumer+
+        #
+        # @raises [RuntimeError] if options does not contain key value strings
+        def create_config(zookeepers, groupId, opts={})
+          config     = connect_opts(zookeepers, groupId)
+          options    = DEFAULTS.merge(config).merge(opts)
+          properties = Hermann.package_properties(options)
+          ConsumerUtil::ConsumerConfig.new(properties)
+        end
+
+        # Connection options to pass to ConsumerConfig
+        def connect_opts(zookeepers, groupId)
+          {
+            'zookeeper.connect' => zookeepers,
+            'group.id' => groupId
+          }
+        end
+    end
+  end
+end

data/lib/hermann/version.rb

@@ -1,3 +1,3 @@
 module Hermann
-  VERSION = '0.18.1'
+  VERSION = '0.19.0'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: hermann
 version: !ruby/object:Gem::Version 
-  version: 0.18.1
+  version: 0.19.0
 platform: ruby
 authors: 
 - R. Tyler Croy
@@ -10,7 +10,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2014-10-15 00:00:00 Z
+date: 2014-10-29 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: concurrent-ruby
@@ -33,15 +33,25 @@ dependencies:
   prerelease: false
   version_requirements: *id002
 - !ruby/object:Gem::Dependency 
-  name: mini_portile
+  name: thread_safe
   requirement: &id003 !ruby/object:Gem::Requirement 
     requirements: 
     - - ~>
       - !ruby/object:Gem::Version 
-        version: 0.6.0
+        version: 0.3.4
   type: :runtime
   prerelease: false
   version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  name: mini_portile
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        version: 0.6.0
+  type: :runtime
+  prerelease: false
+  version_requirements: *id004
 description: Ruby gem for talking to Kafka
 email: 
 - rtyler.croy@lookout.com
@@ -62,8 +72,10 @@ files:
 - lib/hermann/consumer.rb
 - lib/hermann/discovery/zookeeper.rb
 - lib/hermann/errors.rb
+- lib/hermann/java.rb
 - lib/hermann/producer.rb
 - lib/hermann/provider/java_producer.rb
+- lib/hermann/provider/java_simple_consumer.rb
 - lib/hermann/result.rb
 - lib/hermann/timeout.rb
 - lib/hermann/version.rb
@@ -81,13 +93,13 @@ require_paths:
 - ext/hermann
 required_ruby_version: !ruby/object:Gem::Requirement 
   requirements: 
-  - &id004 
+  - &id005 
     - ">="
     - !ruby/object:Gem::Version 
       version: "0"
 required_rubygems_version: !ruby/object:Gem::Requirement 
   requirements: 
-  - *id004
+  - *id005
 requirements: []
 
 rubyforge_project: