jruby-kafka 1.0.0.beta-java → 1.1.0.beta-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 858a7e74d47a3e26ef8593d31d8df34f1e586a11
-  data.tar.gz: c7041f8ae3d5d8c4e03b4ae91dadc53abd8fc6d7
+  metadata.gz: 77a0012bfe5724b42999e603732764a6730bab39
+  data.tar.gz: 0a204a31ab2b78d8809f27d725a2a01fd520112d
 SHA512:
-  metadata.gz: 3d3811e0c53babc95b5c3934f98b4dffa1a775d51f01750b05ae7ed89a1a34ef846d8c4be58b63bb54c284f095af4158fbf085757f61fa6ef67d815e886e1f77
-  data.tar.gz: ab497a15f75483d28fded861fb1273ace051b7a891a288a43363061bb318a09f1991fed7eb6da9bd7271f194bd2a6c9dc4c5e8af0b40a7254dd7070a7641c501
+  metadata.gz: 45b293197c14ee6409d647cacab10aa644153eabe8f8d37e2ad23f6bdb22d0e435252033423e2da6f921694e18abd8b3297d20164fd995a1f46ddbcf00840e25
+  data.tar.gz: f50b5d54e5a24f3b502cdd01c21bb047337cf599508738e8d78fda740bd20326393eb120fab4872301ef756927f961c0b2ab223e8c3a03244a6aae64b7a64bcd

data/lib/jruby-kafka/consumer.rb

@@ -1,3 +1,4 @@
+require 'java'
 require 'jruby-kafka/namespace'
 
 # noinspection JRubyStringImportInspection
@@ -10,35 +11,31 @@ class Kafka::Consumer
   include Java::JavaLang::Runnable
   java_signature 'void run()'
 
-  @m_stream
-  @m_thread_number
-  @m_queue
+  def initialize(a_stream, a_thread_number, a_queue, restart_on_exception, a_sleep_ms)
+    @m_thread_number = a_thread_number
+    @m_stream = a_stream
+    @m_queue = a_queue
+    @m_restart_on_exception = restart_on_exception
+    @m_sleep_ms = 1.0 / 1000.0 * Float(a_sleep_ms)
+  end
 
-    def initialize(a_stream, a_thread_number, a_queue, restart_on_exception, a_sleep_ms)
-      @m_thread_number = a_thread_number
-      @m_stream = a_stream
-      @m_queue = a_queue
-      @m_restart_on_exception = restart_on_exception
-      @m_sleep_ms = 1.0 / 1000.0 * Float(a_sleep_ms)
-    end
-
-    def run
-      it = @m_stream.iterator
-      begin
-        while it.hasNext
-          begin
-            @m_queue << it.next.message
-          end
-        end
-      rescue Exception => e
-        puts("#{self.class.name} caught exception: #{e.class.name}")
-        puts(e.message) if e.message != ''
-        if @m_restart_on_exception == 'true'
-          sleep(@m_sleep_ms)
-          retry
-        else
-          raise e
+  def run
+    it = @m_stream.iterator
+    begin
+      while it.hasNext
+        begin
+          @m_queue << it.next
         end
       end
+    rescue Exception => e
+      puts("#{self.class.name} caught exception: #{e.class.name}")
+      puts(e.message) if e.message != ''
+      if @m_restart_on_exception
+        sleep(@m_sleep_ms)
+        retry
+      else
+        raise e
+      end
     end
+  end
 end

data/lib/jruby-kafka/group.rb

@@ -9,13 +9,6 @@ class Kafka::Group
   java_import 'java.util.concurrent.Executors'
   java_import 'org.I0Itec.zkclient.exception.ZkException'
 
-  @consumer
-  @executor
-  @topic
-  @auto_offset_reset
-  @zk_connect
-  @group_id
-
   # Create a Kafka client group
   #
   # options:
@@ -24,8 +17,11 @@ class Kafka::Group
   # :zk_connect_timeout => "6000" - (optional) The max time that the client waits while establishing a connection to zookeeper.
   # :group_id => "group" - REQUIRED: The group id to consume on.
   # :topic_id => "topic" - REQUIRED: The topic id to consume on.
-  # :reset_beginning => "from-beginning" - (optional) If the consumer does not already have an established offset
-  #   to consume from, start with the earliest message present in the log rather than the latest message.
+  # :reset_beginning => "from-beginning" - (optional) reset the consumer group to start at the 
+  #   earliest message present in the log by clearing any offsets for the group stored in Zookeeper.
+  # :auto_offset_reset => "smallest" or "largest" - (optional, default 'largest') If the consumer does not already 
+  #   have an established offset to consume from, start with the earliest message present in the log (smallest) or 
+  #   after the last message in the log (largest).
   # :consumer_restart_on_error => "true" - (optional) Controls if consumer threads are to restart on caught exceptions.
   #   exceptions are logged.
   def initialize(options={})
@@ -34,9 +30,12 @@ class Kafka::Group
     @zk_connect = options[:zk_connect]
     @group_id = options[:group_id]
     @topic = options[:topic_id]
+    @topics_allowed = options[:allow_topics]
+    @topics_filtered = options[:filter_topics]
     @zk_session_timeout = '6000'
     @zk_connect_timeout = '6000'
     @zk_sync_time = '2000'
+    @reset_beginning = nil
     @auto_offset_reset = 'largest'
     @auto_commit_interval = '1000'
     @running = false
@@ -120,13 +119,15 @@ class Kafka::Group
       @consumer_restart_sleep_ms = "#{options[:consumer_restart_sleep_ms]}"
     end
 
+    if options[:auto_offset_reset]
+      @auto_offset_reset = "#{options[:auto_offset_reset]}"
+    end
 
     if options[:reset_beginning]
-      if options[:reset_beginning] == 'from-beginning'
-        @auto_offset_reset = 'smallest'
-      else
-        @auto_offset_reset = 'largest'
+      if not options[:auto_offset_reset] || options[:auto_offset_reset] != 'smallest'
+        raise KafkaError.new('reset_beginning => from-beginning must be used with auto_offset_reset => smallest')
       end
+      @reset_beginning = "#{options[:reset_beginning]}"
     end
 
     if options[:consumer_id]
@@ -148,7 +149,7 @@ class Kafka::Group
 
   def run(a_num_threads, a_queue)
     begin
-      if @auto_offset_reset == 'smallest'
+      if @reset_beginning == 'from-beginning'
         Java::kafka::utils::ZkUtils.maybeDeletePath(@zk_connect, "/consumers/#{@group_id}")
       end
 
@@ -156,11 +157,9 @@ class Kafka::Group
     rescue ZkException => e
       raise KafkaError.new(e), "Got ZkException: #{e}"
     end
-    topic_count_map = java.util.HashMap.new
+
     thread_value = a_num_threads.to_java Java::int
-    topic_count_map.put(@topic, thread_value)
-    consumer_map = @consumer.createMessageStreams(topic_count_map)
-    streams = Array.new(consumer_map[@topic])
+    streams = get_streams(thread_value)
 
     @executor = Executors.newFixedThreadPool(a_num_threads)
     @executor_submit = @executor.java_method(:submit, [Java::JavaLang::Runnable.java_class])
@@ -180,9 +179,30 @@ class Kafka::Group
   private
 
   def validate_required_arguments(options={})
-    [:zk_connect, :group_id, :topic_id].each do |opt|
+    [:zk_connect, :group_id].each do |opt|
       raise(ArgumentError, "#{opt} is required.") unless options[opt]
     end
+    unless [ options[:topic_id],
+             options[:allow_topics],
+             options[:filter_topics] ].compact.length == 1
+      raise(ArgumentError,
+            "exactly one of topic_id, allow_topics, filter_topics is required.")
+    end
+  end
+
+  def get_streams(threads)
+    if @topic
+      topic_count_map = java.util.HashMap.new
+      topic_count_map.put(@topic, threads)
+      consumer_map = @consumer.createMessageStreams(topic_count_map)
+      Array.new(consumer_map[@topic])
+    elsif @topics_allowed
+      filter = Java::kafka::consumer::Whitelist.new(@topics_allowed)
+      Array.new(@consumer.createMessageStreamsByFilter(filter, threads))
+    else # @topics_filtered
+      filter = Java::kafka::consumer::Blacklist.new(@topics_filtered)
+      Array.new(@consumer.createMessageStreamsByFilter(filter, threads))
+    end
   end
 
   def create_consumer_config

data/lib/jruby-kafka_jars.rb

@@ -1,15 +1,15 @@
 # this is a generated file, to avoid over-writing it just delete this comment
 require 'jar_dependencies'
 
-require_jar( 'log4j', 'log4j', '1.2.17' )
-require_jar( 'org.slf4j', 'slf4j-api', '1.7.5' )
-require_jar( 'org.slf4j', 'slf4j-log4j12', '1.7.5' )
+require_jar( 'org.scala-lang', 'scala-library', '2.9.2' )
+require_jar( 'junit', 'junit', '3.8.1' )
+require_jar( 'com.101tec', 'zkclient', '0.3' )
 require_jar( 'com.yammer.metrics', 'metrics-core', '2.2.0' )
+require_jar( 'org.slf4j', 'slf4j-api', '1.7.10' )
+require_jar( 'org.slf4j', 'slf4j-log4j12', '1.7.10' )
+require_jar( 'log4j', 'log4j', '1.2.17' )
+require_jar( 'jline', 'jline', '0.9.94' )
+require_jar( 'net.sf.jopt-simple', 'jopt-simple', '3.2' )
 require_jar( 'org.apache.zookeeper', 'zookeeper', '3.3.4' )
-require_jar( 'org.scala-lang', 'scala-library', '2.9.2' )
 require_jar( 'org.apache.kafka', 'kafka_2.9.2', '0.8.1.1' )
-require_jar( 'net.sf.jopt-simple', 'jopt-simple', '3.2' )
-require_jar( 'jline', 'jline', '0.9.94' )
-require_jar( 'com.101tec', 'zkclient', '0.3' )
 require_jar( 'org.xerial.snappy', 'snappy-java', '1.0.5' )
-require_jar( 'junit', 'junit', '3.8.1' )

metadata

@@ -1,57 +1,57 @@
 --- !ruby/object:Gem::Specification
 name: jruby-kafka
 version: !ruby/object:Gem::Version
-  version: 1.0.0.beta
+  version: 1.1.0.beta
 platform: java
 authors:
 - Joseph Lawson
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-01-19 00:00:00.000000000 Z
+date: 2015-02-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: jar-dependencies
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: '0.1'
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: '0.1'
+  name: jar-dependencies
   prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: ruby-maven
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '3.1'
+        version: '0.1'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: '3.1'
+  name: ruby-maven
   prerelease: false
   type: :development
-- !ruby/object:Gem::Dependency
-  name: rake
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '10.4'
+        version: '3.1'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: '10.4'
+  name: rake
   prerelease: false
   type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '10.4'
 description: this is primarily to be used as an interface for logstash
 email:
 - joe@joekiller.com
@@ -59,36 +59,38 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- lib/com/101tec/zkclient/0.3/zkclient-0.3.jar
-- lib/com/101tec/zkclient/maven-metadata-local.xml
-- lib/com/yammer/metrics/metrics-core/2.2.0/metrics-core-2.2.0.jar
-- lib/com/yammer/metrics/metrics-core/maven-metadata-local.xml
-- lib/jline/jline/0.9.94/jline-0.9.94.jar
-- lib/jline/jline/maven-metadata-local.xml
 - lib/jruby-kafka.rb
+- lib/jruby-kafka_jars.rb
 - lib/jruby-kafka/consumer.rb
 - lib/jruby-kafka/error.rb
 - lib/jruby-kafka/group.rb
 - lib/jruby-kafka/namespace.rb
 - lib/jruby-kafka/producer.rb
-- lib/jruby-kafka_jars.rb
+- lib/com/101tec/zkclient/0.3/zkclient-0.3.jar
+- lib/com/yammer/metrics/metrics-core/2.2.0/metrics-core-2.2.0.jar
+- lib/jline/jline/0.9.94/jline-0.9.94.jar
 - lib/junit/junit/3.8.1/junit-3.8.1.jar
-- lib/junit/junit/maven-metadata-local.xml
 - lib/log4j/log4j/1.2.17/log4j-1.2.17.jar
-- lib/log4j/log4j/maven-metadata-local.xml
 - lib/net/sf/jopt-simple/jopt-simple/3.2/jopt-simple-3.2.jar
-- lib/net/sf/jopt-simple/jopt-simple/maven-metadata-local.xml
 - lib/org/apache/kafka/kafka_2.9.2/0.8.1.1/kafka_2.9.2-0.8.1.1.jar
-- lib/org/apache/kafka/kafka_2.9.2/maven-metadata-local.xml
 - lib/org/apache/zookeeper/zookeeper/3.3.4/zookeeper-3.3.4.jar
-- lib/org/apache/zookeeper/zookeeper/maven-metadata-local.xml
 - lib/org/scala-lang/scala-library/2.9.2/scala-library-2.9.2.jar
-- lib/org/scala-lang/scala-library/maven-metadata-local.xml
+- lib/org/slf4j/slf4j-api/1.7.10/slf4j-api-1.7.10.jar
 - lib/org/slf4j/slf4j-api/1.7.5/slf4j-api-1.7.5.jar
-- lib/org/slf4j/slf4j-api/maven-metadata-local.xml
+- lib/org/slf4j/slf4j-log4j12/1.7.10/slf4j-log4j12-1.7.10.jar
 - lib/org/slf4j/slf4j-log4j12/1.7.5/slf4j-log4j12-1.7.5.jar
-- lib/org/slf4j/slf4j-log4j12/maven-metadata-local.xml
 - lib/org/xerial/snappy/snappy-java/1.0.5/snappy-java-1.0.5.jar
+- lib/com/101tec/zkclient/maven-metadata-local.xml
+- lib/com/yammer/metrics/metrics-core/maven-metadata-local.xml
+- lib/jline/jline/maven-metadata-local.xml
+- lib/junit/junit/maven-metadata-local.xml
+- lib/log4j/log4j/maven-metadata-local.xml
+- lib/net/sf/jopt-simple/jopt-simple/maven-metadata-local.xml
+- lib/org/apache/kafka/kafka_2.9.2/maven-metadata-local.xml
+- lib/org/apache/zookeeper/zookeeper/maven-metadata-local.xml
+- lib/org/scala-lang/scala-library/maven-metadata-local.xml
+- lib/org/slf4j/slf4j-api/maven-metadata-local.xml
+- lib/org/slf4j/slf4j-log4j12/maven-metadata-local.xml
 - lib/org/xerial/snappy/snappy-java/maven-metadata-local.xml
 homepage: https://github.com/joekiller/jruby-kafka
 licenses:
@@ -110,9 +112,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: 1.3.1
 requirements:
 - jar 'org.apache.kafka:kafka_2.9.2', '0.8.1.1'
-- jar 'org.slf4j:slf4j-log4j12', '1.7.5'
+- jar 'org.slf4j:slf4j-log4j12', '1.7.10'
 rubyforge_project:
-rubygems_version: 2.4.5
+rubygems_version: 2.1.9
 signing_key:
 specification_version: 4
 summary: jruby Kafka wrapper