jruby-kafka 1.7.2-java → 2.0-java

checksums.yaml

@@ -1,15 +1,7 @@
 ---
-!binary "U0hBMQ==":
-  metadata.gz: !binary |-
-    YTEyMzAyMjc4MTQ0YjIyMDI5MTNlN2EzNzA1Y2VkODA1OWEzNGMwNw==
-  data.tar.gz: !binary |-
-    MjE4OTEyYmU1MDQ4MmYzODg5NTcwMzM4YzMyN2U3M2QxYTQ5OWZiNw==
+SHA1:
+  metadata.gz: ac687c16b318e83142d9b294a3767a91d1b948c8
+  data.tar.gz: 0cf05c7b5dfde8ba46d7bf367fd1a53855614a21
 SHA512:
-  metadata.gz: !binary |-
-    MTNiYzQ0N2Q3NDBhM2EyZmEyZDI3MjMwMTAwNzVmYWEwZmQ4OTlmMGY3Zjky
-    Nzc2ZjYyMTNlMWY1ZjBhZTY3NjRlMGRhMGUwMTQ3MDExMjczZWFiY2I4MWE1
-    MjhjYTVjOWFjZWIzYTNhZGE4ODNjNDg1OGY4NjAwMzcwM2U2YmQ=
-  data.tar.gz: !binary |-
-    OWM2MGM5NGQ0ZDljNzRlMjExMWI2ODgwYzIyM2FhYmNlMDMwNDVkNWExOTQ4
-    MGRhODQ2M2ViMDllNzJlZTljNjcyMjgzYzliZWEyNzRmZTZmZTlkZTA4Mjk2
-    MDQwNzY5ZDA0Njc0MGMyODUzZDc2ZmNjNTY3ODgwODVjMGY5NjU=
+  metadata.gz: d9035f2bfcd3a193eb26fd1a83de291b4dda0934e7e50d15cb50f96d3da43147f6acb8e9b2c81483dd8e73b643d2178b9dcdaeed90a0b58bf0ffbf562dd7a1ea
+  data.tar.gz: 315a7c39f1b98d4c3c307bc69c892148136d1d320f5bb50a3ac6b3fe2ebb69a288d53649679b1858e256036af19518684d1153c508d7974db93dca7a56961760

data/lib/com/101tec/zkclient/maven-metadata-local.xml

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<metadata>
+  <groupId>com.101tec</groupId>
+  <artifactId>zkclient</artifactId>
+  <versioning>
+    <release>0.3</release>
+    <versions>
+      <version>0.3</version>
+    </versions>
+    <lastUpdated>20151231160701</lastUpdated>
+  </versioning>
+</metadata>

data/lib/com/yammer/metrics/metrics-core/maven-metadata-local.xml

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<metadata>
+  <groupId>com.yammer.metrics</groupId>
+  <artifactId>metrics-core</artifactId>
+  <versioning>
+    <release>2.2.0</release>
+    <versions>
+      <version>2.2.0</version>
+    </versions>
+    <lastUpdated>20151231160701</lastUpdated>
+  </versioning>
+</metadata>

data/lib/io/netty/netty/maven-metadata-local.xml

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<metadata>
+  <groupId>io.netty</groupId>
+  <artifactId>netty</artifactId>
+  <versioning>
+    <release>3.7.0.Final</release>
+    <versions>
+      <version>3.7.0.Final</version>
+    </versions>
+    <lastUpdated>20151231160701</lastUpdated>
+  </versioning>
+</metadata>

data/lib/jline/jline/maven-metadata-local.xml

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<metadata>
+  <groupId>jline</groupId>
+  <artifactId>jline</artifactId>
+  <versioning>
+    <release>0.9.94</release>
+    <versions>
+      <version>0.9.94</version>
+    </versions>
+    <lastUpdated>20151231160701</lastUpdated>
+  </versioning>
+</metadata>

data/lib/jruby-kafka/consumer.rb

@@ -1,41 +1,115 @@
 require 'java'
-require 'jruby-kafka/version'
+require 'jruby-kafka/namespace'
 
-# noinspection JRubyStringImportInspection
 class Kafka::Consumer
-  java_import 'kafka.consumer.ConsumerIterator'
-  java_import 'kafka.consumer.KafkaStream'
-  java_import 'kafka.common.ConsumerRebalanceFailedException'
-  java_import 'kafka.consumer.ConsumerTimeoutException'
-
-  include Java::JavaLang::Runnable
-  java_signature 'void run()'
-
-  def initialize(a_stream, a_thread_number, a_queue, restart_on_exception, a_sleep_ms)
-    @m_thread_number = a_thread_number
-    @m_stream = a_stream
-    @m_queue = a_queue
-    @m_restart_on_exception = restart_on_exception
-    @m_sleep_ms = 1.0 / 1000.0 * Float(a_sleep_ms)
+  # Create a Kafka high-level consumer.
+  #
+  # @param [Hash] config the consumer configuration.
+  #
+  # @option config [String]  :zookeeper_connect The connection string for the zookeeper connection in
+  #   the form "host:port[path][,host:port[path]*". Required.
+  # @option config [String]  :group_id The consumer group this instance is a member of. Required.
+  # @option config [String]  :topic The topic to consume from.
+  # @option config [String]  :include_topics The inclusive (white) regular expression filter matching
+  #   topics to consume from.
+  # @option config [String]  :exclude_topics The exclusive (black) regular expression filter matching
+  #   topics not to consume from.
+  # @option config [Integer] :num_streams (1) The number of message streams to create.
+  # @option config [String]  :key_decoder ('kafka.serializer.DefaultDecoder') Java class name for
+  #   message key decoder.
+  # @option config [String]  :msg_decoder ('kafka.serializer.DefaultDecoder') Java class name for
+  #   message value decoder.
+  #
+  # One and only one of :topic, :include_topics, or :exclude_topics must be provided.
+  #
+  # For other configuration properties and their default values see 
+  # https://kafka.apache.org/08/configuration.html#consumerconfigs and
+  # https://github.com/apache/kafka/blob/0.8.2.2/core/src/main/scala/kafka/consumer/ConsumerConfig.scala#L90-L182.
+  #
+  def initialize(config={})
+    validate_arguments config
+
+    @properties     =  config.clone
+    @topic          =  @properties.delete :topic
+    @include_topics =  @properties.delete :include_topics
+    @exclude_topics =  @properties.delete :exclude_topics
+    @num_streams    = (@properties.delete(:num_streams) || 1).to_java Java::int
+    @key_decoder    =  @properties.delete(:key_decoder) || 'kafka.serializer.DefaultDecoder'
+    @msg_decoder    =  @properties.delete(:msg_decoder) || 'kafka.serializer.DefaultDecoder'
+
+    @consumer = Java::KafkaConsumer::Consumer.createJavaConsumerConnector create_config
   end
 
-  def run
-    it = @m_stream.iterator
-    begin
-      while it.hasNext
-        begin
-          @m_queue << it.next
-        end
-      end
-    rescue Exception => e
-      puts("#{self.class.name} caught exception: #{e.class.name}")
-      puts(e.message) if e.message != ''
-      if @m_restart_on_exception
-        sleep(@m_sleep_ms)
-        retry
-      else
-        raise e
-      end
+  # Start fetching messages.
+  #
+  # @return [Array<Java::KafkaConsumer::KafkaStream>] list of stream, as specified by the 
+  #   :num_stream configuration parameter. A stream is essentially a queue of incomnig messages
+  #   from Kafka topic partitions.
+  #
+  # @see http://apache.osuosl.org/kafka/0.8.2.2/scaladoc/index.html#kafka.consumer.KafkaStream
+  # 
+  # @note KafkaStreams instances are not thread-safe.
+  def message_streams
+    key_decoder_i = Java::JavaClass.for_name(@key_decoder).
+      constructor('kafka.utils.VerifiableProperties').new_instance nil
+    msg_decoder_i = Java::JavaClass.for_name(@msg_decoder).
+      constructor('kafka.utils.VerifiableProperties').new_instance nil
+
+    if @topic
+      topic_count_map = java.util.HashMap.new @topic => @num_streams
+      @consumer.
+        createMessageStreams(topic_count_map, key_decoder_i, msg_decoder_i)[@topic].
+        to_a
+
+    else
+      filter =  @include_topics ? 
+        Java::KafkaConsumer::Whitelist.new(@include_topics) : 
+        Java::KafkaConsumer::Blacklist.new(@exclude_topics)
+
+      @consumer.
+        createMessageStreamsByFilter(filter, @num_streams, key_decoder_i, msg_decoder_i).
+        to_a
+
     end
   end
+
+  # Commit the offsets of all topic partitions connected by this consumer.
+  #
+  # Useful for when the :auto_commit_enable configuration parameter is false.
+  #
+  # @return void
+  def commitOffsets
+    @consumer.commitOffsets
+  end
+
+  # Shutdown the consumer.
+  #
+  # @return void
+  def shutdown
+    @consumer.shutdown if @consumer
+    nil
+  end
+  
+  private
+
+  def validate_arguments(options)
+    [:zookeeper_connect, :group_id].each do |opt|
+      raise ArgumentError, "Parameter :#{opt} is required." unless options[opt]
+    end
+
+    unless [ options[:topic], options[:include_topics], options[:exclude_topics] ].one?
+      raise ArgumentError, "Exactly one of :topic, :include_topics, :exclude_topics is required."
+    end
+  end
+
+  def create_config
+    properties = java.util.Properties.new
+    @properties.each do |k,v|
+      k = k.to_s.gsub '_', '.'
+      v = v.to_s 
+      properties.setProperty k, v
+    end
+    Java::KafkaConsumer::ConsumerConfig.new properties
+  end
 end
+

data/lib/jruby-kafka/error.rb

@@ -1,4 +1,4 @@
-require 'jruby-kafka/version'
+require 'jruby-kafka/namespace'
 
 class KafkaError < StandardError
   attr_reader :object

data/lib/jruby-kafka/kafka-producer.rb

@@ -1,9 +1,10 @@
-require 'jruby-kafka/version'
+require 'jruby-kafka/namespace'
 require 'jruby-kafka/error'
 
 # noinspection JRubyStringImportInspection
 class Kafka::KafkaProducer
   java_import 'org.apache.kafka.clients.producer.ProducerRecord'
+  java_import 'org.apache.kafka.clients.producer.Callback'
   KAFKA_PRODUCER = Java::org.apache.kafka.clients.producer.KafkaProducer
 
   VALIDATIONS = {
@@ -27,7 +28,19 @@ class Kafka::KafkaProducer
     send.buffer.bytes         timeout.ms                value.serializer
   ]
 
-  attr_reader :producer, :send_method, :options
+  class RubyCallback
+    include Callback
+
+    def initialize(cb)
+      @cb = cb
+    end
+    
+    def onCompletion(metadata, exception)
+      @cb.call(metadata, exception)
+    end
+  end
+
+  attr_reader :producer, :send_method, :send_cb_method, :options
 
   def initialize(opts = {})
     @options = opts.reduce({}) do |opts_array, (k, v)|
@@ -37,17 +50,22 @@ class Kafka::KafkaProducer
       opts_array
     end
     validate_arguments
-    @send_method = proc { throw StandardError.new 'Producer is not connected' }
+    @send_method = @send_cb_method = proc { throw StandardError.new 'Producer is not connected' }
   end
 
   def connect
     @producer = KAFKA_PRODUCER.new(create_producer_config)
     @send_method = producer.java_method :send, [ProducerRecord]
+    @send_cb_method = producer.java_method :send, [ProducerRecord, Callback]
   end
 
   # throws FailedToSendMessageException or if not connected, StandardError.
-  def send_msg(topic, partition, key, value)
-    send_method.call(ProducerRecord.new(topic, partition, key, value))
+  def send_msg(topic, partition, key, value, &block)
+    if block
+      send_cb_method.call(ProducerRecord.new(topic, partition, key, value), RubyCallback.new(block))
+    else
+      send_method.call(ProducerRecord.new(topic, partition, key, value))
+    end
   end
 
   def close

data/lib/jruby-kafka/namespace.rb

@@ -0,0 +1,3 @@
+module Kafka
+
+end

data/lib/jruby-kafka/producer.rb

@@ -1,5 +1,5 @@
 # basically we are porting this https://cwiki.apache.org/confluence/display/KAFKA/0.8.0+Producer+Example
-require 'jruby-kafka/version'
+require 'jruby-kafka/namespace'
 require 'jruby-kafka/error'
 
 # noinspection JRubyStringImportInspection
@@ -11,11 +11,10 @@ class Kafka::Producer
   java_import 'kafka.message.NoCompressionCodec'
   java_import 'kafka.message.GZIPCompressionCodec'
   java_import 'kafka.message.SnappyCompressionCodec'
-  java_import 'kafka.message.LZ4CompressionCodec'
 
   VALIDATIONS = {
     :'request.required.acks' => %w[ 0 1 -1 ],
-    :'required.codecs' => [NoCompressionCodec.name, GZIPCompressionCodec.name, SnappyCompressionCodec.name, LZ4CompressionCodec.name],
+    :'required.codecs' => [NoCompressionCodec.name, GZIPCompressionCodec.name, SnappyCompressionCodec.name],
     :'producer.type' => %w[ sync async ]
   }
 

data/lib/jruby-kafka.rb

@@ -1,6 +1,7 @@
-require 'jruby-kafka/version'
 require 'jruby-kafka_jars.rb'
 require 'jruby-kafka/consumer'
-require 'jruby-kafka/group'
 require 'jruby-kafka/producer'
 require 'jruby-kafka/kafka-producer'
+
+module Kafka
+end

data/lib/jruby-kafka_jars.rb

@@ -1,38 +1,18 @@
 # this is a generated file, to avoid over-writing it just delete this comment
-begin
-  require 'jar_dependencies'
-rescue LoadError
-  require 'log4j/log4j/1.2.17/log4j-1.2.17.jar'
-  require 'org/apache/kafka/kafka_2.10/0.8.2.2/kafka_2.10-0.8.2.2.jar'
-  require 'org/apache/zookeeper/zookeeper/3.4.6/zookeeper-3.4.6.jar'
-  require 'org/slf4j/slf4j-log4j12/1.7.13/slf4j-log4j12-1.7.13.jar'
-  require 'org/slf4j/slf4j-api/1.7.13/slf4j-api-1.7.13.jar'
-  require 'net/sf/jopt-simple/jopt-simple/3.2/jopt-simple-3.2.jar'
-  require 'io/netty/netty/3.7.0.Final/netty-3.7.0.Final.jar'
-  require 'net/jpountz/lz4/lz4/1.2.0/lz4-1.2.0.jar'
-  require 'junit/junit/3.8.1/junit-3.8.1.jar'
-  require 'org/apache/kafka/kafka-clients/0.8.2.2/kafka-clients-0.8.2.2.jar'
-  require 'com/yammer/metrics/metrics-core/2.2.0/metrics-core-2.2.0.jar'
-  require 'jline/jline/0.9.94/jline-0.9.94.jar'
-  require 'org/xerial/snappy/snappy-java/1.1.1.7/snappy-java-1.1.1.7.jar'
-  require 'org/scala-lang/scala-library/2.10.4/scala-library-2.10.4.jar'
-  require 'com/101tec/zkclient/0.3/zkclient-0.3.jar'
-end
+require 'jar_dependencies'
 
-if defined? Jars
-  require_jar( 'log4j', 'log4j', '1.2.17' )
-  require_jar( 'org.apache.kafka', 'kafka_2.10', '0.8.2.2' )
-  require_jar( 'org.apache.zookeeper', 'zookeeper', '3.4.6' )
-  require_jar( 'org.slf4j', 'slf4j-log4j12', '1.7.13' )
-  require_jar( 'org.slf4j', 'slf4j-api', '1.7.13' )
-  require_jar( 'net.sf.jopt-simple', 'jopt-simple', '3.2' )
-  require_jar( 'io.netty', 'netty', '3.7.0.Final' )
-  require_jar( 'net.jpountz.lz4', 'lz4', '1.2.0' )
-  require_jar( 'junit', 'junit', '3.8.1' )
-  require_jar( 'org.apache.kafka', 'kafka-clients', '0.8.2.2' )
-  require_jar( 'com.yammer.metrics', 'metrics-core', '2.2.0' )
-  require_jar( 'jline', 'jline', '0.9.94' )
-  require_jar( 'org.xerial.snappy', 'snappy-java', '1.1.1.7' )
-  require_jar( 'org.scala-lang', 'scala-library', '2.10.4' )
-  require_jar( 'com.101tec', 'zkclient', '0.3' )
-end
+require_jar( 'io.netty', 'netty', '3.7.0.Final' )
+require_jar( 'log4j', 'log4j', '1.2.17' )
+require_jar( 'org.xerial.snappy', 'snappy-java', '1.1.1.7' )
+require_jar( 'jline', 'jline', '0.9.94' )
+require_jar( 'net.sf.jopt-simple', 'jopt-simple', '3.2' )
+require_jar( 'org.slf4j', 'slf4j-api', '1.7.13' )
+require_jar( 'org.apache.kafka', 'kafka-clients', '0.8.2.2' )
+require_jar( 'junit', 'junit', '3.8.1' )
+require_jar( 'com.101tec', 'zkclient', '0.3' )
+require_jar( 'com.yammer.metrics', 'metrics-core', '2.2.0' )
+require_jar( 'org.apache.zookeeper', 'zookeeper', '3.4.6' )
+require_jar( 'org.apache.kafka', 'kafka_2.10', '0.8.2.2' )
+require_jar( 'org.slf4j', 'slf4j-log4j12', '1.7.13' )
+require_jar( 'org.scala-lang', 'scala-library', '2.10.4' )
+require_jar( 'net.jpountz.lz4', 'lz4', '1.2.0' )

data/lib/junit/junit/maven-metadata-local.xml

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<metadata>
+  <groupId>junit</groupId>
+  <artifactId>junit</artifactId>
+  <versioning>
+    <release>3.8.1</release>
+    <versions>
+      <version>3.8.1</version>
+    </versions>
+    <lastUpdated>20151231160701</lastUpdated>
+  </versioning>
+</metadata>

data/lib/log4j/log4j/maven-metadata-local.xml

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<metadata>
+  <groupId>log4j</groupId>
+  <artifactId>log4j</artifactId>
+  <versioning>
+    <release>1.2.17</release>
+    <versions>
+      <version>1.2.17</version>
+    </versions>
+    <lastUpdated>20151231160701</lastUpdated>
+  </versioning>
+</metadata>

data/lib/net/jpountz/lz4/lz4/maven-metadata-local.xml

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<metadata>
+  <groupId>net.jpountz.lz4</groupId>
+  <artifactId>lz4</artifactId>
+  <versioning>
+    <release>1.2.0</release>
+    <versions>
+      <version>1.2.0</version>
+    </versions>
+    <lastUpdated>20151231160701</lastUpdated>
+  </versioning>
+</metadata>

data/lib/net/sf/jopt-simple/jopt-simple/maven-metadata-local.xml

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<metadata>
+  <groupId>net.sf.jopt-simple</groupId>
+  <artifactId>jopt-simple</artifactId>
+  <versioning>
+    <release>3.2</release>
+    <versions>
+      <version>3.2</version>
+    </versions>
+    <lastUpdated>20151231160701</lastUpdated>
+  </versioning>
+</metadata>

data/lib/org/apache/kafka/kafka-clients/maven-metadata-local.xml

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<metadata>
+  <groupId>org.apache.kafka</groupId>
+  <artifactId>kafka-clients</artifactId>
+  <versioning>
+    <release>0.8.2.2</release>
+    <versions>
+      <version>0.8.2.2</version>
+    </versions>
+    <lastUpdated>20151231160701</lastUpdated>
+  </versioning>
+</metadata>

data/lib/org/apache/kafka/kafka_2.10/maven-metadata-local.xml

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<metadata>
+  <groupId>org.apache.kafka</groupId>
+  <artifactId>kafka_2.10</artifactId>
+  <versioning>
+    <release>0.8.2.2</release>
+    <versions>
+      <version>0.8.2.2</version>
+    </versions>
+    <lastUpdated>20151231160701</lastUpdated>
+  </versioning>
+</metadata>

data/lib/org/apache/zookeeper/zookeeper/maven-metadata-local.xml

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<metadata>
+  <groupId>org.apache.zookeeper</groupId>
+  <artifactId>zookeeper</artifactId>
+  <versioning>
+    <release>3.4.6</release>
+    <versions>
+      <version>3.4.6</version>
+    </versions>
+    <lastUpdated>20151231160701</lastUpdated>
+  </versioning>
+</metadata>

data/lib/org/scala-lang/scala-library/maven-metadata-local.xml

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<metadata>
+  <groupId>org.scala-lang</groupId>
+  <artifactId>scala-library</artifactId>
+  <versioning>
+    <release>2.10.4</release>
+    <versions>
+      <version>2.10.4</version>
+    </versions>
+    <lastUpdated>20151231160701</lastUpdated>
+  </versioning>
+</metadata>

data/lib/org/slf4j/slf4j-api/maven-metadata-local.xml

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<metadata>
+  <groupId>org.slf4j</groupId>
+  <artifactId>slf4j-api</artifactId>
+  <versioning>
+    <release>1.7.13</release>
+    <versions>
+      <version>1.7.13</version>
+    </versions>
+    <lastUpdated>20151231160701</lastUpdated>
+  </versioning>
+</metadata>

data/lib/org/slf4j/slf4j-log4j12/maven-metadata-local.xml

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<metadata>
+  <groupId>org.slf4j</groupId>
+  <artifactId>slf4j-log4j12</artifactId>
+  <versioning>
+    <release>1.7.13</release>
+    <versions>
+      <version>1.7.13</version>
+    </versions>
+    <lastUpdated>20151231160701</lastUpdated>
+  </versioning>
+</metadata>

data/lib/org/xerial/snappy/snappy-java/maven-metadata-local.xml

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<metadata>
+  <groupId>org.xerial.snappy</groupId>
+  <artifactId>snappy-java</artifactId>
+  <versioning>
+    <release>1.1.1.7</release>
+    <versions>
+      <version>1.1.1.7</version>
+    </versions>
+    <lastUpdated>20151231160701</lastUpdated>
+  </versioning>
+</metadata>

data/lib/rubygems/jar-dependencies/maven-metadata-local.xml

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<metadata>
+  <groupId>rubygems</groupId>
+  <artifactId>jar-dependencies</artifactId>
+  <versioning>
+    <release>0.3.1</release>
+    <versions>
+      <version>0.3.1</version>
+    </versions>
+    <lastUpdated>20151231160701</lastUpdated>
+  </versioning>
+</metadata>

data/lib/rubygems/rake/maven-metadata-local.xml

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<metadata>
+  <groupId>rubygems</groupId>
+  <artifactId>rake</artifactId>
+  <versioning>
+    <release>10.4.2</release>
+    <versions>
+      <version>10.4.2</version>
+    </versions>
+    <lastUpdated>20151231160701</lastUpdated>
+  </versioning>
+</metadata>

data/lib/rubygems/ruby-maven/maven-metadata-local.xml

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<metadata>
+  <groupId>rubygems</groupId>
+  <artifactId>ruby-maven</artifactId>
+  <versioning>
+    <release>3.3.8</release>
+    <versions>
+      <version>3.3.8</version>
+    </versions>
+    <lastUpdated>20151231160701</lastUpdated>
+  </versioning>
+</metadata>

data/lib/rubygems/ruby-maven-libs/maven-metadata-local.xml

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<metadata>
+  <groupId>rubygems</groupId>
+  <artifactId>ruby-maven-libs</artifactId>
+  <versioning>
+    <release>3.3.3</release>
+    <versions>
+      <version>3.3.3</version>
+    </versions>
+    <lastUpdated>20151231160701</lastUpdated>
+  </versioning>
+</metadata>

metadata

@@ -1,72 +1,58 @@
 --- !ruby/object:Gem::Specification
 name: jruby-kafka
 version: !ruby/object:Gem::Version
-  version: 1.7.2
+  version: '2.0'
 platform: java
 authors:
 - Joseph Lawson
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-08-12 00:00:00.000000000 Z
+date: 2015-12-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: jar-dependencies
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.3.2
-  type: :development
+        version: '0'
+  name: jar-dependencies
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.3.2
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.5'
-  type: :development
+        version: '3.3'
+  name: ruby-maven
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.5'
+        version: '3.3'
 - !ruby/object:Gem::Dependency
-  name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.4'
-  type: :development
+        version: '10.4'
+  name: rake
   prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: '3.4'
-- !ruby/object:Gem::Dependency
-  name: ruby-maven
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: '3.3'
   type: :development
-  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.3'
-description: JRuby wrapper for Kafka
+        version: '10.4'
+description: this is primarily to be used as an interface for logstash
 email:
 - joe@joekiller.com
 executables: []
@@ -74,52 +60,70 @@ extensions: []
 extra_rdoc_files: []
 files:
 - lib/com/101tec/zkclient/0.3/zkclient-0.3.jar
+- lib/com/101tec/zkclient/maven-metadata-local.xml
 - lib/com/yammer/metrics/metrics-core/2.2.0/metrics-core-2.2.0.jar
+- lib/com/yammer/metrics/metrics-core/maven-metadata-local.xml
 - lib/io/netty/netty/3.7.0.Final/netty-3.7.0.Final.jar
+- lib/io/netty/netty/maven-metadata-local.xml
 - lib/jline/jline/0.9.94/jline-0.9.94.jar
+- lib/jline/jline/maven-metadata-local.xml
 - lib/jruby-kafka.rb
 - lib/jruby-kafka/consumer.rb
 - lib/jruby-kafka/error.rb
-- lib/jruby-kafka/group.rb
 - lib/jruby-kafka/kafka-producer.rb
+- lib/jruby-kafka/namespace.rb
 - lib/jruby-kafka/producer.rb
-- lib/jruby-kafka/version.rb
 - lib/jruby-kafka_jars.rb
 - lib/junit/junit/3.8.1/junit-3.8.1.jar
+- lib/junit/junit/maven-metadata-local.xml
 - lib/log4j/log4j/1.2.17/log4j-1.2.17.jar
+- lib/log4j/log4j/maven-metadata-local.xml
 - lib/net/jpountz/lz4/lz4/1.2.0/lz4-1.2.0.jar
+- lib/net/jpountz/lz4/lz4/maven-metadata-local.xml
 - lib/net/sf/jopt-simple/jopt-simple/3.2/jopt-simple-3.2.jar
+- lib/net/sf/jopt-simple/jopt-simple/maven-metadata-local.xml
 - lib/org/apache/kafka/kafka-clients/0.8.2.2/kafka-clients-0.8.2.2.jar
+- lib/org/apache/kafka/kafka-clients/maven-metadata-local.xml
 - lib/org/apache/kafka/kafka_2.10/0.8.2.2/kafka_2.10-0.8.2.2.jar
+- lib/org/apache/kafka/kafka_2.10/maven-metadata-local.xml
 - lib/org/apache/zookeeper/zookeeper/3.4.6/zookeeper-3.4.6.jar
+- lib/org/apache/zookeeper/zookeeper/maven-metadata-local.xml
 - lib/org/scala-lang/scala-library/2.10.4/scala-library-2.10.4.jar
+- lib/org/scala-lang/scala-library/maven-metadata-local.xml
 - lib/org/slf4j/slf4j-api/1.7.13/slf4j-api-1.7.13.jar
+- lib/org/slf4j/slf4j-api/maven-metadata-local.xml
 - lib/org/slf4j/slf4j-log4j12/1.7.13/slf4j-log4j12-1.7.13.jar
+- lib/org/slf4j/slf4j-log4j12/maven-metadata-local.xml
 - lib/org/xerial/snappy/snappy-java/1.1.1.7/snappy-java-1.1.1.7.jar
+- lib/org/xerial/snappy/snappy-java/maven-metadata-local.xml
+- lib/rubygems/jar-dependencies/maven-metadata-local.xml
+- lib/rubygems/rake/maven-metadata-local.xml
+- lib/rubygems/ruby-maven-libs/maven-metadata-local.xml
+- lib/rubygems/ruby-maven/maven-metadata-local.xml
 homepage: https://github.com/joekiller/jruby-kafka
 licenses:
 - Apache 2.0
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements:
 - jar 'org.apache.kafka:kafka_2.10', '0.8.2.2'
 - jar 'org.slf4j:slf4j-log4j12', '1.7.13'
-rubyforge_project: 
-rubygems_version: 2.4.5
-signing_key: 
+rubyforge_project:
+rubygems_version: 2.4.8
+signing_key:
 specification_version: 4
 summary: jruby Kafka wrapper
 test_files: []

data/lib/jruby-kafka/group.rb

@@ -1,257 +0,0 @@
-# basically we are porting this https://cwiki.apache.org/confluence/display/KAFKA/Consumer+Group+Example
-require 'jruby-kafka/version'
-require 'jruby-kafka/consumer'
-require 'jruby-kafka/error'
-
-# noinspection JRubyStringImportInspection
-class Kafka::Group
-  java_import 'java.util.concurrent.ExecutorService'
-  java_import 'java.util.concurrent.Executors'
-  java_import 'org.I0Itec.zkclient.exception.ZkException'
-
-  # Create a Kafka client group
-  #
-  # options:
-  # :zk_connect => "localhost:2181" - REQUIRED: The connection string for the
-  #   zookeeper connection in the form host:port. Multiple URLS can be given to allow fail-over.
-  # :zk_connect_timeout => "6000" - (optional) The max time that the client waits while establishing a connection to zookeeper.
-  # :group_id => "group" - REQUIRED: The group id to consume on.
-  # :topic_id => "topic" - REQUIRED: The topic id to consume on.
-  # :reset_beginning => "from-beginning" - (optional) reset the consumer group to start at the
-  #   earliest message present in the log by clearing any offsets for the group stored in Zookeeper.
-  # :auto_offset_reset => "smallest" or "largest" - (optional, default 'largest') If the consumer does not already
-  #   have an established offset to consume from, start with the earliest message present in the log (smallest) or
-  #   after the last message in the log (largest).
-  # :consumer_restart_on_error => "true" - (optional) Controls if consumer threads are to restart on caught exceptions.
-  #   exceptions are logged.
-  def initialize(options={})
-    validate_required_arguments(options)
-
-    @zk_connect = options[:zk_connect]
-    @group_id = options[:group_id]
-    @topic = options[:topic_id]
-    @topics_allowed = options[:allow_topics]
-    @topics_filtered = options[:filter_topics]
-    @zk_session_timeout = '6000'
-    @zk_connect_timeout = '6000'
-    @zk_sync_time = '2000'
-    @reset_beginning = nil
-    @auto_offset_reset = 'largest'
-    @auto_commit_interval = '1000'
-    @running = false
-    @rebalance_max_retries = '4'
-    @rebalance_backoff_ms = '2000'
-    @socket_timeout_ms = "#{30 * 1000}"
-    @socket_receive_buffer_bytes = "#{64 * 1024}"
-    @fetch_message_max_bytes = "#{1024 * 1024}"
-    @auto_commit_enable = "#{true}"
-    @queued_max_message_chunks = '10'
-    @fetch_min_bytes = '1'
-    @fetch_wait_max_ms = '100'
-    @refresh_leader_backoff_ms = '200'
-    @consumer_timeout_ms = '-1'
-    @consumer_restart_on_error = "#{false}"
-    @consumer_restart_sleep_ms = '0'
-    @consumer_id = nil
-    @key_decoder_class = "kafka.serializer.DefaultDecoder"
-    @value_decoder_class = "kafka.serializer.DefaultDecoder"
-    @dual_commit_enabled = "#{true}"
-    @offsets_storage = "zookeeper"
-
-    if options[:zk_connect_timeout]
-      @zk_connect_timeout = "#{options[:zk_connect_timeout]}"
-    end
-    if options[:zk_session_timeout]
-      @zk_session_timeout = "#{options[:zk_session_timeout]}"
-    end
-    if options[:zk_sync_time]
-      @zk_sync_time = "#{options[:zk_sync_time]}"
-    end
-    if options[:auto_commit_interval]
-      @auto_commit_interval = "#{options[:auto_commit_interval]}"
-    end
-
-    if options[:rebalance_max_retries]
-      @rebalance_max_retries = "#{options[:rebalance_max_retries]}"
-    end
-
-    if options[:rebalance_backoff_ms]
-      @rebalance_backoff_ms = "#{options[:rebalance_backoff_ms]}"
-    end
-
-    if options[:socket_timeout_ms]
-      @socket_timeout_ms = "#{options[:socket_timeout_ms]}"
-    end
-
-    if options[:socket_receive_buffer_bytes]
-      @socket_receive_buffer_bytes = "#{options[:socket_receive_buffer_bytes]}"
-    end
-
-    if options[:fetch_message_max_bytes]
-      @fetch_message_max_bytes = "#{options[:fetch_message_max_bytes]}"
-    end
-
-    if options[:auto_commit_enable]
-      @auto_commit_enable = "#{options[:auto_commit_enable]}"
-    end
-
-    if options[:queued_max_message_chunks]
-      @queued_max_message_chunks = "#{options[:queued_max_message_chunks]}"
-    end
-
-    if options[:fetch_min_bytes]
-      @fetch_min_bytes = "#{options[:fetch_min_bytes]}"
-    end
-
-    if options[:fetch_wait_max_ms]
-      @fetch_wait_max_ms = "#{options[:fetch_wait_max_ms]}"
-    end
-
-    if options[:refresh_leader_backoff_ms]
-      @refresh_leader_backoff_ms = "#{options[:refresh_leader_backoff_ms]}"
-    end
-
-    if options[:consumer_timeout_ms]
-      @consumer_timeout_ms = "#{options[:consumer_timeout_ms]}"
-    end
-
-    if options[:consumer_restart_on_error]
-      @consumer_restart_on_error = "#{options[:consumer_restart_on_error]}"
-    end
-
-    if options[:consumer_restart_sleep_ms]
-      @consumer_restart_sleep_ms = "#{options[:consumer_restart_sleep_ms]}"
-    end
-
-    if options[:auto_offset_reset]
-      @auto_offset_reset = "#{options[:auto_offset_reset]}"
-    end
-
-    if options[:key_decoder_class]
-      @key_decoder_class = "#{options[:key_decoder_class]}"
-    end
-
-    if options[:value_decoder_class]
-      @value_decoder_class = "#{options[:value_decoder_class]}"
-    end
-
-    if options[:reset_beginning]
-      if not options[:auto_offset_reset] || options[:auto_offset_reset] != 'smallest'
-        raise KafkaError.new('reset_beginning => from-beginning must be used with auto_offset_reset => smallest')
-      end
-      @reset_beginning = "#{options[:reset_beginning]}"
-    end
-
-    if options[:consumer_id]
-      @consumer_id = options[:consumer_id]
-    end
-
-    if options[:dual_commit_enabled]
-      @dual_commit_enabled = "#{options[:dual_commit_enabled]}"
-    end
-    if options[:offsets_storage]
-      @offsets_storage = "#{options[:offsets_storage]}"
-    end
-  end
-
-  public
-
-  def shutdown
-    if @consumer
-      @consumer.shutdown
-    end
-    if @executor
-      @executor.shutdown
-    end
-    @running = false
-  end
-
-  def run(a_num_threads, a_queue)
-    begin
-      if @reset_beginning == 'from-beginning'
-        Java::kafka::utils::ZkUtils.maybeDeletePath(@zk_connect, "/consumers/#{@group_id}")
-      end
-
-      @consumer = Java::kafka::consumer::Consumer.createJavaConsumerConnector(create_consumer_config)
-    rescue ZkException => e
-      raise KafkaError.new(e), "Got ZkException: #{e}"
-    end
-
-    thread_value = a_num_threads.to_java Java::int
-    streams = get_streams(thread_value)
-
-    @executor = Executors.newFixedThreadPool(a_num_threads)
-    @executor_submit = @executor.java_method(:submit, [Java::JavaLang::Runnable.java_class])
-
-    thread_number = 0
-    streams.each do |stream|
-      @executor_submit.call(Kafka::Consumer.new(stream, thread_number, a_queue, @consumer_restart_on_error, @consumer_restart_sleep_ms))
-      thread_number += 1
-    end
-    @running = true
-  end
-
-  def running?
-    @running
-  end
-
-  private
-
-  def validate_required_arguments(options={})
-    [:zk_connect, :group_id].each do |opt|
-      raise(ArgumentError, "#{opt} is required.") unless options[opt]
-    end
-    unless [ options[:topic_id],
-             options[:allow_topics],
-             options[:filter_topics] ].compact.length == 1
-      raise(ArgumentError,
-            "exactly one of topic_id, allow_topics, filter_topics is required.")
-    end
-  end
-
-  def get_streams(threads)
-    constructor_param_class_name = "kafka.utils.VerifiableProperties"
-    key_decoder_instance = Java::JavaClass.for_name(@key_decoder_class).constructor(constructor_param_class_name).new_instance(nil)
-    value_decoder_instance = Java::JavaClass.for_name(@value_decoder_class).constructor(constructor_param_class_name).new_instance(nil)
-    if @topic
-      topic_count_map = java.util.HashMap.new
-      topic_count_map.put(@topic, threads)
-      consumer_map = @consumer.createMessageStreams(topic_count_map, key_decoder_instance, value_decoder_instance)
-      Array.new(consumer_map[@topic])
-    elsif @topics_allowed
-      filter = Java::kafka::consumer::Whitelist.new(@topics_allowed)
-      Array.new(@consumer.createMessageStreamsByFilter(filter, threads, key_decoder_instance, value_decoder_instance))
-    else # @topics_filtered
-      filter = Java::kafka::consumer::Blacklist.new(@topics_filtered)
-      Array.new(@consumer.createMessageStreamsByFilter(filter, threads, key_decoder_instance, value_decoder_instance))
-    end
-  end
-
-  def create_consumer_config
-    properties = java.util.Properties.new
-    properties.put('zookeeper.connect', @zk_connect)
-    properties.put('group.id', @group_id)
-    properties.put('zookeeper.connection.timeout.ms', @zk_connect_timeout)
-    properties.put('zookeeper.session.timeout.ms', @zk_session_timeout)
-    properties.put('zookeeper.sync.time.ms', @zk_sync_time)
-    properties.put('auto.commit.interval.ms', @auto_commit_interval)
-    properties.put('auto.offset.reset', @auto_offset_reset)
-    properties.put('rebalance.max.retries', @rebalance_max_retries)
-    properties.put('rebalance.backoff.ms', @rebalance_backoff_ms)
-    properties.put('socket.timeout.ms', @socket_timeout_ms)
-    properties.put('socket.receive.buffer.bytes', @socket_receive_buffer_bytes)
-    properties.put('fetch.message.max.bytes', @fetch_message_max_bytes)
-    properties.put('auto.commit.enable', @auto_commit_enable)
-    properties.put('queued.max.message.chunks', @queued_max_message_chunks)
-    properties.put('fetch.min.bytes', @fetch_min_bytes)
-    properties.put('fetch.wait.max.ms', @fetch_wait_max_ms)
-    properties.put('refresh.leader.backoff.ms', @refresh_leader_backoff_ms)
-    properties.put('consumer.timeout.ms', @consumer_timeout_ms)
-    properties.put('dual.commit.enabled', @dual_commit_enabled)
-    properties.put('offsets.storage', @offsets_storage)
-    unless @consumer_id.nil?
-      properties.put('consumer.id', @consumer_id)
-    end
-    Java::kafka::consumer::ConsumerConfig.new(properties)
-  end
-end

data/lib/jruby-kafka/version.rb

@@ -1,4 +0,0 @@
-module Kafka
-  VERSION = '1.7.2'.freeze
-  JAR_DEPENDENCIES_VERSION = '0.3.2'.freeze
-end