jruby-kafka 2.1-java → 2.2.2-java

checksums.yaml

@@ -1,7 +1,15 @@
 ---
-SHA1:
-  metadata.gz: 583ea35b720a5b6300f4baf5e368cc499115051e
-  data.tar.gz: e163c0f3b84a6134a2ceb269cc98c93cad6845f7
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    M2NjNzliNzVlOGViODcyZTEyN2UwNTA2MTg5ZWI5OTM3NDZjNDYwNA==
+  data.tar.gz: !binary |-
+    ODY0Y2MyNTQyN2QzMjg4N2M3MTVmZDZkNTMzYjA2YTE0NDk0MjYwNQ==
 SHA512:
-  metadata.gz: 646ce387223e4cba598b8bdfa1a95873e25b438b5ea59837ef710e0520e0828e5b3affd84421e9eebac733cf14e7cc3baa05b8fcbebc5b66a562e902b505ec78
-  data.tar.gz: 23e7d76a2baa94066a2daea8ac33fa609932f62e05f5a8f889e04e21df52398d58ba84789b08c8a9b7aeb77cb9947a43553cf84972a955ef06e85be01caf63f2
+  metadata.gz: !binary |-
+    MDU2OGU4YzRkNjRjNzg4ZTllNGZiM2YzZmM3NTA5NmFhYmU1NTMxYzEyM2Yx
+    MjdlNjM2NWIwMzRkYjE5ZWNmNDJkMWQ2OWYxYmY3ZTIyOGYzMGFjNTUzNWFl
+    ODg0YWZkNjdjNjI0ZTFlN2Q2OTdkNzY0NzkwOTg5MmRjNDRkMWU=
+  data.tar.gz: !binary |-
+    MzgwNDgzMWNhYWZkNGI5YjFlNmVhYjkxZjczNzQ3MjM3NTUyNTAwZjQ5OThi
+    YjUxYzk3MjVjOGQyMTYyNDUwNDgwNGI0YzhkNzg0ZGMxOTAyYjUxMjU4YWFk
+    ZmIyNmFmMmQ0OTllY2I0NGIwMWRlMGI1NDIwY2NjMzU0M2FmMDg=

data/lib/jruby-kafka/consumer.rb

@@ -1,8 +1,14 @@
 require 'java'
 require 'jruby-kafka/namespace'
+require 'jruby-kafka/utility'
 
 class Kafka::Consumer
   java_import 'org.I0Itec.zkclient.exception.ZkException'
+  java_import 'kafka.consumer.ConsumerConfig'
+  java_import 'kafka.consumer.Consumer'
+  java_import 'kafka.consumer.Whitelist'
+  java_import 'kafka.consumer.Blacklist'
+  java_import 'kafka.utils.ZkUtils'
   # Create a Kafka high-level consumer.
   #
   # @param [Hash] config the consumer configuration.
@@ -40,8 +46,14 @@ class Kafka::Consumer
     @key_decoder     =  @properties.delete(:key_decoder) || 'kafka.serializer.DefaultDecoder'
     @msg_decoder     =  @properties.delete(:msg_decoder) || 'kafka.serializer.DefaultDecoder'
     @reset_beginning =  @properties.delete :reset_beginning
+    @consumer        =  nil
+  end
 
-    @consumer = Java::KafkaConsumer::Consumer.createJavaConsumerConnector create_config
+  # Create connection to Kafka and Zookeeper.
+  #
+  # @return void
+  def connect
+    @consumer = Consumer.createJavaConsumerConnector ConsumerConfig.new Kafka::Utility.java_properties @properties
   end
 
   # Start fetching messages.
@@ -54,9 +66,10 @@ class Kafka::Consumer
   # 
   # @note KafkaStreams instances are not thread-safe.
   def message_streams
+    connect if @consumer.nil?
     begin
       if @reset_beginning == 'from-beginning'
-        Java::kafka::utils::ZkUtils.maybeDeletePath(@properties[:zookeeper_connect], "/consumers/#{@properties[:group_id]}")
+        ZkUtils.maybeDeletePath(@properties[:zookeeper_connect], "/consumers/#{@properties[:group_id]}")
       end
     rescue ZkException => e
       raise KafkaError.new(e), "Got ZkException: #{e}"
@@ -74,8 +87,8 @@ class Kafka::Consumer
 
     else
       filter =  @include_topics ? 
-        Java::KafkaConsumer::Whitelist.new(@include_topics) : 
-        Java::KafkaConsumer::Blacklist.new(@exclude_topics)
+        Whitelist.new(@include_topics) :
+        Blacklist.new(@exclude_topics)
 
       @consumer.
         createMessageStreamsByFilter(filter, @num_streams, key_decoder_i, msg_decoder_i).
@@ -98,11 +111,11 @@ class Kafka::Consumer
   # @return void
   def shutdown
     @consumer.shutdown if @consumer
+    @consumer = nil
     nil
   end
   
   private
-
   def validate_arguments(options)
     [:zookeeper_connect, :group_id].each do |opt|
       raise ArgumentError, "Parameter :#{opt} is required." unless options[opt]
@@ -118,15 +131,4 @@ class Kafka::Consumer
       end
     end
   end
-
-  def create_config
-    properties = java.util.Properties.new
-    @properties.each do |k,v|
-      k = k.to_s.gsub '_', '.'
-      v = v.to_s 
-      properties.setProperty k, v
-    end
-    Java::KafkaConsumer::ConsumerConfig.new properties
-  end
 end
-

data/lib/jruby-kafka/kafka-producer.rb

@@ -1,5 +1,6 @@
 require 'jruby-kafka/namespace'
 require 'jruby-kafka/error'
+require 'jruby-kafka/utility'
 
 # noinspection JRubyStringImportInspection
 class Kafka::KafkaProducer
@@ -7,25 +8,8 @@ class Kafka::KafkaProducer
   java_import 'org.apache.kafka.clients.producer.Callback'
   KAFKA_PRODUCER = Java::org.apache.kafka.clients.producer.KafkaProducer
 
-  VALIDATIONS = {
-      :'required.codecs' => %w[
-      none gzip snappy lz4
-    ]
-  }
-
-  REQUIRED = %w[
-    bootstrap.servers key.serializer
-  ]
-
-  KNOWN = %w[
-    acks                      batch.size                block.on.buffer.full
-    bootstrap.servers         buffer.memory             client.id
-    compression.type          key.serializer            linger.ms
-    max.in.flight.requests.per.connection               max.request.size
-    metadata.fetch.timeout.ms metadata.max.age.ms       metric.reporters
-    metrics.num.samples       metrics.sample.window.ms  receive.buffer.bytes
-    reconnect.backoff.ms      retries                   retry.backoff.ms
-    send.buffer.bytes         timeout.ms                value.serializer
+  REQUIRED = [
+    :bootstrap_servers, :key_serializer
   ]
 
   class RubyCallback
@@ -43,18 +27,13 @@ class Kafka::KafkaProducer
   attr_reader :producer, :send_method, :send_cb_method, :options
 
   def initialize(opts = {})
-    @options = opts.reduce({}) do |opts_array, (k, v)|
-      unless v.nil?
-        opts_array[k.to_s.gsub(/_/, '.')] = v
-      end
-      opts_array
-    end
-    validate_arguments
+    Kafka::Utility.validate_arguments REQUIRED, opts
+    @options = opts
     @send_method = @send_cb_method = proc { throw StandardError.new 'Producer is not connected' }
   end
 
   def connect
-    @producer = KAFKA_PRODUCER.new(create_producer_config)
+    @producer = KAFKA_PRODUCER.new(Kafka::Utility.java_properties @options)
     @send_method = producer.java_method :send, [ProducerRecord]
     @send_cb_method = producer.java_method :send, [ProducerRecord, Callback.java_class]
   end
@@ -71,24 +50,4 @@ class Kafka::KafkaProducer
   def close
     @producer.close
   end
-
-  private
-
-  def validate_arguments
-    errors = []
-    missing = REQUIRED.reject { |opt| options[opt] }
-    errors = ["Required settings: #{ missing.join(', ')}"] if missing.any?
-    invalid = VALIDATIONS.reject { |opt, valid| options[opt].nil? or valid.include? options[opt].to_s }
-    errors += invalid.map { |opt, valid| "#{ opt } should be one of: [#{ valid.join(', ')}]" }
-    fail StandardError.new "Invalid configuration arguments: #{ errors.join('; ') }" if errors.any?
-    options.keys.each do |opt|
-      STDERR.puts "WARNING: Unknown configuration key: #{opt}" unless KNOWN.include? opt
-    end
-  end
-
-  def create_producer_config
-    properties = java.util.Properties.new
-    options.each { |opt, value| properties.put opt, value.to_s }
-    properties
-  end
 end

data/lib/jruby-kafka/producer.rb

@@ -1,6 +1,7 @@
 # basically we are porting this https://cwiki.apache.org/confluence/display/KAFKA/0.8.0+Producer+Example
 require 'jruby-kafka/namespace'
 require 'jruby-kafka/error'
+require 'jruby-kafka/utility'
 
 # noinspection JRubyStringImportInspection
 class Kafka::Producer
@@ -8,30 +9,9 @@ class Kafka::Producer
   java_import 'kafka.producer.ProducerConfig'
   java_import 'kafka.producer.KeyedMessage'
   KAFKA_PRODUCER = Java::kafka.javaapi.producer.Producer
-  java_import 'kafka.message.NoCompressionCodec'
-  java_import 'kafka.message.GZIPCompressionCodec'
-  java_import 'kafka.message.SnappyCompressionCodec'
 
-  VALIDATIONS = {
-    :'request.required.acks' => %w[ 0 1 -1 ],
-    :'required.codecs' => [NoCompressionCodec.name, GZIPCompressionCodec.name, SnappyCompressionCodec.name],
-    :'producer.type' => %w[ sync async ]
-  }
-
-  REQUIRED = %w[
-    metadata.broker.list
-  ]
-
-  # List of all available options extracted from http://kafka.apache.org/documentation.html#producerconfigs Apr. 27, 2014
-  # If new options are added, they should just work. Please add them to the list so that we can get handy warnings.
-  KNOWN = %w[
-    metadata.broker.list      request.required.acks         request.timeout.ms
-    producer.type             serializer.class              key.serializer.class
-    partitioner.class         compression.codec             compressed.topics
-    message.send.max.retries  retry.backoff.ms              topic.metadata.refresh.interval.ms
-    queue.buffering.max.ms    queue.buffering.max.messages  queue.enqueue.timeout.ms
-    batch.num.messages        send.buffer.bytes             client.id
-    broker.list               serializer.encoding
+  REQUIRED = [
+    :metadata_broker_list
   ]
 
   attr_reader :producer, :send_method, :options
@@ -41,27 +21,22 @@ class Kafka::Producer
   # options:
   # metadata_broker_list: ["localhost:9092"] - REQUIRED: a seed list of kafka brokers
   def initialize(opts = {})
-    @options = opts.reduce({}) do |opts_array, (k, v)|
-      unless v.nil?
-        opts_array[k.to_s.gsub(/_/, '.')] = v
-      end
-      opts_array
+    @options = opts
+    if options[:broker_list]
+      options[:metadata_broker_list] = options.delete :broker_list
     end
-    if options['broker.list']
-      options['metadata.broker.list'] = options.delete 'broker.list'
+    if options[:metadata_broker_list].is_a? Array
+      options[:metadata_broker_list] = options[:metadata_broker_list].join(',')
     end
-    if options['metadata.broker.list'].is_a? Array
-      options['metadata.broker.list'] = options['metadata.broker.list'].join(',')
+    if options[:compressed_topics].is_a? Array
+      options[:compressed_topics] = options[:compressed_topics].join(',')
     end
-    if options['compressed.topics'].is_a? Array
-      options['compressed.topics'] = options['compressed.topics'].join(',')
-    end
-    validate_arguments
+    Kafka::Utility.validate_arguments REQUIRED, options
     @send_method = proc { throw StandardError.new 'Producer is not connected' }
   end
 
   def connect
-    @producer = KAFKA_PRODUCER.new(create_producer_config)
+    @producer = KAFKA_PRODUCER.new(ProducerConfig.new Kafka::Utility.java_properties @options)
     @send_method = producer.java_method :send, [KeyedMessage]
   end
 
@@ -78,24 +53,4 @@ class Kafka::Producer
   def close
     @producer.close
   end
-
-  private
-
-  def validate_arguments
-    errors = []
-    missing = REQUIRED.reject { |opt| options[opt] }
-    errors = ["Required settings: #{ missing.join(', ')}"] if missing.any?
-    invalid = VALIDATIONS.reject { |opt, valid| options[opt].nil? or valid.include? options[opt].to_s }
-    errors += invalid.map { |opt, valid| "#{ opt } should be one of: [#{ valid.join(', ')}]" }
-    fail StandardError.new "Invalid configuration arguments: #{ errors.join('; ') }" if errors.any?
-    options.keys.each do |opt|
-      STDERR.puts "WARNING: Unknown configuration key: #{opt}" unless KNOWN.include? opt
-    end
-  end
-
-  def create_producer_config
-    properties = java.util.Properties.new
-    options.each { |opt, value| properties.put opt, value.to_s }
-    ProducerConfig.new(properties)
-  end
 end

data/lib/jruby-kafka/utility.rb

@@ -0,0 +1,20 @@
+require 'java'
+require 'jruby-kafka/namespace'
+
+class Kafka::Utility
+  def self.java_properties(properties)
+    java_properties = java.util.Properties.new
+    properties.each do |k,v|
+      k = k.to_s.gsub '_', '.'
+      v = v.to_s
+      java_properties.setProperty k, v
+    end
+    java_properties
+  end
+
+  def self.validate_arguments(required_options, options)
+    required_options.each do |opt|
+      raise ArgumentError, "Parameter :#{opt} is required." unless options[opt]
+    end
+  end
+end

data/lib/jruby-kafka.rb

@@ -1,5 +1,5 @@
-require 'jruby-kafka/version'
 require 'jruby-kafka_jars.rb'
 require 'jruby-kafka/consumer'
 require 'jruby-kafka/producer'
 require 'jruby-kafka/kafka-producer'
+require 'jruby-kafka/utility'

data/lib/jruby-kafka_jars.rb

@@ -1,20 +1,20 @@
 # this is a generated file, to avoid over-writing it just delete this comment
 require 'jar_dependencies'
 
-require_jar( 'io.netty', 'netty', '3.7.0.Final' )
-require_jar( 'log4j', 'log4j', '1.2.17' )
-require_jar( 'net.sf.jopt-simple', 'jopt-simple', '3.2' )
-require_jar( 'org.xerial.snappy', 'snappy-java', '1.1.1.7' )
-require_jar( 'jline', 'jline', '0.9.94' )
-require_jar( 'org.slf4j', 'slf4j-api', '1.7.13' )
-require_jar( 'org.apache.kafka', 'kafka-clients', '0.8.2.2' )
-require_jar( 'org.apache.kafka', 'kafka_2.11', '0.8.2.2' )
-require_jar( 'org.scala-lang.modules', 'scala-xml_2.11', '1.0.2' )
-require_jar( 'junit', 'junit', '3.8.1' )
-require_jar( 'com.101tec', 'zkclient', '0.3' )
-require_jar( 'com.yammer.metrics', 'metrics-core', '2.2.0' )
 require_jar( 'org.scala-lang.modules', 'scala-parser-combinators_2.11', '1.0.2' )
+require_jar( 'log4j', 'log4j', '1.2.17' )
 require_jar( 'org.scala-lang', 'scala-library', '2.11.5' )
 require_jar( 'org.apache.zookeeper', 'zookeeper', '3.4.6' )
 require_jar( 'org.slf4j', 'slf4j-log4j12', '1.7.13' )
+require_jar( 'org.scala-lang.modules', 'scala-xml_2.11', '1.0.2' )
+require_jar( 'org.slf4j', 'slf4j-api', '1.7.13' )
+require_jar( 'io.netty', 'netty', '3.7.0.Final' )
+require_jar( 'net.sf.jopt-simple', 'jopt-simple', '3.2' )
 require_jar( 'net.jpountz.lz4', 'lz4', '1.2.0' )
+require_jar( 'junit', 'junit', '3.8.1' )
+require_jar( 'org.apache.kafka', 'kafka-clients', '0.8.2.2' )
+require_jar( 'com.yammer.metrics', 'metrics-core', '2.2.0' )
+require_jar( 'org.apache.kafka', 'kafka_2.11', '0.8.2.2' )
+require_jar( 'jline', 'jline', '0.9.94' )
+require_jar( 'org.xerial.snappy', 'snappy-java', '1.1.1.7' )
+require_jar( 'com.101tec', 'zkclient', '0.3' )

metadata

@@ -1,72 +1,86 @@
 --- !ruby/object:Gem::Specification
 name: jruby-kafka
 version: !ruby/object:Gem::Version
-  version: '2.1'
+  version: 2.2.2
 platform: java
 authors:
 - Joseph Lawson
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-03-09 00:00:00.000000000 Z
+date: 2016-03-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - <
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - <
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: jar-dependencies
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: 0.3.2
-  name: jar-dependencies
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: 0.3.2
 - !ruby/object:Gem::Dependency
+  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: '10.5'
-  name: rake
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: '10.5'
 - !ruby/object:Gem::Dependency
+  name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: '3.4'
-  name: rspec
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: '3.4'
 - !ruby/object:Gem::Dependency
+  name: ruby-maven
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: '3.3'
-  name: ruby-maven
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: '3.3'
-description: this is primarily to be used as an interface for logstash
+description: A ready to go interface to Kafka for JRuby.
 email:
 - joe@joekiller.com
 executables: []
@@ -83,7 +97,7 @@ files:
 - lib/jruby-kafka/kafka-producer.rb
 - lib/jruby-kafka/namespace.rb
 - lib/jruby-kafka/producer.rb
-- lib/jruby-kafka/version.rb
+- lib/jruby-kafka/utility.rb
 - lib/jruby-kafka_jars.rb
 - lib/junit/junit/3.8.1/junit-3.8.1.jar
 - lib/log4j/log4j/1.2.17/log4j-1.2.17.jar
@@ -102,26 +116,26 @@ homepage: https://github.com/joekiller/jruby-kafka
 licenses:
 - Apache 2.0
 metadata: {}
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements:
 - jar 'org.apache.kafka:kafka_2.11', '0.8.2.2'
 - jar 'org.slf4j:slf4j-log4j12', '1.7.13'
-rubyforge_project:
+rubyforge_project: 
 rubygems_version: 2.4.5
-signing_key:
+signing_key: 
 specification_version: 4
 summary: jruby Kafka wrapper
 test_files: []

data/lib/jruby-kafka/version.rb

@@ -1,4 +0,0 @@
-module Kafka
-  VERSION = '2.1'.freeze
-  JAR_DEPENDENCIES_VERSION = '0.3.2'.freeze
-end