jruby-kafka 2.2.2-java → 3.0-java

checksums.yaml

@@ -1,15 +1,7 @@
 ---
-!binary "U0hBMQ==":
-  metadata.gz: !binary |-
-    M2NjNzliNzVlOGViODcyZTEyN2UwNTA2MTg5ZWI5OTM3NDZjNDYwNA==
-  data.tar.gz: !binary |-
-    ODY0Y2MyNTQyN2QzMjg4N2M3MTVmZDZkNTMzYjA2YTE0NDk0MjYwNQ==
+SHA1:
+  metadata.gz: 3eea2f7be96fb9132520a94b6c71b9175e0518e1
+  data.tar.gz: 520858d8b78e96da2ffc5fcbb30d9e35dfe59598
 SHA512:
-  metadata.gz: !binary |-
-    MDU2OGU4YzRkNjRjNzg4ZTllNGZiM2YzZmM3NTA5NmFhYmU1NTMxYzEyM2Yx
-    MjdlNjM2NWIwMzRkYjE5ZWNmNDJkMWQ2OWYxYmY3ZTIyOGYzMGFjNTUzNWFl
-    ODg0YWZkNjdjNjI0ZTFlN2Q2OTdkNzY0NzkwOTg5MmRjNDRkMWU=
-  data.tar.gz: !binary |-
-    MzgwNDgzMWNhYWZkNGI5YjFlNmVhYjkxZjczNzQ3MjM3NTUyNTAwZjQ5OThi
-    YjUxYzk3MjVjOGQyMTYyNDUwNDgwNGI0YzhkNzg0ZGMxOTAyYjUxMjU4YWFk
-    ZmIyNmFmMmQ0OTllY2I0NGIwMWRlMGI1NDIwY2NjMzU0M2FmMDg=
+  metadata.gz: 3b55b5c2671ae57091c253769ec2d929ca020ffd9606006070ae6159944d1cea0fca13a7895c7e70068f85d2835478bcb7ac917a1a4fa759a1ef9ac84ea53175
+  data.tar.gz: f0c96aba4dec69a653a0ef8decc866f1ac24c09d20fa9cc6c60e003a003b0e42db9e770a983a4f3128da3d5a07a811a9dfef4ad3db086b8e73435b7994244452

data/lib/jruby-kafka/consumer.rb

@@ -1,14 +1,8 @@
 require 'java'
 require 'jruby-kafka/namespace'
-require 'jruby-kafka/utility'
 
 class Kafka::Consumer
   java_import 'org.I0Itec.zkclient.exception.ZkException'
-  java_import 'kafka.consumer.ConsumerConfig'
-  java_import 'kafka.consumer.Consumer'
-  java_import 'kafka.consumer.Whitelist'
-  java_import 'kafka.consumer.Blacklist'
-  java_import 'kafka.utils.ZkUtils'
   # Create a Kafka high-level consumer.
   #
   # @param [Hash] config the consumer configuration.
@@ -46,14 +40,8 @@ class Kafka::Consumer
     @key_decoder     =  @properties.delete(:key_decoder) || 'kafka.serializer.DefaultDecoder'
     @msg_decoder     =  @properties.delete(:msg_decoder) || 'kafka.serializer.DefaultDecoder'
     @reset_beginning =  @properties.delete :reset_beginning
-    @consumer        =  nil
-  end
 
-  # Create connection to Kafka and Zookeeper.
-  #
-  # @return void
-  def connect
-    @consumer = Consumer.createJavaConsumerConnector ConsumerConfig.new Kafka::Utility.java_properties @properties
+    @consumer = Java::KafkaConsumer::Consumer.createJavaConsumerConnector create_config
   end
 
   # Start fetching messages.
@@ -66,10 +54,9 @@ class Kafka::Consumer
   # 
   # @note KafkaStreams instances are not thread-safe.
   def message_streams
-    connect if @consumer.nil?
     begin
       if @reset_beginning == 'from-beginning'
-        ZkUtils.maybeDeletePath(@properties[:zookeeper_connect], "/consumers/#{@properties[:group_id]}")
+        Java::kafka::utils::ZkUtils.maybeDeletePath(@properties[:zookeeper_connect], "/consumers/#{@properties[:group_id]}")
       end
     rescue ZkException => e
       raise KafkaError.new(e), "Got ZkException: #{e}"
@@ -87,8 +74,8 @@ class Kafka::Consumer
 
     else
       filter =  @include_topics ? 
-        Whitelist.new(@include_topics) :
-        Blacklist.new(@exclude_topics)
+        Java::KafkaConsumer::Whitelist.new(@include_topics) : 
+        Java::KafkaConsumer::Blacklist.new(@exclude_topics)
 
       @consumer.
         createMessageStreamsByFilter(filter, @num_streams, key_decoder_i, msg_decoder_i).
@@ -111,11 +98,11 @@ class Kafka::Consumer
   # @return void
   def shutdown
     @consumer.shutdown if @consumer
-    @consumer = nil
     nil
   end
   
   private
+
   def validate_arguments(options)
     [:zookeeper_connect, :group_id].each do |opt|
       raise ArgumentError, "Parameter :#{opt} is required." unless options[opt]
@@ -131,4 +118,15 @@ class Kafka::Consumer
       end
     end
   end
+
+  def create_config
+    properties = java.util.Properties.new
+    @properties.each do |k,v|
+      k = k.to_s.gsub '_', '.'
+      v = v.to_s 
+      properties.setProperty k, v
+    end
+    Java::KafkaConsumer::ConsumerConfig.new properties
+  end
 end
+

data/lib/jruby-kafka/kafka-producer.rb

@@ -1,6 +1,5 @@
 require 'jruby-kafka/namespace'
 require 'jruby-kafka/error'
-require 'jruby-kafka/utility'
 
 # noinspection JRubyStringImportInspection
 class Kafka::KafkaProducer
@@ -8,8 +7,25 @@ class Kafka::KafkaProducer
   java_import 'org.apache.kafka.clients.producer.Callback'
   KAFKA_PRODUCER = Java::org.apache.kafka.clients.producer.KafkaProducer
 
-  REQUIRED = [
-    :bootstrap_servers, :key_serializer
+  VALIDATIONS = {
+      :'required.codecs' => %w[
+      none gzip snappy lz4
+    ]
+  }
+
+  REQUIRED = %w[
+    bootstrap.servers key.serializer
+  ]
+
+  KNOWN = %w[
+    acks                      batch.size                block.on.buffer.full
+    bootstrap.servers         buffer.memory             client.id
+    compression.type          key.serializer            linger.ms
+    max.in.flight.requests.per.connection               max.request.size
+    metadata.fetch.timeout.ms metadata.max.age.ms       metric.reporters
+    metrics.num.samples       metrics.sample.window.ms  receive.buffer.bytes
+    reconnect.backoff.ms      retries                   retry.backoff.ms
+    send.buffer.bytes         timeout.ms                value.serializer
   ]
 
   class RubyCallback
@@ -27,13 +43,18 @@ class Kafka::KafkaProducer
   attr_reader :producer, :send_method, :send_cb_method, :options
 
   def initialize(opts = {})
-    Kafka::Utility.validate_arguments REQUIRED, opts
-    @options = opts
+    @options = opts.reduce({}) do |opts_array, (k, v)|
+      unless v.nil?
+        opts_array[k.to_s.gsub(/_/, '.')] = v
+      end
+      opts_array
+    end
+    validate_arguments
     @send_method = @send_cb_method = proc { throw StandardError.new 'Producer is not connected' }
   end
 
   def connect
-    @producer = KAFKA_PRODUCER.new(Kafka::Utility.java_properties @options)
+    @producer = KAFKA_PRODUCER.new(create_producer_config)
     @send_method = producer.java_method :send, [ProducerRecord]
     @send_cb_method = producer.java_method :send, [ProducerRecord, Callback.java_class]
   end
@@ -50,4 +71,24 @@ class Kafka::KafkaProducer
   def close
     @producer.close
   end
+
+  private
+
+  def validate_arguments
+    errors = []
+    missing = REQUIRED.reject { |opt| options[opt] }
+    errors = ["Required settings: #{ missing.join(', ')}"] if missing.any?
+    invalid = VALIDATIONS.reject { |opt, valid| options[opt].nil? or valid.include? options[opt].to_s }
+    errors += invalid.map { |opt, valid| "#{ opt } should be one of: [#{ valid.join(', ')}]" }
+    fail StandardError.new "Invalid configuration arguments: #{ errors.join('; ') }" if errors.any?
+    options.keys.each do |opt|
+      STDERR.puts "WARNING: Unknown configuration key: #{opt}" unless KNOWN.include? opt
+    end
+  end
+
+  def create_producer_config
+    properties = java.util.Properties.new
+    options.each { |opt, value| properties.put opt, value.to_s }
+    properties
+  end
 end

data/lib/jruby-kafka/producer.rb

@@ -1,7 +1,6 @@
 # basically we are porting this https://cwiki.apache.org/confluence/display/KAFKA/0.8.0+Producer+Example
 require 'jruby-kafka/namespace'
 require 'jruby-kafka/error'
-require 'jruby-kafka/utility'
 
 # noinspection JRubyStringImportInspection
 class Kafka::Producer
@@ -9,9 +8,30 @@ class Kafka::Producer
   java_import 'kafka.producer.ProducerConfig'
   java_import 'kafka.producer.KeyedMessage'
   KAFKA_PRODUCER = Java::kafka.javaapi.producer.Producer
+  java_import 'kafka.message.NoCompressionCodec'
+  java_import 'kafka.message.GZIPCompressionCodec'
+  java_import 'kafka.message.SnappyCompressionCodec'
 
-  REQUIRED = [
-    :metadata_broker_list
+  VALIDATIONS = {
+    :'request.required.acks' => %w[ 0 1 -1 ],
+    :'required.codecs' => [NoCompressionCodec.name, GZIPCompressionCodec.name, SnappyCompressionCodec.name],
+    :'producer.type' => %w[ sync async ]
+  }
+
+  REQUIRED = %w[
+    metadata.broker.list
+  ]
+
+  # List of all available options extracted from http://kafka.apache.org/documentation.html#producerconfigs Apr. 27, 2014
+  # If new options are added, they should just work. Please add them to the list so that we can get handy warnings.
+  KNOWN = %w[
+    metadata.broker.list      request.required.acks         request.timeout.ms
+    producer.type             serializer.class              key.serializer.class
+    partitioner.class         compression.codec             compressed.topics
+    message.send.max.retries  retry.backoff.ms              topic.metadata.refresh.interval.ms
+    queue.buffering.max.ms    queue.buffering.max.messages  queue.enqueue.timeout.ms
+    batch.num.messages        send.buffer.bytes             client.id
+    broker.list               serializer.encoding
   ]
 
   attr_reader :producer, :send_method, :options
@@ -21,22 +41,27 @@ class Kafka::Producer
   # options:
   # metadata_broker_list: ["localhost:9092"] - REQUIRED: a seed list of kafka brokers
   def initialize(opts = {})
-    @options = opts
-    if options[:broker_list]
-      options[:metadata_broker_list] = options.delete :broker_list
+    @options = opts.reduce({}) do |opts_array, (k, v)|
+      unless v.nil?
+        opts_array[k.to_s.gsub(/_/, '.')] = v
+      end
+      opts_array
     end
-    if options[:metadata_broker_list].is_a? Array
-      options[:metadata_broker_list] = options[:metadata_broker_list].join(',')
+    if options['broker.list']
+      options['metadata.broker.list'] = options.delete 'broker.list'
     end
-    if options[:compressed_topics].is_a? Array
-      options[:compressed_topics] = options[:compressed_topics].join(',')
+    if options['metadata.broker.list'].is_a? Array
+      options['metadata.broker.list'] = options['metadata.broker.list'].join(',')
     end
-    Kafka::Utility.validate_arguments REQUIRED, options
+    if options['compressed.topics'].is_a? Array
+      options['compressed.topics'] = options['compressed.topics'].join(',')
+    end
+    validate_arguments
     @send_method = proc { throw StandardError.new 'Producer is not connected' }
   end
 
   def connect
-    @producer = KAFKA_PRODUCER.new(ProducerConfig.new Kafka::Utility.java_properties @options)
+    @producer = KAFKA_PRODUCER.new(create_producer_config)
     @send_method = producer.java_method :send, [KeyedMessage]
   end
 
@@ -53,4 +78,24 @@ class Kafka::Producer
   def close
     @producer.close
   end
+
+  private
+
+  def validate_arguments
+    errors = []
+    missing = REQUIRED.reject { |opt| options[opt] }
+    errors = ["Required settings: #{ missing.join(', ')}"] if missing.any?
+    invalid = VALIDATIONS.reject { |opt, valid| options[opt].nil? or valid.include? options[opt].to_s }
+    errors += invalid.map { |opt, valid| "#{ opt } should be one of: [#{ valid.join(', ')}]" }
+    fail StandardError.new "Invalid configuration arguments: #{ errors.join('; ') }" if errors.any?
+    options.keys.each do |opt|
+      STDERR.puts "WARNING: Unknown configuration key: #{opt}" unless KNOWN.include? opt
+    end
+  end
+
+  def create_producer_config
+    properties = java.util.Properties.new
+    options.each { |opt, value| properties.put opt, value.to_s }
+    ProducerConfig.new(properties)
+  end
 end

data/lib/jruby-kafka/version.rb

@@ -0,0 +1,4 @@
+module Kafka
+  VERSION = '3.0'.freeze
+  JAR_DEPENDENCIES_VERSION = '0.3.2'.freeze
+end

data/lib/jruby-kafka.rb

@@ -1,5 +1,5 @@
+require 'jruby-kafka/version'
 require 'jruby-kafka_jars.rb'
 require 'jruby-kafka/consumer'
 require 'jruby-kafka/producer'
 require 'jruby-kafka/kafka-producer'
-require 'jruby-kafka/utility'

data/lib/jruby-kafka_jars.rb

@@ -1,20 +1,20 @@
 # this is a generated file, to avoid over-writing it just delete this comment
 require 'jar_dependencies'
 
-require_jar( 'org.scala-lang.modules', 'scala-parser-combinators_2.11', '1.0.2' )
-require_jar( 'log4j', 'log4j', '1.2.17' )
-require_jar( 'org.scala-lang', 'scala-library', '2.11.5' )
-require_jar( 'org.apache.zookeeper', 'zookeeper', '3.4.6' )
-require_jar( 'org.slf4j', 'slf4j-log4j12', '1.7.13' )
-require_jar( 'org.scala-lang.modules', 'scala-xml_2.11', '1.0.2' )
-require_jar( 'org.slf4j', 'slf4j-api', '1.7.13' )
 require_jar( 'io.netty', 'netty', '3.7.0.Final' )
+require_jar( 'log4j', 'log4j', '1.2.17' )
+require_jar( 'com.101tec', 'zkclient', '0.7' )
+require_jar( 'org.xerial.snappy', 'snappy-java', '1.1.1.7' )
 require_jar( 'net.sf.jopt-simple', 'jopt-simple', '3.2' )
-require_jar( 'net.jpountz.lz4', 'lz4', '1.2.0' )
+require_jar( 'jline', 'jline', '0.9.94' )
+require_jar( 'org.scala-lang', 'scala-library', '2.11.7' )
+require_jar( 'org.apache.kafka', 'kafka-clients', '0.9.0.0' )
+require_jar( 'org.slf4j', 'slf4j-api', '1.7.13' )
+require_jar( 'org.apache.kafka', 'kafka_2.11', '0.9.0.0' )
 require_jar( 'junit', 'junit', '3.8.1' )
-require_jar( 'org.apache.kafka', 'kafka-clients', '0.8.2.2' )
 require_jar( 'com.yammer.metrics', 'metrics-core', '2.2.0' )
-require_jar( 'org.apache.kafka', 'kafka_2.11', '0.8.2.2' )
-require_jar( 'jline', 'jline', '0.9.94' )
-require_jar( 'org.xerial.snappy', 'snappy-java', '1.1.1.7' )
-require_jar( 'com.101tec', 'zkclient', '0.3' )
+require_jar( 'org.apache.zookeeper', 'zookeeper', '3.4.6' )
+require_jar( 'org.slf4j', 'slf4j-log4j12', '1.7.13' )
+require_jar( 'org.scala-lang.modules', 'scala-parser-combinators_2.11', '1.0.4' )
+require_jar( 'org.scala-lang.modules', 'scala-xml_2.11', '1.0.4' )
+require_jar( 'net.jpountz.lz4', 'lz4', '1.2.0' )

metadata

@@ -1,93 +1,79 @@
 --- !ruby/object:Gem::Specification
 name: jruby-kafka
 version: !ruby/object:Gem::Version
-  version: 2.2.2
+  version: '3.0'
 platform: java
 authors:
 - Joseph Lawson
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-03-22 00:00:00.000000000 Z
+date: 2016-02-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: concurrent-ruby
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - <
-      - !ruby/object:Gem::Version
-        version: '2.0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - <
-      - !ruby/object:Gem::Version
-        version: '2.0'
-- !ruby/object:Gem::Dependency
-  name: jar-dependencies
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: 0.3.2
-  type: :development
+  name: jar-dependencies
   prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: 0.3.2
 - !ruby/object:Gem::Dependency
-  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: '10.5'
-  type: :development
+  name: rake
   prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: '10.5'
 - !ruby/object:Gem::Dependency
-  name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: '3.4'
-  type: :development
+  name: rspec
   prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: '3.4'
 - !ruby/object:Gem::Dependency
-  name: ruby-maven
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: '3.3'
-  type: :development
+  name: ruby-maven
   prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: '3.3'
-description: A ready to go interface to Kafka for JRuby.
+description: this is primarily to be used as an interface for logstash
 email:
 - joe@joekiller.com
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- lib/com/101tec/zkclient/0.3/zkclient-0.3.jar
+- lib/com/101tec/zkclient/0.7/zkclient-0.7.jar
 - lib/com/yammer/metrics/metrics-core/2.2.0/metrics-core-2.2.0.jar
 - lib/io/netty/netty/3.7.0.Final/netty-3.7.0.Final.jar
 - lib/jline/jline/0.9.94/jline-0.9.94.jar
@@ -97,18 +83,18 @@ files:
 - lib/jruby-kafka/kafka-producer.rb
 - lib/jruby-kafka/namespace.rb
 - lib/jruby-kafka/producer.rb
-- lib/jruby-kafka/utility.rb
+- lib/jruby-kafka/version.rb
 - lib/jruby-kafka_jars.rb
 - lib/junit/junit/3.8.1/junit-3.8.1.jar
 - lib/log4j/log4j/1.2.17/log4j-1.2.17.jar
 - lib/net/jpountz/lz4/lz4/1.2.0/lz4-1.2.0.jar
 - lib/net/sf/jopt-simple/jopt-simple/3.2/jopt-simple-3.2.jar
-- lib/org/apache/kafka/kafka-clients/0.8.2.2/kafka-clients-0.8.2.2.jar
-- lib/org/apache/kafka/kafka_2.11/0.8.2.2/kafka_2.11-0.8.2.2.jar
+- lib/org/apache/kafka/kafka-clients/0.9.0.0/kafka-clients-0.9.0.0.jar
+- lib/org/apache/kafka/kafka_2.11/0.9.0.0/kafka_2.11-0.9.0.0.jar
 - lib/org/apache/zookeeper/zookeeper/3.4.6/zookeeper-3.4.6.jar
-- lib/org/scala-lang/modules/scala-parser-combinators_2.11/1.0.2/scala-parser-combinators_2.11-1.0.2.jar
-- lib/org/scala-lang/modules/scala-xml_2.11/1.0.2/scala-xml_2.11-1.0.2.jar
-- lib/org/scala-lang/scala-library/2.11.5/scala-library-2.11.5.jar
+- lib/org/scala-lang/modules/scala-parser-combinators_2.11/1.0.4/scala-parser-combinators_2.11-1.0.4.jar
+- lib/org/scala-lang/modules/scala-xml_2.11/1.0.4/scala-xml_2.11-1.0.4.jar
+- lib/org/scala-lang/scala-library/2.11.7/scala-library-2.11.7.jar
 - lib/org/slf4j/slf4j-api/1.7.13/slf4j-api-1.7.13.jar
 - lib/org/slf4j/slf4j-log4j12/1.7.13/slf4j-log4j12-1.7.13.jar
 - lib/org/xerial/snappy/snappy-java/1.1.1.7/snappy-java-1.1.1.7.jar
@@ -116,26 +102,26 @@ homepage: https://github.com/joekiller/jruby-kafka
 licenses:
 - Apache 2.0
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements:
-- jar 'org.apache.kafka:kafka_2.11', '0.8.2.2'
+- jar 'org.apache.kafka:kafka_2.11', '0.9.0.0'
 - jar 'org.slf4j:slf4j-log4j12', '1.7.13'
-rubyforge_project: 
+rubyforge_project:
 rubygems_version: 2.4.5
-signing_key: 
+signing_key:
 specification_version: 4
 summary: jruby Kafka wrapper
 test_files: []

data/lib/jruby-kafka/utility.rb

@@ -1,20 +0,0 @@
-require 'java'
-require 'jruby-kafka/namespace'
-
-class Kafka::Utility
-  def self.java_properties(properties)
-    java_properties = java.util.Properties.new
-    properties.each do |k,v|
-      k = k.to_s.gsub '_', '.'
-      v = v.to_s
-      java_properties.setProperty k, v
-    end
-    java_properties
-  end
-
-  def self.validate_arguments(required_options, options)
-    required_options.each do |opt|
-      raise ArgumentError, "Parameter :#{opt} is required." unless options[opt]
-    end
-  end
-end