jruby-kafka 3.2-java → 3.3-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5d8bd0569f81f64429f450dfc7fb1ba4535da82b
-  data.tar.gz: 5bacffc343af3b196e2841f49b68cd8c1c6e80dc
+  metadata.gz: 4d17ccab91e382e75bded41bc235dfcf54384034
+  data.tar.gz: 0942e26a59f26f83b4e329cf832efd862aa450b2
 SHA512:
-  metadata.gz: da3dcfb455b352a7ac92ab2fcc77be7c69a7d675a34c1ebf7ad30c48a2fc356e492478c4d2d34f3b1502149b2fe0d9dfba60eb10a14401386568a40ff68a595d
-  data.tar.gz: 279da18f0d1cedb2db4e04956ee585a21302894330149d531e6ab5c42a924cec145f852aa54ea2e5284a7c2a3d4e3d21ffddb190ea3832b06aa905dc1d4b3acb
+  metadata.gz: f6865c7fb4f91557852f6464e4b5f6739727082e7b07ca387667b0d925e173c59e8dc2a1a6b8bcad4546c9e234080754a7f6916722a000eea7c8aa8a4800c904
+  data.tar.gz: ecb86c2213e38e6623f7fdc93605820ca19b7810a82a205c6033d6e093ff62fe3202ee902e5221724655f3e6699736e7692de2d7e40fa03eba4acb49180f148f

data/lib/jruby-kafka.rb

@@ -4,3 +4,4 @@ require 'jruby-kafka/consumer'
 require 'jruby-kafka/producer'
 require 'jruby-kafka/kafka-producer'
 require 'jruby-kafka/kafka-consumer'
+require 'jruby-kafka/utility'

data/lib/jruby-kafka/consumer.rb

@@ -1,8 +1,14 @@
 require 'java'
 require 'jruby-kafka/namespace'
+require 'jruby-kafka/utility'
 
 class Kafka::Consumer
   java_import 'org.I0Itec.zkclient.exception.ZkException'
+  java_import 'kafka.consumer.ConsumerConfig'
+  java_import 'kafka.consumer.Consumer'
+  java_import 'kafka.consumer.Whitelist'
+  java_import 'kafka.consumer.Blacklist'
+  java_import 'kafka.utils.ZkUtils'
   # Create a Kafka high-level consumer.
   #
   # @param [Hash] config the consumer configuration.
@@ -41,7 +47,7 @@ class Kafka::Consumer
     @msg_decoder     =  @properties.delete(:msg_decoder) || 'kafka.serializer.DefaultDecoder'
     @reset_beginning =  @properties.delete :reset_beginning
 
-    @consumer = Java::KafkaConsumer::Consumer.createJavaConsumerConnector create_config
+    @consumer = Consumer.createJavaConsumerConnector ConsumerConfig.new Kafka::Utility.java_properties @properties
   end
 
   # Start fetching messages.
@@ -56,7 +62,7 @@ class Kafka::Consumer
   def message_streams
     begin
       if @reset_beginning == 'from-beginning'
-        Java::kafka::utils::ZkUtils.maybeDeletePath(@properties[:zookeeper_connect], "/consumers/#{@properties[:group_id]}")
+        ZkUtils.maybeDeletePath(@properties[:zookeeper_connect], "/consumers/#{@properties[:group_id]}")
       end
     rescue ZkException => e
       raise KafkaError.new(e), "Got ZkException: #{e}"
@@ -74,8 +80,8 @@ class Kafka::Consumer
 
     else
       filter =  @include_topics ? 
-        Java::KafkaConsumer::Whitelist.new(@include_topics) : 
-        Java::KafkaConsumer::Blacklist.new(@exclude_topics)
+        Whitelist.new(@include_topics) :
+        Blacklist.new(@exclude_topics)
 
       @consumer.
         createMessageStreamsByFilter(filter, @num_streams, key_decoder_i, msg_decoder_i).
@@ -118,15 +124,5 @@ class Kafka::Consumer
       end
     end
   end
-
-  def create_config
-    properties = java.util.Properties.new
-    @properties.each do |k,v|
-      k = k.to_s.gsub '_', '.'
-      v = v.to_s 
-      properties.setProperty k, v
-    end
-    Java::KafkaConsumer::ConsumerConfig.new properties
-  end
 end
 

data/lib/jruby-kafka/kafka-consumer.rb

@@ -1,8 +1,14 @@
 require 'java'
 require 'jruby-kafka/namespace'
-require "concurrent"
+require 'concurrent'
+require 'jruby-kafka/utility'
 
 class Kafka::KafkaConsumer
+
+  REQUIRED = [
+    :bootstrap_servers, :key_deserializer, :value_deserializer
+  ]
+
   KAFKA_CONSUMER = Java::org.apache.kafka.clients.consumer.KafkaConsumer
   # Create a Kafka high-level consumer.
   #
@@ -19,11 +25,11 @@ class Kafka::KafkaConsumer
   # https://kafka.apache.org/090/javadoc/org/apache/kafka/clients/consumer/ConsumerConfig.html.
   #
   def initialize(config={})
-    validate_arguments config
+    Kafka::Utility.validate_arguments REQUIRED, config
     @properties      =  config.clone
     @topics          =  @properties.delete :topics
     @stop_called     =  Concurrent::AtomicBoolean.new(false)
-    @consumer        =  KAFKA_CONSUMER.new(create_config)
+    @consumer        =  KAFKA_CONSUMER.new(Kafka::Utility.java_properties @properties)
     @subscribed      =  false
     subscribe
   end
@@ -55,23 +61,5 @@ class Kafka::KafkaConsumer
   def close
     @consumer.close
   end
-
-  private
-
-  def validate_arguments(options)
-    [:bootstrap_servers, :key_deserializer, :value_deserializer].each do |opt|
-      raise ArgumentError, "Parameter :#{opt} is required." unless options[opt]
-    end
-  end
-
-  def create_config
-    properties = java.util.Properties.new
-    @properties.each do |k,v|
-      k = k.to_s.gsub '_', '.'
-      v = v.to_s 
-      properties.setProperty k, v
-    end
-    properties
-  end
 end
 

data/lib/jruby-kafka/kafka-producer.rb

@@ -1,5 +1,6 @@
 require 'jruby-kafka/namespace'
 require 'jruby-kafka/error'
+require 'jruby-kafka/utility'
 
 # noinspection JRubyStringImportInspection
 class Kafka::KafkaProducer
@@ -7,25 +8,8 @@ class Kafka::KafkaProducer
   java_import 'org.apache.kafka.clients.producer.Callback'
   KAFKA_PRODUCER = Java::org.apache.kafka.clients.producer.KafkaProducer
 
-  VALIDATIONS = {
-      :'required.codecs' => %w[
-      none gzip snappy lz4
-    ]
-  }
-
-  REQUIRED = %w[
-    bootstrap.servers key.serializer
-  ]
-
-  KNOWN = %w[
-    acks                      batch.size                block.on.buffer.full
-    bootstrap.servers         buffer.memory             client.id
-    compression.type          key.serializer            linger.ms
-    max.in.flight.requests.per.connection               max.request.size
-    metadata.fetch.timeout.ms metadata.max.age.ms       metric.reporters
-    metrics.num.samples       metrics.sample.window.ms  receive.buffer.bytes
-    reconnect.backoff.ms      retries                   retry.backoff.ms
-    send.buffer.bytes         timeout.ms                value.serializer
+  REQUIRED = [
+    :bootstrap_servers, :key_serializer
   ]
 
   class RubyCallback
@@ -43,18 +27,13 @@ class Kafka::KafkaProducer
   attr_reader :producer, :send_method, :send_cb_method, :options
 
   def initialize(opts = {})
-    @options = opts.reduce({}) do |opts_array, (k, v)|
-      unless v.nil?
-        opts_array[k.to_s.gsub(/_/, '.')] = v
-      end
-      opts_array
-    end
-    validate_arguments
+    Kafka::Utility.validate_arguments REQUIRED, opts
+    @options = opts
     @send_method = @send_cb_method = proc { throw StandardError.new 'Producer is not connected' }
   end
 
   def connect
-    @producer = KAFKA_PRODUCER.new(create_producer_config)
+    @producer = KAFKA_PRODUCER.new(Kafka::Utility.java_properties @options)
     @send_method = producer.java_method :send, [ProducerRecord]
     @send_cb_method = producer.java_method :send, [ProducerRecord, Callback.java_class]
   end
@@ -71,24 +50,4 @@ class Kafka::KafkaProducer
   def close
     @producer.close
   end
-
-  private
-
-  def validate_arguments
-    errors = []
-    missing = REQUIRED.reject { |opt| options[opt] }
-    errors = ["Required settings: #{ missing.join(', ')}"] if missing.any?
-    invalid = VALIDATIONS.reject { |opt, valid| options[opt].nil? or valid.include? options[opt].to_s }
-    errors += invalid.map { |opt, valid| "#{ opt } should be one of: [#{ valid.join(', ')}]" }
-    fail StandardError.new "Invalid configuration arguments: #{ errors.join('; ') }" if errors.any?
-    options.keys.each do |opt|
-      STDERR.puts "WARNING: Unknown configuration key: #{opt}" unless KNOWN.include? opt
-    end
-  end
-
-  def create_producer_config
-    properties = java.util.Properties.new
-    options.each { |opt, value| properties.put opt, value.to_s }
-    properties
-  end
 end

data/lib/jruby-kafka/producer.rb

@@ -1,6 +1,7 @@
 # basically we are porting this https://cwiki.apache.org/confluence/display/KAFKA/0.8.0+Producer+Example
 require 'jruby-kafka/namespace'
 require 'jruby-kafka/error'
+require 'jruby-kafka/utility'
 
 # noinspection JRubyStringImportInspection
 class Kafka::Producer
@@ -8,30 +9,9 @@ class Kafka::Producer
   java_import 'kafka.producer.ProducerConfig'
   java_import 'kafka.producer.KeyedMessage'
   KAFKA_PRODUCER = Java::kafka.javaapi.producer.Producer
-  java_import 'kafka.message.NoCompressionCodec'
-  java_import 'kafka.message.GZIPCompressionCodec'
-  java_import 'kafka.message.SnappyCompressionCodec'
 
-  VALIDATIONS = {
-    :'request.required.acks' => %w[ 0 1 -1 ],
-    :'required.codecs' => [NoCompressionCodec.name, GZIPCompressionCodec.name, SnappyCompressionCodec.name],
-    :'producer.type' => %w[ sync async ]
-  }
-
-  REQUIRED = %w[
-    metadata.broker.list
-  ]
-
-  # List of all available options extracted from http://kafka.apache.org/documentation.html#producerconfigs Apr. 27, 2014
-  # If new options are added, they should just work. Please add them to the list so that we can get handy warnings.
-  KNOWN = %w[
-    metadata.broker.list      request.required.acks         request.timeout.ms
-    producer.type             serializer.class              key.serializer.class
-    partitioner.class         compression.codec             compressed.topics
-    message.send.max.retries  retry.backoff.ms              topic.metadata.refresh.interval.ms
-    queue.buffering.max.ms    queue.buffering.max.messages  queue.enqueue.timeout.ms
-    batch.num.messages        send.buffer.bytes             client.id
-    broker.list               serializer.encoding
+  REQUIRED = [
+    :metadata_broker_list
   ]
 
   attr_reader :producer, :send_method, :options
@@ -41,27 +21,22 @@ class Kafka::Producer
   # options:
   # metadata_broker_list: ["localhost:9092"] - REQUIRED: a seed list of kafka brokers
   def initialize(opts = {})
-    @options = opts.reduce({}) do |opts_array, (k, v)|
-      unless v.nil?
-        opts_array[k.to_s.gsub(/_/, '.')] = v
-      end
-      opts_array
+    @options = opts
+    if options[:broker_list]
+      options[:metadata_broker_list] = options.delete :broker_list
     end
-    if options['broker.list']
-      options['metadata.broker.list'] = options.delete 'broker.list'
+    if options[:metadata_broker_list].is_a? Array
+      options[:metadata_broker_list] = options[:metadata_broker_list].join(',')
     end
-    if options['metadata.broker.list'].is_a? Array
-      options['metadata.broker.list'] = options['metadata.broker.list'].join(',')
+    if options[:compressed_topics].is_a? Array
+      options[:compressed_topics] = options[:compressed_topics].join(',')
     end
-    if options['compressed.topics'].is_a? Array
-      options['compressed.topics'] = options['compressed.topics'].join(',')
-    end
-    validate_arguments
+    Kafka::Utility.validate_arguments REQUIRED, options
     @send_method = proc { throw StandardError.new 'Producer is not connected' }
   end
 
   def connect
-    @producer = KAFKA_PRODUCER.new(create_producer_config)
+    @producer = KAFKA_PRODUCER.new(ProducerConfig.new Kafka::Utility.java_properties @options)
     @send_method = producer.java_method :send, [KeyedMessage]
   end
 
@@ -78,24 +53,4 @@ class Kafka::Producer
   def close
     @producer.close
   end
-
-  private
-
-  def validate_arguments
-    errors = []
-    missing = REQUIRED.reject { |opt| options[opt] }
-    errors = ["Required settings: #{ missing.join(', ')}"] if missing.any?
-    invalid = VALIDATIONS.reject { |opt, valid| options[opt].nil? or valid.include? options[opt].to_s }
-    errors += invalid.map { |opt, valid| "#{ opt } should be one of: [#{ valid.join(', ')}]" }
-    fail StandardError.new "Invalid configuration arguments: #{ errors.join('; ') }" if errors.any?
-    options.keys.each do |opt|
-      STDERR.puts "WARNING: Unknown configuration key: #{opt}" unless KNOWN.include? opt
-    end
-  end
-
-  def create_producer_config
-    properties = java.util.Properties.new
-    options.each { |opt, value| properties.put opt, value.to_s }
-    ProducerConfig.new(properties)
-  end
 end

data/lib/jruby-kafka/utility.rb

@@ -0,0 +1,20 @@
+require 'java'
+require 'jruby-kafka/namespace'
+
+class Kafka::Utility
+  def self.java_properties(properties)
+    java_properties = java.util.Properties.new
+    properties.each do |k,v|
+      k = k.to_s.gsub '_', '.'
+      v = v.to_s
+      java_properties.setProperty k, v
+    end
+    java_properties
+  end
+
+  def self.validate_arguments(required_options, options)
+    required_options.each do |opt|
+      raise ArgumentError, "Parameter :#{opt} is required." unless options[opt]
+    end
+  end
+end

data/lib/jruby-kafka/version.rb

@@ -1,4 +1,4 @@
 module Kafka
-  VERSION = '3.2'.freeze
+  VERSION = '3.3'.freeze
   JAR_DEPENDENCIES_VERSION = '0.3.2'.freeze
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: jruby-kafka
 version: !ruby/object:Gem::Version
-  version: '3.2'
+  version: '3.3'
 platform: java
 authors:
 - Joseph Lawson
@@ -98,6 +98,7 @@ files:
 - lib/jruby-kafka/kafka-producer.rb
 - lib/jruby-kafka/namespace.rb
 - lib/jruby-kafka/producer.rb
+- lib/jruby-kafka/utility.rb
 - lib/jruby-kafka/version.rb
 - lib/jruby-kafka_jars.rb
 - lib/junit/junit/3.8.1/junit-3.8.1.jar