jruby-kafka-lockjar 1.4.0.pre-java

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: adedc59b55590e41c4a622bd463dec1ebf998d70
+  data.tar.gz: b27119eee15c967e74be17c0ccf05d4e8002b246
+SHA512:
+  metadata.gz: 16e36cbb128ee4c4a2488f4142229a52af9949491844a52ebe646135ac1d38c8c64820808f2273382fe8922799b0bbe175dad266d473d82bab3e15d66b4c0d4e
+  data.tar.gz: 10c1b52c9f8f3cfdc18d8b5bb1b4bf8adac9c0ab12b763efe41bb723d3537f49704bb3b60d28469c522852c68d2c5b80ea4857bdc2f28a6892fc2d2c584151cb

data/Jarfile

@@ -0,0 +1,2 @@
+jar 'org.apache.kafka:kafka_2.10:0.8.2.1'
+jar 'org.slf4j:slf4j-log4j12:1.7.10'

data/lib/jruby-kafka.rb

@@ -0,0 +1,10 @@
+require 'lock_jar'
+LockJar.lock File.join(File.dirname(__FILE__), '../Jarfile'), lockfile: 'Jarfile.jruby-kafka.lock'
+LockJar.load 'Jarfile.jruby-kafka.lock'
+require 'jruby-kafka/consumer'
+require 'jruby-kafka/group'
+require 'jruby-kafka/producer'
+require 'jruby-kafka/kafka-producer'
+
+module Kafka
+end

data/lib/jruby-kafka/consumer.rb

@@ -0,0 +1,41 @@
+require 'java'
+require 'jruby-kafka/namespace'
+
+# noinspection JRubyStringImportInspection
+class Kafka::Consumer
+  java_import 'kafka.consumer.ConsumerIterator'
+  java_import 'kafka.consumer.KafkaStream'
+  java_import 'kafka.common.ConsumerRebalanceFailedException'
+  java_import 'kafka.consumer.ConsumerTimeoutException'
+
+  include Java::JavaLang::Runnable
+  java_signature 'void run()'
+
+  def initialize(a_stream, a_thread_number, a_queue, restart_on_exception, a_sleep_ms)
+    @m_thread_number = a_thread_number
+    @m_stream = a_stream
+    @m_queue = a_queue
+    @m_restart_on_exception = restart_on_exception
+    @m_sleep_ms = 1.0 / 1000.0 * Float(a_sleep_ms)
+  end
+
+  def run
+    it = @m_stream.iterator
+    begin
+      while it.hasNext
+        begin
+          @m_queue << it.next
+        end
+      end
+    rescue Exception => e
+      puts("#{self.class.name} caught exception: #{e.class.name}")
+      puts(e.message) if e.message != ''
+      if @m_restart_on_exception
+        sleep(@m_sleep_ms)
+        retry
+      else
+        raise e
+      end
+    end
+  end
+end

data/lib/jruby-kafka/error.rb

@@ -0,0 +1,9 @@
+require 'jruby-kafka/namespace'
+
+class KafkaError < StandardError
+  attr_reader :object
+
+  def initialize(object)
+    @object = object
+  end
+end

data/lib/jruby-kafka/group.rb

@@ -0,0 +1,246 @@
+# basically we are porting this https://cwiki.apache.org/confluence/display/KAFKA/Consumer+Group+Example
+require 'jruby-kafka/namespace'
+require 'jruby-kafka/consumer'
+require 'jruby-kafka/error'
+
+# noinspection JRubyStringImportInspection
+class Kafka::Group
+  java_import 'java.util.concurrent.ExecutorService'
+  java_import 'java.util.concurrent.Executors'
+  java_import 'org.I0Itec.zkclient.exception.ZkException'
+
+  # Create a Kafka client group
+  #
+  # options:
+  # :zk_connect => "localhost:2181" - REQUIRED: The connection string for the
+  #   zookeeper connection in the form host:port. Multiple URLS can be given to allow fail-over.
+  # :zk_connect_timeout => "6000" - (optional) The max time that the client waits while establishing a connection to zookeeper.
+  # :group_id => "group" - REQUIRED: The group id to consume on.
+  # :topic_id => "topic" - REQUIRED: The topic id to consume on.
+  # :reset_beginning => "from-beginning" - (optional) reset the consumer group to start at the 
+  #   earliest message present in the log by clearing any offsets for the group stored in Zookeeper.
+  # :auto_offset_reset => "smallest" or "largest" - (optional, default 'largest') If the consumer does not already 
+  #   have an established offset to consume from, start with the earliest message present in the log (smallest) or 
+  #   after the last message in the log (largest).
+  # :consumer_restart_on_error => "true" - (optional) Controls if consumer threads are to restart on caught exceptions.
+  #   exceptions are logged.
+  def initialize(options={})
+    validate_required_arguments(options)
+
+    @zk_connect = options[:zk_connect]
+    @group_id = options[:group_id]
+    @topic = options[:topic_id]
+    @topics_allowed = options[:allow_topics]
+    @topics_filtered = options[:filter_topics]
+    @zk_session_timeout = '6000'
+    @zk_connect_timeout = '6000'
+    @zk_sync_time = '2000'
+    @reset_beginning = nil
+    @auto_offset_reset = 'largest'
+    @auto_commit_interval = '1000'
+    @running = false
+    @rebalance_max_retries = '4'
+    @rebalance_backoff_ms = '2000'
+    @socket_timeout_ms = "#{30 * 1000}"
+    @socket_receive_buffer_bytes = "#{64 * 1024}"
+    @fetch_message_max_bytes = "#{1024 * 1024}"
+    @auto_commit_enable = "#{true}"
+    @queued_max_message_chunks = '10'
+    @fetch_min_bytes = '1'
+    @fetch_wait_max_ms = '100'
+    @refresh_leader_backoff_ms = '200'
+    @consumer_timeout_ms = '-1'
+    @consumer_restart_on_error = "#{false}"
+    @consumer_restart_sleep_ms = '0'
+    @consumer_id = nil
+    @key_decoder_class = "kafka.serializer.DefaultDecoder"
+    @value_decoder_class = "kafka.serializer.DefaultDecoder"
+
+    if options[:zk_connect_timeout]
+      @zk_connect_timeout = "#{options[:zk_connect_timeout]}"
+    end
+    if options[:zk_session_timeout]
+      @zk_session_timeout = "#{options[:zk_session_timeout]}"
+    end
+    if options[:zk_sync_time]
+      @zk_sync_time = "#{options[:zk_sync_time]}"
+    end
+    if options[:auto_commit_interval]
+      @auto_commit_interval = "#{options[:auto_commit_interval]}"
+    end
+
+    if options[:rebalance_max_retries]
+      @rebalance_max_retries = "#{options[:rebalance_max_retries]}"
+    end
+
+    if options[:rebalance_backoff_ms]
+      @rebalance_backoff_ms = "#{options[:rebalance_backoff_ms]}"
+    end
+
+    if options[:socket_timeout_ms]
+      @socket_timeout_ms = "#{options[:socket_timeout_ms]}"
+    end
+
+    if options[:socket_receive_buffer_bytes]
+      @socket_receive_buffer_bytes = "#{options[:socket_receive_buffer_bytes]}"
+    end
+
+    if options[:fetch_message_max_bytes]
+      @fetch_message_max_bytes = "#{options[:fetch_message_max_bytes]}"
+    end
+
+    if options[:auto_commit_enable]
+      @auto_commit_enable = "#{options[:auto_commit_enable]}"
+    end
+
+    if options[:queued_max_message_chunks]
+      @queued_max_message_chunks = "#{options[:queued_max_message_chunks]}"
+    end
+
+    if options[:fetch_min_bytes]
+      @fetch_min_bytes = "#{options[:fetch_min_bytes]}"
+    end
+
+    if options[:fetch_wait_max_ms]
+      @fetch_wait_max_ms = "#{options[:fetch_wait_max_ms]}"
+    end
+
+    if options[:refresh_leader_backoff_ms]
+      @refresh_leader_backoff_ms = "#{options[:refresh_leader_backoff_ms]}"
+    end
+
+    if options[:consumer_timeout_ms]
+      @consumer_timeout_ms = "#{options[:consumer_timeout_ms]}"
+    end
+
+    if options[:consumer_restart_on_error]
+      @consumer_restart_on_error = "#{options[:consumer_restart_on_error]}"
+    end
+
+    if options[:consumer_restart_sleep_ms]
+      @consumer_restart_sleep_ms = "#{options[:consumer_restart_sleep_ms]}"
+    end
+
+    if options[:auto_offset_reset]
+      @auto_offset_reset = "#{options[:auto_offset_reset]}"
+    end
+
+    if options[:key_decoder_class]
+      @key_decoder_class = "#{options[:key_decoder_class]}"
+    end
+
+    if options[:value_decoder_class]
+      @value_decoder_class = "#{options[:value_decoder_class]}"
+    end
+
+    if options[:reset_beginning]
+      if not options[:auto_offset_reset] || options[:auto_offset_reset] != 'smallest'
+        raise KafkaError.new('reset_beginning => from-beginning must be used with auto_offset_reset => smallest')
+      end
+      @reset_beginning = "#{options[:reset_beginning]}"
+    end
+
+    if options[:consumer_id]
+      @consumer_id = options[:consumer_id]
+    end
+  end
+
+  public
+
+  def shutdown
+    if @consumer
+      @consumer.shutdown
+    end
+    if @executor
+      @executor.shutdown
+    end
+    @running = false
+  end
+
+  def run(a_num_threads, a_queue)
+    begin
+      if @reset_beginning == 'from-beginning'
+        Java::kafka::utils::ZkUtils.maybeDeletePath(@zk_connect, "/consumers/#{@group_id}")
+      end
+
+      @consumer = Java::kafka::consumer::Consumer.createJavaConsumerConnector(create_consumer_config)
+    rescue ZkException => e
+      raise KafkaError.new(e), "Got ZkException: #{e}"
+    end
+
+    thread_value = a_num_threads.to_java Java::int
+    streams = get_streams(thread_value)
+
+    @executor = Executors.newFixedThreadPool(a_num_threads)
+    @executor_submit = @executor.java_method(:submit, [Java::JavaLang::Runnable.java_class])
+
+    thread_number = 0
+    streams.each do |stream|
+      @executor_submit.call(Kafka::Consumer.new(stream, thread_number, a_queue, @consumer_restart_on_error, @consumer_restart_sleep_ms))
+      thread_number += 1
+    end
+    @running = true
+  end
+
+  def running?
+    @running
+  end
+
+  private
+
+  def validate_required_arguments(options={})
+    [:zk_connect, :group_id].each do |opt|
+      raise(ArgumentError, "#{opt} is required.") unless options[opt]
+    end
+    unless [ options[:topic_id],
+             options[:allow_topics],
+             options[:filter_topics] ].compact.length == 1
+      raise(ArgumentError,
+            "exactly one of topic_id, allow_topics, filter_topics is required.")
+    end
+  end
+
+  def get_streams(threads)
+    constructor_param_class_name = "kafka.utils.VerifiableProperties"
+    key_decoder_instance = Java::JavaClass.for_name(@key_decoder_class).constructor(constructor_param_class_name).new_instance(nil)
+    value_decoder_instance = Java::JavaClass.for_name(@value_decoder_class).constructor(constructor_param_class_name).new_instance(nil)
+    if @topic
+      topic_count_map = java.util.HashMap.new
+      topic_count_map.put(@topic, threads)
+      consumer_map = @consumer.createMessageStreams(topic_count_map, key_decoder_instance, value_decoder_instance)
+      Array.new(consumer_map[@topic])
+    elsif @topics_allowed
+      filter = Java::kafka::consumer::Whitelist.new(@topics_allowed)
+      Array.new(@consumer.createMessageStreamsByFilter(filter, threads, key_decoder_instance, value_decoder_instance))
+    else # @topics_filtered
+      filter = Java::kafka::consumer::Blacklist.new(@topics_filtered)
+      Array.new(@consumer.createMessageStreamsByFilter(filter, threads, key_decoder_instance, value_decoder_instance))
+    end
+  end
+
+  def create_consumer_config
+    properties = java.util.Properties.new
+    properties.put('zookeeper.connect', @zk_connect)
+    properties.put('group.id', @group_id)
+    properties.put('zookeeper.connection.timeout.ms', @zk_connect_timeout)
+    properties.put('zookeeper.session.timeout.ms', @zk_session_timeout)
+    properties.put('zookeeper.sync.time.ms', @zk_sync_time)
+    properties.put('auto.commit.interval.ms', @auto_commit_interval)
+    properties.put('auto.offset.reset', @auto_offset_reset)
+    properties.put('rebalance.max.retries', @rebalance_max_retries)
+    properties.put('rebalance.backoff.ms', @rebalance_backoff_ms)
+    properties.put('socket.timeout.ms', @socket_timeout_ms)
+    properties.put('socket.receive.buffer.bytes', @socket_receive_buffer_bytes)
+    properties.put('fetch.message.max.bytes', @fetch_message_max_bytes)
+    properties.put('auto.commit.enable', @auto_commit_enable)
+    properties.put('queued.max.message.chunks', @queued_max_message_chunks)
+    properties.put('fetch.min.bytes', @fetch_min_bytes)
+    properties.put('fetch.wait.max.ms', @fetch_wait_max_ms)
+    properties.put('refresh.leader.backoff.ms', @refresh_leader_backoff_ms)
+    properties.put('consumer.timeout.ms', @consumer_timeout_ms)
+    unless @consumer_id.nil?
+      properties.put('consumer.id', @consumer_id)
+    end
+    Java::kafka::consumer::ConsumerConfig.new(properties)
+  end
+end

data/lib/jruby-kafka/kafka-producer.rb

@@ -0,0 +1,76 @@
+require 'jruby-kafka/namespace'
+require 'jruby-kafka/error'
+
+# noinspection JRubyStringImportInspection
+class Kafka::KafkaProducer
+  java_import 'org.apache.kafka.clients.producer.ProducerRecord'
+  KAFKA_PRODUCER = Java::org.apache.kafka.clients.producer.KafkaProducer
+
+  VALIDATIONS = {
+      :'required.codecs' => %w[
+      none gzip snappy lz4
+    ]
+  }
+
+  REQUIRED = %w[
+    bootstrap.servers key.serializer
+  ]
+
+  KNOWN = %w[
+    acks                      batch.size                block.on.buffer.full
+    bootstrap.servers         buffer.memory             client.id
+    compression.type          key.serializer            linger.ms
+    max.in.flight.requests.per.connection               max.request.size
+    metadata.fetch.timeout.ms metadata.max.age.ms       metric.reporters
+    metrics.num.samples       metrics.sample.window.ms  receive.buffer.bytes
+    reconnect.backoff.ms      retries                   retry.backoff.ms
+    send.buffer.bytes         timeout.ms                value.serializer
+  ]
+
+  attr_reader :producer, :send_method, :options
+
+  def initialize(opts = {})
+    @options = opts.reduce({}) do |opts_array, (k, v)|
+      unless v.nil?
+        opts_array[k.to_s.gsub(/_/, '.')] = v
+      end
+      opts_array
+    end
+    validate_arguments
+    @send_method = proc { throw StandardError.new 'Producer is not connected' }
+  end
+
+  def connect
+    @producer = KAFKA_PRODUCER.new(create_producer_config)
+    @send_method = producer.java_method :send, [ProducerRecord]
+  end
+
+  # throws FailedToSendMessageException or if not connected, StandardError.
+  def send_msg(topic, partition, key, value)
+    send_method.call(ProducerRecord.new(topic, partition, key, value))
+  end
+
+  def close
+    @producer.close
+  end
+
+  private
+
+  def validate_arguments
+    errors = []
+    missing = REQUIRED.reject { |opt| options[opt] }
+    errors = ["Required settings: #{ missing.join(', ')}"] if missing.any?
+    invalid = VALIDATIONS.reject { |opt, valid| options[opt].nil? or valid.include? options[opt].to_s }
+    errors += invalid.map { |opt, valid| "#{ opt } should be one of: [#{ valid.join(', ')}]" }
+    fail StandardError.new "Invalid configuration arguments: #{ errors.join('; ') }" if errors.any?
+    options.keys.each do |opt|
+      STDERR.puts "WARNING: Unknown configuration key: #{opt}" unless KNOWN.include? opt
+    end
+  end
+
+  def create_producer_config
+    properties = java.util.Properties.new
+    options.each { |opt, value| properties.put opt, value.to_s }
+    properties
+  end
+end

data/lib/jruby-kafka/namespace.rb

@@ -0,0 +1,3 @@
+module Kafka
+
+end

data/lib/jruby-kafka/producer.rb

@@ -0,0 +1,101 @@
+# basically we are porting this https://cwiki.apache.org/confluence/display/KAFKA/0.8.0+Producer+Example
+require 'jruby-kafka/namespace'
+require 'jruby-kafka/error'
+
+# noinspection JRubyStringImportInspection
+class Kafka::Producer
+  extend Gem::Deprecate
+  java_import 'kafka.producer.ProducerConfig'
+  java_import 'kafka.producer.KeyedMessage'
+  KAFKA_PRODUCER = Java::kafka.javaapi.producer.Producer
+  java_import 'kafka.message.NoCompressionCodec'
+  java_import 'kafka.message.GZIPCompressionCodec'
+  java_import 'kafka.message.SnappyCompressionCodec'
+
+  VALIDATIONS = {
+    :'request.required.acks' => %w[ 0 1 -1 ],
+    :'required.codecs' => [NoCompressionCodec.name, GZIPCompressionCodec.name, SnappyCompressionCodec.name],
+    :'producer.type' => %w[ sync async ]
+  }
+
+  REQUIRED = %w[
+    metadata.broker.list
+  ]
+
+  # List of all available options extracted from http://kafka.apache.org/documentation.html#producerconfigs Apr. 27, 2014
+  # If new options are added, they should just work. Please add them to the list so that we can get handy warnings.
+  KNOWN = %w[
+    metadata.broker.list      request.required.acks         request.timeout.ms
+    producer.type             serializer.class              key.serializer.class
+    partitioner.class         compression.codec             compressed.topics
+    message.send.max.retries  retry.backoff.ms              topic.metadata.refresh.interval.ms
+    queue.buffering.max.ms    queue.buffering.max.messages  queue.enqueue.timeout.ms
+    batch.num.messages        send.buffer.bytes             client.id
+    broker.list               serializer.encoding
+  ]
+
+  attr_reader :producer, :send_method, :options
+
+  # Create a Kafka Producer
+  #
+  # options:
+  # metadata_broker_list: ["localhost:9092"] - REQUIRED: a seed list of kafka brokers
+  def initialize(opts = {})
+    @options = opts.reduce({}) do |opts_array, (k, v)|
+      unless v.nil?
+        opts_array[k.to_s.gsub(/_/, '.')] = v
+      end
+      opts_array
+    end
+    if options['broker.list']
+      options['metadata.broker.list'] = options.delete 'broker.list'
+    end
+    if options['metadata.broker.list'].is_a? Array
+      options['metadata.broker.list'] = options['metadata.broker.list'].join(',')
+    end
+    if options['compressed.topics'].is_a? Array
+      options['compressed.topics'] = options['compressed.topics'].join(',')
+    end
+    validate_arguments
+    @send_method = proc { throw StandardError.new 'Producer is not connected' }
+  end
+
+  def connect
+    @producer = KAFKA_PRODUCER.new(create_producer_config)
+    @send_method = producer.java_method :send, [KeyedMessage]
+  end
+
+  # throws FailedToSendMessageException or if not connected, StandardError.
+  def send_msg(topic, key, msg)
+    send_method.call(KeyedMessage.new(topic, key, msg))
+  end
+
+  def sendMsg(topic, key, msg)
+    send_msg(topic, key, msg)
+  end
+  deprecate :sendMsg, :send_msg, 2015, 01
+
+  def close
+    @producer.close
+  end
+
+  private
+
+  def validate_arguments
+    errors = []
+    missing = REQUIRED.reject { |opt| options[opt] }
+    errors = ["Required settings: #{ missing.join(', ')}"] if missing.any?
+    invalid = VALIDATIONS.reject { |opt, valid| options[opt].nil? or valid.include? options[opt].to_s }
+    errors += invalid.map { |opt, valid| "#{ opt } should be one of: [#{ valid.join(', ')}]" }
+    fail StandardError.new "Invalid configuration arguments: #{ errors.join('; ') }" if errors.any?
+    options.keys.each do |opt|
+      STDERR.puts "WARNING: Unknown configuration key: #{opt}" unless KNOWN.include? opt
+    end
+  end
+
+  def create_producer_config
+    properties = java.util.Properties.new
+    options.each { |opt, value| properties.put opt, value.to_s }
+    ProducerConfig.new(properties)
+  end
+end

metadata

@@ -0,0 +1,82 @@
+--- !ruby/object:Gem::Specification
+name: jruby-kafka-lockjar
+version: !ruby/object:Gem::Version
+  version: 1.4.0.pre
+platform: java
+authors:
+- Joseph Lawson
+- Darrick Wiebe
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2015-05-20 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: lock_jar
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: rake
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '10.4'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '10.4'
+  prerelease: false
+  type: :development
+description: Fork of jruby-kafka that uses lockjar instead of ruby-maven
+email:
+- joe@joekiller.com
+- dw@xnlogic.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- Jarfile
+- lib/jruby-kafka.rb
+- lib/jruby-kafka/consumer.rb
+- lib/jruby-kafka/error.rb
+- lib/jruby-kafka/group.rb
+- lib/jruby-kafka/kafka-producer.rb
+- lib/jruby-kafka/namespace.rb
+- lib/jruby-kafka/producer.rb
+homepage: https://github.com/xnlogic/jruby-kafka-lockjar
+licenses:
+- Apache 2.0
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>'
+    - !ruby/object:Gem::Version
+      version: 1.3.1
+requirements: []
+rubyforge_project:
+rubygems_version: 2.4.5
+signing_key:
+specification_version: 4
+summary: JRuby Kafka wrapper
+test_files: []