jruby-kafka 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 432b700c2a60813577f72680a4d7921959a07ebd
+  data.tar.gz: d99addc2c02cba26700a0c9b7172aeb559aaf082
+SHA512:
+  metadata.gz: 7d460eab892d5f04cbe2a9a61b751a54bc0af22b78b886e02fbd9d0bf2890c4264d683e8dca95112bd2d091816c681a3ba368e52815ffe063df9009882030292
+  data.tar.gz: b32eeb94554fe4d39379c2a283aa995914bbd1ea858aa0b1781477b8c9f6ca31c7af4ecd93cdc45360f01e13c4627cc5e304cb861a334c47e48dab2889d71e1d

data/lib/jruby-kafka.rb

@@ -0,0 +1,6 @@
+require "jruby-kafka/consumer"
+require "jruby-kafka/group"
+
+module Kafka
+
+end

data/lib/jruby-kafka/config.rb

@@ -0,0 +1,8 @@
+require "java"
+require "jruby-kafka/namespace"
+
+class Kafka::Config
+  def initialize(options={})
+
+  end
+end

data/lib/jruby-kafka/consumer.rb

@@ -0,0 +1,27 @@
+require "java"
+require "jruby-kafka/namespace"
+
+java_import 'kafka.consumer.ConsumerIterator'
+java_import 'kafka.consumer.KafkaStream'
+
+class Kafka::Consumer
+  include Java::JavaLang::Runnable
+  java_signature 'void run()'
+
+  @m_stream
+  @m_threadNumber
+  @m_queue
+
+    def initialize(a_stream, a_threadNumber, a_queue)
+      @m_threadNumber = a_threadNumber
+      @m_stream = a_stream
+      @m_queue = a_queue
+    end
+
+    def run
+      it = @m_stream.iterator()
+      while it.hasNext()
+        @m_queue << it.next().message()
+      end
+    end
+end

data/lib/jruby-kafka/group.rb

@@ -0,0 +1,98 @@
+# basically we are porting this https://cwiki.apache.org/confluence/display/KAFKA/Consumer+Group+Example
+
+require "java"
+
+require "jruby-kafka/namespace"
+require "jruby-kafka/consumer"
+
+java_import 'java.util.concurrent.ExecutorService'
+java_import 'java.util.concurrent.Executors'
+
+class Kafka::Group
+  @consumer
+  @executor
+  @topic
+  @auto_offset_reset
+  @zk_connect
+  @group_id
+
+  # Create a Kafka client group
+  #
+  # options:
+  # :zk_connect_opt => "localhost:2181" - REQUIRED: The connection string for the
+  #   zookeeper connection in the form host:port. Multiple URLS can be given to allow fail-over.
+  # :group_id_opt => "group" - REQUIRED: The group id to consume on.
+  # :topic_id_opt => "topic" - REQUIRED: The topic id to consume on.
+  # :reset_beginning_opt => "from-beginning" - (optional) If the consumer does not already have an established offset
+  #   to consume from, start with the earliest message present in the log rather than the latest message.
+  def initialize(options={})
+    validate_required_arguments(options)
+
+    @zk_connect = options[:zk_connect_opt]
+    @group_id = options[:group_id_opt]
+    @topic = options[:topic_id_opt]
+
+
+    if options[:reset_beginning_opt]
+      if options[:reset_beginning_opt] == 'from-beginning'
+        @auto_offset_reset = 'smallest'
+      else
+        @auto_offset_reset = 'largest'
+      end
+    else
+      @auto_offset_reset = 'largest'
+    end
+
+    if @auto_offset_reset == 'smallest'
+      Java::kafka::utils::ZkUtils.maybeDeletePath(@zk_connect, "/consumers/#{@group_id}")
+    end
+
+    @consumer = Java::kafka::consumer::Consumer.createJavaConsumerConnector(createConsumerConfig())
+  end
+
+  private
+  def validate_required_arguments(options={})
+    [:zk_connect_opt, :group_id_opt, :topic_id_opt].each do |opt|
+      raise(ArgumentError, "#{opt} is required.") unless options[opt]
+    end
+  end
+
+  public
+  def shutdown()
+    if @consumer
+      @consumer.shutdown()
+    end
+    if @executor
+      @executor.shutdown()
+    end
+  end
+
+  public
+  def run(a_numThreads, a_queue)
+    topicCountMap = java.util.HashMap.new()
+    thread_value = a_numThreads.to_java Java::int
+    topicCountMap.put(@topic, thread_value)
+    consumerMap = @consumer.createMessageStreams(topicCountMap)
+    streams = Array.new(consumerMap[@topic])
+
+    @executor = Executors.newFixedThreadPool(a_numThreads)
+
+    threadNumber = 0
+    for stream in streams
+      @executor.submit(Kafka::Consumer.new(stream, threadNumber, a_queue))
+      threadNumber += 1
+    end
+  end
+
+  private
+  def createConsumerConfig()
+    properties = java.util.Properties.new()
+    properties.put("zookeeper.connect", @zk_connect)
+    properties.put("group.id", @group_id)
+    properties.put("zookeeper.session.timeout.ms", "400")
+    properties.put("zookeeper.sync.time.ms", "200")
+    properties.put("auto.commit.interval.ms", "1000")
+    properties.put("auto.offset.reset", @auto_offset_reset)
+    return Java::kafka::consumer::ConsumerConfig.new(properties)
+  end
+end

data/lib/jruby-kafka/namespace.rb

@@ -0,0 +1,3 @@
+module Kafka
+
+end

metadata

@@ -0,0 +1,50 @@
+--- !ruby/object:Gem::Specification
+name: jruby-kafka
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Joseph Lawson
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2013-10-02 00:00:00.000000000 Z
+dependencies: []
+description: this is primarily to be used as an interface for logstash
+email:
+- joe@joekiller.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/jruby-kafka.rb
+- lib/jruby-kafka/config.rb
+- lib/jruby-kafka/consumer.rb
+- lib/jruby-kafka/group.rb
+- lib/jruby-kafka/namespace.rb
+homepage: https://github.com/joekiller/jruby-kafka
+licenses:
+- Apache 2.0
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.0.3
+signing_key:
+specification_version: 4
+summary: jruby Kafka wrapper
+test_files: []