jruby-kafka 0.0.1 → 0.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 432b700c2a60813577f72680a4d7921959a07ebd
-  data.tar.gz: d99addc2c02cba26700a0c9b7172aeb559aaf082
+  metadata.gz: f29e4e146f6d94e9aa3622e67897435f6303f7fe
+  data.tar.gz: 6d588c93d8349ed20c27d6d24997acbaaf0effa8
 SHA512:
-  metadata.gz: 7d460eab892d5f04cbe2a9a61b751a54bc0af22b78b886e02fbd9d0bf2890c4264d683e8dca95112bd2d091816c681a3ba368e52815ffe063df9009882030292
-  data.tar.gz: b32eeb94554fe4d39379c2a283aa995914bbd1ea858aa0b1781477b8c9f6ca31c7af4ecd93cdc45360f01e13c4627cc5e304cb861a334c47e48dab2889d71e1d
+  metadata.gz: af2d509b566451f0a37b36ec87e1929bf5e2ea34a01b9ff57d9a561e9f8d12ae512b01a346794f74d19cd094dd4a32fe3c26266027b6584387421de0597b5167
+  data.tar.gz: 040f39ffb72f187efce499345212b67155d07a9552067d9456fc85697701413f9953c3cf142dc6ce7d35b30bddcaf3563b616d02c76463ffb0a90d050633c84b

data/lib/jruby-kafka/client.rb

@@ -0,0 +1,6 @@
+require "java"
+require "jruby-kafka/namespace"
+
+class Kafka::Client
+
+end

data/lib/jruby-kafka/error.rb

@@ -0,0 +1,9 @@
+require "jruby-kafka/namespace"
+
+class KafkaError < StandardError
+  attr_reader :object
+
+  def initialize(object)
+    @object = object
+  end
+end

data/lib/jruby-kafka/group.rb

@@ -4,9 +4,11 @@ require "java"
 
 require "jruby-kafka/namespace"
 require "jruby-kafka/consumer"
+require "jruby-kafka/error"
 
 java_import 'java.util.concurrent.ExecutorService'
 java_import 'java.util.concurrent.Executors'
+java_import 'org.I0Itec.zkclient.exception.ZkException'
 
 class Kafka::Group
   @consumer
@@ -19,40 +21,53 @@ class Kafka::Group
   # Create a Kafka client group
   #
   # options:
-  # :zk_connect_opt => "localhost:2181" - REQUIRED: The connection string for the
+  # :zk_connect => "localhost:2181" - REQUIRED: The connection string for the
   #   zookeeper connection in the form host:port. Multiple URLS can be given to allow fail-over.
-  # :group_id_opt => "group" - REQUIRED: The group id to consume on.
-  # :topic_id_opt => "topic" - REQUIRED: The topic id to consume on.
-  # :reset_beginning_opt => "from-beginning" - (optional) If the consumer does not already have an established offset
+  # :zk_connect_timeout => "6000" - (optional) The max time that the client waits while establishing a connection to zookeeper.
+  # :group_id => "group" - REQUIRED: The group id to consume on.
+  # :topic_id => "topic" - REQUIRED: The topic id to consume on.
+  # :reset_beginning => "from-beginning" - (optional) If the consumer does not already have an established offset
   #   to consume from, start with the earliest message present in the log rather than the latest message.
+  #
   def initialize(options={})
     validate_required_arguments(options)
 
-    @zk_connect = options[:zk_connect_opt]
-    @group_id = options[:group_id_opt]
-    @topic = options[:topic_id_opt]
+    @zk_connect = options[:zk_connect]
+    @group_id = options[:group_id]
+    @topic = options[:topic_id]
+    @zk_session_timeout = '6000'
+    @zk_connect_timeout = '6000'
+    @zk_sync_time = '2000'
+    @auto_offset_reset = 'largest'
+    @auto_commit_interval = '1000'
+    @running = false
+
+    if options[:zk_connect_timeout]
+      @zk_connect_timeout = options[:zk_connect_timeout]
+    end
+    if options[:zk_session_timeout]
+      @zk_session_timeout = options[:zk_session_timeout]
+    end
+    if options[:zk_sync_time]
+      @zk_sync_time = options[:zk_sync_time]
+    end
+    if options[:auto_commit_interval]
+      @auto_commit_interval = options[:auto_commit_interval]
+    end
 
 
-    if options[:reset_beginning_opt]
-      if options[:reset_beginning_opt] == 'from-beginning'
+    if options[:reset_beginning]
+      if options[:reset_beginning] == 'from-beginning'
         @auto_offset_reset = 'smallest'
       else
         @auto_offset_reset = 'largest'
       end
-    else
-      @auto_offset_reset = 'largest'
-    end
-
-    if @auto_offset_reset == 'smallest'
-      Java::kafka::utils::ZkUtils.maybeDeletePath(@zk_connect, "/consumers/#{@group_id}")
     end
-
-    @consumer = Java::kafka::consumer::Consumer.createJavaConsumerConnector(createConsumerConfig())
   end
 
   private
   def validate_required_arguments(options={})
-    [:zk_connect_opt, :group_id_opt, :topic_id_opt].each do |opt|
+    [:zk_connect, :group_id, :topic_id].each do |opt|
       raise(ArgumentError, "#{opt} is required.") unless options[opt]
     end
   end
@@ -65,10 +80,20 @@ class Kafka::Group
     if @executor
       @executor.shutdown()
     end
+    @running = false
   end
 
   public
   def run(a_numThreads, a_queue)
+    begin
+      if @auto_offset_reset == 'smallest'
+        Java::kafka::utils::ZkUtils.maybeDeletePath(@zk_connect, "/consumers/#{@group_id}")
+      end
+
+      @consumer = Java::kafka::consumer::Consumer.createJavaConsumerConnector(createConsumerConfig())
+    rescue ZkException => e
+      raise KafkaError.new(e), "Got ZkException: #{e}"
+    end
     topicCountMap = java.util.HashMap.new()
     thread_value = a_numThreads.to_java Java::int
     topicCountMap.put(@topic, thread_value)
@@ -76,12 +101,19 @@ class Kafka::Group
     streams = Array.new(consumerMap[@topic])
 
     @executor = Executors.newFixedThreadPool(a_numThreads)
+    @executor_submit = @executor.java_method(:submit, [Java::JavaLang::Runnable.java_class])
 
     threadNumber = 0
     for stream in streams
-      @executor.submit(Kafka::Consumer.new(stream, threadNumber, a_queue))
+      @executor_submit.call(Kafka::Consumer.new(stream, threadNumber, a_queue))
       threadNumber += 1
     end
+    @running = true
+  end
+
+  public
+  def running?
+    @running
   end
 
   private
@@ -89,9 +121,10 @@ class Kafka::Group
     properties = java.util.Properties.new()
     properties.put("zookeeper.connect", @zk_connect)
     properties.put("group.id", @group_id)
-    properties.put("zookeeper.session.timeout.ms", "400")
-    properties.put("zookeeper.sync.time.ms", "200")
-    properties.put("auto.commit.interval.ms", "1000")
+    properties.put("zookeeper.connection.timeout.ms", @zk_connect_timeout)
+    properties.put("zookeeper.session.timeout.ms", @zk_session_timeout)
+    properties.put("zookeeper.sync.time.ms", @zk_sync_time)
+    properties.put("auto.commit.interval.ms", @auto_commit_interval)
     properties.put("auto.offset.reset", @auto_offset_reset)
     return Java::kafka::consumer::ConsumerConfig.new(properties)
   end

data/lib/kafka.rb

@@ -0,0 +1,5 @@
+require "jruby-kafka/client"
+
+module Kafka
+
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: jruby-kafka
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.3
 platform: ruby
 authors:
 - Joseph Lawson
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2013-10-02 00:00:00.000000000 Z
+date: 2013-10-16 00:00:00.000000000 Z
 dependencies: []
 description: this is primarily to be used as an interface for logstash
 email:
@@ -18,8 +18,11 @@ extensions: []
 extra_rdoc_files: []
 files:
 - lib/jruby-kafka.rb
+- lib/kafka.rb
+- lib/jruby-kafka/client.rb
 - lib/jruby-kafka/config.rb
 - lib/jruby-kafka/consumer.rb
+- lib/jruby-kafka/error.rb
 - lib/jruby-kafka/group.rb
 - lib/jruby-kafka/namespace.rb
 homepage: https://github.com/joekiller/jruby-kafka