logstash-kafka 0.7.0-java → 0.7.1-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 00adb861cf25fabd675f94d437b0e4dc36a7fbad
-  data.tar.gz: ce5d97a0e3b9081bb60f595364da29125e2b6a5d
+  metadata.gz: 6d4335bb55722df77c77e125ed449af1314267d6
+  data.tar.gz: e30d2c5517d8d33fafc354f00996eee9bd1bbb19
 SHA512:
-  metadata.gz: 1b0675791f6af79c8cc64371787cb322ba108bdc2d431cf55bf49a543b92a8d4cab09efeb593803b50b05d7dd88f6f3404d76f3207de665c816d5b061183abd2
-  data.tar.gz: 919df956e078237ee9f01de276227b5f61ab454faf830e794962c76fc2132ce0cd928137aeaa648ef36e4fc8e2e9b66a3e893101cde6f150595ac017d401edec
+  metadata.gz: f98d93c2e93a03884331f7931b4b4b76b3eb67600fc3420cbbca2d81581fa9c0876df5d8c51078aae95f1a0e8354febb545b34f37c911f8f98e7f6dd5250c32c
+  data.tar.gz: 944d346a9daf69dbe79cf701abe66d62362acbc7df3d47d94d000f7dee7cecb42727476ebe31d568432e01867c1869c0c774bdfcb34bcf5149f7c36e9dc5abdd

data/lib/logstash/inputs/kafka.rb

@@ -6,7 +6,7 @@ require 'jruby-kafka'
 # by Kafka to read messages from the broker. It also maintains the state of what has been
 # consumed using Zookeeper. The default input codec is json
 #
-# The only required configuration is the topic name. By default it will connect to a Zookeeper
+# You must configure topic_id, white_list or black_list. By default it will connect to a Zookeeper
 # running on localhost. All the broker information is read from Zookeeper state
 #
 # Ideally you should have as many threads as the number of partitions for a perfect balance --
@@ -37,16 +37,19 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
   # the same consumer group.
   config :group_id, :validate => :string, :default => 'logstash'
   # The topic to consume messages from
-  config :topic_id, :validate => :string, :required => true
-  # Specify whether to jump to beginning of the queue when there is no initial offset in
-  # ZooKeeper, or if an offset is out of range. If this is false, messages are consumed
-  # from the latest offset
-  #
-  # If reset_beginning is true, the consumer will check ZooKeeper to see if any other group members
-  # are present and active. If not, the consumer deletes any offset information in the ZooKeeper
-  # and starts at the smallest offset. If other group members are present reset_beginning will not
-  # work and the consumer threads will rejoin the consumer group.
+  config :topic_id, :validate => :string, :default => nil
+  # Whitelist of topics to include for consumption.
+  config :white_list, :validate => :string, :default => nil
+  # Blacklist of topics to exclude from consumption.
+  config :black_list, :validate => :string, :default => nil
+  # Reset the consumer group to start at the earliest message present in the log by clearing any
+  # offsets for the group stored in Zookeeper. This is destructive! Must be used in conjunction
+  # with auto_offset_reset => 'smallest'
   config :reset_beginning, :validate => :boolean, :default => false
+  # "smallest" or "largest" - (optional, default 'largest') If the consumer does not already 
+  # have an established offset or offset is invalid, start with the earliest message present in the log (smallest) or 
+  # after the last message in the log (largest).
+  config :auto_offset_reset, :validate => %w( largest smallest ), :default => 'largest'
   # Number of threads to read from the partitions. Ideally you should have as many threads as the
   # number of partitions for a perfect balance. More threads than partitions means that some
   # threads will be idle. Less threads means a single thread could be consuming from more than
@@ -86,19 +89,28 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
         :zk_connect => @zk_connect,
         :group_id => @group_id,
         :topic_id => @topic_id,
+        :auto_offset_reset => @auto_offset_reset,
         :rebalance_max_retries => @rebalance_max_retries,
         :rebalance_backoff_ms => @rebalance_backoff_ms,
         :consumer_timeout_ms => @consumer_timeout_ms,
         :consumer_restart_on_error => @consumer_restart_on_error,
         :consumer_restart_sleep_ms => @consumer_restart_sleep_ms,
         :consumer_id => @consumer_id,
-        :fetch_message_max_bytes => @fetch_message_max_bytes
+        :fetch_message_max_bytes => @fetch_message_max_bytes,
+        :allow_topics => @white_list,
+        :filter_topics => @black_list
     }
     if @reset_beginning
       options[:reset_beginning] = 'from-beginning'
     end # if :reset_beginning
+    topic_or_filter = [@topic_id, @white_list, @black_list].compact
+    if topic_or_filter.count == 0
+      raise('topic_id, white_list or black_list required.')
+    elsif topic_or_filter.count > 1
+      raise('Invalid combination of topic_id, white_list or black_list. Use only one.')
+    end
     @kafka_client_queue = SizedQueue.new(@queue_size)
-    @consumer_group = Kafka::Group.new(options)
+    @consumer_group = create_consumer_group(options)
     @logger.info('Registering kafka', :group_id => @group_id, :topic_id => @topic_id, :zk_connect => @zk_connect)
   end # def register
 
@@ -111,14 +123,14 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
       begin
         while true
           event = @kafka_client_queue.pop
-          queue_event("#{event}",logstash_queue)
+          queue_event(event, logstash_queue)
         end
       rescue LogStash::ShutdownSignal
         @logger.info('Kafka got shutdown signal')
         @consumer_group.shutdown
       end
       until @kafka_client_queue.empty?
-        queue_event("#{@kafka_client_queue.pop}",logstash_queue)
+        queue_event(@kafka_client_queue.pop,logstash_queue)
       end
       @logger.info('Done running kafka input')
     rescue => e
@@ -134,17 +146,26 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
   end # def run
 
   private
-  def queue_event(msg, output_queue)
+  def create_consumer_group(options)
+    Kafka::Group.new(options)
+  end
+
+  private
+  def queue_event(message_and_metadata, output_queue)
     begin
-      @codec.decode(msg) do |event|
+      @codec.decode(message_and_metadata.message) do |event|
         decorate(event)
         if @decorate_events
-          event['kafka'] = {'msg_size' => msg.bytesize, 'topic' => @topic_id, 'consumer_group' => @group_id}
+          event['kafka'] = {'msg_size' => message_and_metadata.message.bytesize,
+                            'topic' => message_and_metadata.topic,
+                            'consumer_group' => @group_id,
+                            'partition' => message_and_metadata.partition,
+                            'key' => message_and_metadata.key}
         end
         output_queue << event
       end # @codec.decode
     rescue => e # parse or event creation error
-      @logger.error('Failed to create event', :message => msg, :exception => e,
+      @logger.error('Failed to create event', :message => message_and_metadata.message, :exception => e,
                     :backtrace => e.backtrace)
     end # begin
   end # def queue_event

data/lib/logstash/outputs/kafka.rb

@@ -131,9 +131,9 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
 
     @logger.info('Registering kafka producer', :topic_id => @topic_id, :broker_list => @broker_list)
 
-    @codec.on_event do |event|
+    @codec.on_event do |event, data|
       begin
-        @producer.send_msg(@topic_id,nil,event)
+        @producer.send_msg(event.sprintf(@topic_id),nil,data)
       rescue LogStash::ShutdownSignal
         @logger.info('Kafka producer got shutdown signal')
       rescue => e

metadata

@@ -1,30 +1,106 @@
 --- !ruby/object:Gem::Specification
 name: logstash-kafka
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 0.7.1
 platform: java
 authors:
 - Joseph Lawson
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-01-20 00:00:00.000000000 Z
+date: 2015-02-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: jruby-kafka
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.4.0
+    - - <
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+  name: logstash
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.4.0
+    - - <
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-codec-plain
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-codec-json
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: jar-dependencies
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
       - !ruby/object:Gem::Version
-        version: 1.0.0.beta
+        version: 1.1.0.beta
+  name: jruby-kafka
+  prerelease: false
   type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.1.0.beta
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-devutils
   prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - '>='
       - !ruby/object:Gem::Version
-        version: 1.0.0.beta
-description: this is primarily to be used as an interface for logstash
+        version: '0'
+description: Kafka input and output plugins for Logstash
 email:
 - joe@joekiller.com
 executables: []
@@ -33,28 +109,28 @@ extra_rdoc_files: []
 files:
 - lib/logstash/inputs/kafka.rb
 - lib/logstash/outputs/kafka.rb
-homepage: https://github.com/joekiller/jruby-kafka
+homepage: https://github.com/joekiller/logstash-kafka
 licenses:
 - Apache 2.0
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.2.2
-signing_key: 
+rubyforge_project:
+rubygems_version: 2.1.9
+signing_key:
 specification_version: 4
-summary: jruby Kafka wrapper
+summary: Provides input and output plugin functionality for Logstash
 test_files: []