logstash-input-kafka 0.1.9 → 0.1.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0cf869a56fae132e42e6a9733672292db9df822a
-  data.tar.gz: f8e8e1fe70b33fbf5516131d1fb51a9d87ba3f66
+  metadata.gz: 276ddeae9feef8142105c787ed19827747e6d924
+  data.tar.gz: 80064509099da476a1b19c3d655fad96633ca7e8
 SHA512:
-  metadata.gz: 25cf9ff2e77349460df96cd4f7c37aa75a72c5bbc10b5ae5dce0986ad38775357a08e3358c2af343d075ee7868a01abbfa7f3647a679e386173c800f6bbc2370
-  data.tar.gz: 5021919e331ea96f3955615954740ebeb37230b6e333c7b481b646953a6861823dd754f51dd00b035a2e3c1307d822919356ede73bb32cc425f8d425131f029c
+  metadata.gz: dd14bff5887a805b851523880ea76839057e4e01f08c6ca69dbca2d718c67b9426467104312d94825f247b2dd3a5efcfed021744c5a5fece98e7de77d8a0f572
+  data.tar.gz: c4bc212388522afdfdefb84ad0eaa3d91dee786d45abd492c9b724a1d8ec0fb06f6cbad2f5d0ee6ee7020a836697ef2d73c5fb62ceba0e3ad3efc04a179a1c85

data/DEVELOPER.md

@@ -14,10 +14,13 @@ See http://kafka.apache.org/documentation.html#consumerconfigs for details about
 
     input {
         kafka {
-            topic_id => ... # string (required), The topic to consume messages from
+            topic_id => ... # string (optional), default: nil, The topic to consume messages from. Can be a java regular expression for whitelist of topics.
+            white_list => ... # string (optional), default: nil, Blacklist of topics to exclude from consumption.
+            black_list => ... # string (optional), default: nil, Whitelist of topics to include for consumption.
             zk_connect => ... # string (optional), default: "localhost:2181", Specifies the ZooKeeper connection string in the form hostname:port
             group_id => ... # string (optional), default: "logstash", A string that uniquely identifies the group of consumer processes
             reset_beginning => ... # boolean (optional), default: false, Specify whether to jump to beginning of the queue when there is no initial offset in ZK
+            auto_offset_reset => ... # string (optional), one of [ "largest", "smallest"] default => 'largest', Where consumer should start if group does not already have an established offset or offset is invalid
             consumer_threads => ... # number (optional), default: 1, Number of threads to read from the partitions
             queue_size => ... # number (optional), default: 20, Internal Logstash queue size used to hold events in memory 
             rebalance_max_retries => ... # number (optional), default: 4
@@ -26,8 +29,8 @@ See http://kafka.apache.org/documentation.html#consumerconfigs for details about
             consumer_restart_on_error => ... # boolean (optional), default: true
             consumer_restart_sleep_ms => ... # number (optional), default: 0
             decorate_events => ... # boolean (optional), default: false, Option to add Kafka metadata like topic, message size to the event
-            consumer_id => ... # string (optional) default: nil
-            fetch_message_max_bytes => ... # number (optional) default: 1048576
+            consumer_id => ... # string (optional), default: nil
+            fetch_message_max_bytes => ... # number (optional), default: 1048576
         }
     }
 

data/lib/logstash/inputs/kafka.rb

@@ -1,13 +1,13 @@
 require 'logstash/namespace'
 require 'logstash/inputs/base'
-require 'logstash-input-kafka_jars'
+require 'jruby-kafka'
 
 # This input will read events from a Kafka topic. It uses the high level consumer API provided
 # by Kafka to read messages from the broker. It also maintains the state of what has been
 # consumed using Zookeeper. The default input codec is json
 #
-# The only required configuration is the topic name. By default it will connect to a Zookeeper
-# running on localhost. All the broker information is read from Zookeeper state
+# You must configure `topic_id`, `white_list` or `black_list`. By default it will connect to a
+# Zookeeper running on localhost. All the broker information is read from Zookeeper state
 #
 # Ideally you should have as many threads as the number of partitions for a perfect balance --
 # more threads than partitions means that some threads will be idle
@@ -36,16 +36,19 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
   # the same consumer group.
   config :group_id, :validate => :string, :default => 'logstash'
   # The topic to consume messages from
-  config :topic_id, :validate => :string, :required => true
-  # Specify whether to jump to beginning of the queue when there is no initial offset in
-  # ZooKeeper, or if an offset is out of range. If this is `false`, messages are consumed
-  # from the latest offset
-  #
-  # If `reset_beginning` is true, the consumer will check ZooKeeper to see if any other group members
-  # are present and active. If not, the consumer deletes any offset information in the ZooKeeper
-  # and starts at the smallest offset. If other group members are present `reset_beginning` will not
-  # work and the consumer threads will rejoin the consumer group.
+  config :topic_id, :validate => :string, :default => nil
+  # Whitelist of topics to include for consumption.
+  config :white_list, :validate => :string, :default => nil
+  # Blacklist of topics to exclude from consumption.
+  config :black_list, :validate => :string, :default => nil
+  # Reset the consumer group to start at the earliest message present in the log by clearing any
+  # offsets for the group stored in Zookeeper. This is destructive! Must be used in conjunction
+  # with auto_offset_reset => 'smallest'
   config :reset_beginning, :validate => :boolean, :default => false
+  # `smallest` or `largest` - (optional, default `largest`) If the consumer does not already
+  # have an established offset or offset is invalid, start with the earliest message present in the
+  # log (`smallest`) or after the last message in the log (`largest`).
+  config :auto_offset_reset, :validate => %w( largest smallest ), :default => 'largest'
   # Number of threads to read from the partitions. Ideally you should have as many threads as the
   # number of partitions for a perfect balance. More threads than partitions means that some
   # threads will be idle. Less threads means a single thread could be consuming from more than
@@ -81,23 +84,30 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
   public
   def register
     LogStash::Logger.setup_log4j(@logger)
-    require 'jruby-kafka'
-
     options = {
         :zk_connect => @zk_connect,
         :group_id => @group_id,
         :topic_id => @topic_id,
+        :auto_offset_reset => @auto_offset_reset,
         :rebalance_max_retries => @rebalance_max_retries,
         :rebalance_backoff_ms => @rebalance_backoff_ms,
         :consumer_timeout_ms => @consumer_timeout_ms,
         :consumer_restart_on_error => @consumer_restart_on_error,
         :consumer_restart_sleep_ms => @consumer_restart_sleep_ms,
         :consumer_id => @consumer_id,
-        :fetch_message_max_bytes => @fetch_message_max_bytes
+        :fetch_message_max_bytes => @fetch_message_max_bytes,
+        :allow_topics => @white_list,
+        :filter_topics => @black_list
     }
     if @reset_beginning
       options[:reset_beginning] = 'from-beginning'
     end # if :reset_beginning
+    topic_or_filter = [@topic_id, @white_list, @black_list].compact
+    if topic_or_filter.count == 0
+      raise LogStash::ConfigurationError, 'topic_id, white_list or black_list required.'
+    elsif topic_or_filter.count > 1
+      raise LogStash::ConfigurationError, 'Invalid combination of topic_id, white_list or black_list. Use only one.'
+    end
     @kafka_client_queue = SizedQueue.new(@queue_size)
     @consumer_group = create_consumer_group(options)
     @logger.info('Registering kafka', :group_id => @group_id, :topic_id => @topic_id, :zk_connect => @zk_connect)
@@ -113,14 +123,14 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
       begin
         while true
           event = @kafka_client_queue.pop
-          queue_event("#{event}",logstash_queue)
+          queue_event(event, logstash_queue)
         end
       rescue LogStash::ShutdownSignal
         @logger.info('Kafka got shutdown signal')
         @consumer_group.shutdown
       end
       until @kafka_client_queue.empty?
-        queue_event("#{@kafka_client_queue.pop}",logstash_queue)
+        queue_event(@kafka_client_queue.pop,logstash_queue)
       end
       @logger.info('Done running kafka input')
     rescue => e
@@ -141,17 +151,21 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
   end
 
   private
-  def queue_event(msg, output_queue)
+  def queue_event(message_and_metadata, output_queue)
     begin
-      @codec.decode(msg) do |event|
+      @codec.decode("#{message_and_metadata.message}") do |event|
         decorate(event)
         if @decorate_events
-          event['kafka'] = {'msg_size' => msg.bytesize, 'topic' => @topic_id, 'consumer_group' => @group_id}
+          event['kafka'] = {'msg_size' => event['message'].bytesize,
+                            'topic' => message_and_metadata.topic,
+                            'consumer_group' => @group_id,
+                            'partition' => message_and_metadata.partition,
+                            'key' => message_and_metadata.key}
         end
         output_queue << event
       end # @codec.decode
     rescue => e # parse or event creation error
-      @logger.error('Failed to create event', :message => msg, :exception => e,
+      @logger.error('Failed to create event', :message => "#{message_and_metadata.message}", :exception => e,
                     :backtrace => e.backtrace)
     end # begin
   end # def queue_event

data/logstash-input-kafka.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-kafka'
-  s.version         = '0.1.9'
+  s.version         = '0.1.10'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = 'This input will read events from a Kafka topic. It uses the high level consumer API provided by Kafka to read messages from the broker'
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
@@ -19,10 +19,6 @@ Gem::Specification.new do |s|
   # Special flag to let us know this is actually a logstash plugin
   s.metadata = { 'logstash_plugin' => 'true', 'group' => 'input'}
 
-  # Jar dependencies
-  s.requirements << "jar 'org.apache.kafka:kafka_2.9.2', '0.8.1.1'"
-  s.requirements << "jar 'org.slf4j:slf4j-log4j12', '1.7.10'"
-
   # Gem dependencies
   s.add_runtime_dependency 'logstash', '>= 1.4.0', '< 2.0.0'
   s.add_runtime_dependency 'logstash-codec-json'
@@ -39,7 +35,7 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'ruby-maven', '3.1.1.0.8'
   s.add_runtime_dependency "maven-tools", '1.0.7'
 
-  s.add_runtime_dependency 'jruby-kafka', ['>=0.2.1']
+  s.add_runtime_dependency 'jruby-kafka', ['>= 1.1.0', '< 2.0.0']
 
   s.add_development_dependency 'logstash-devutils'
 end

data/spec/inputs/kafka_spec.rb

@@ -12,15 +12,29 @@ class LogStash::Inputs::TestKafka < LogStash::Inputs::Kafka
   end
 end
 
+class TestMessageAndMetadata
+  attr_reader :topic, :partition, :key, :message
+  def initialize(topic, partition, key, message)
+    @topic = topic
+    @partition = partition
+    @key = key
+    @message = message
+  end
+end
+
 
 class TestKafkaGroup < Kafka::Group
   def run(a_num_threads, a_queue)
-    a_queue << 'Kafka message'
+    blah = TestMessageAndMetadata.new(@topic, 0, nil, 'Kafka message')
+    a_queue << blah
   end
 end
 
 describe 'inputs/kafka' do
   let (:kafka_config) {{'topic_id' => 'test'}}
+  let (:empty_config) {{}}
+  let (:bad_kafka_config) {{'topic_id' => 'test', 'white_list' => 'other_topic'}}
+  let (:white_list_kafka_config) {{'white_list' => 'other_topic'}}
   let (:decorated_kafka_config) {{'topic_id' => 'test', 'decorate_events' => true}}
 
   it "should register" do
@@ -28,6 +42,21 @@ describe 'inputs/kafka' do
     expect {input.register}.to_not raise_error
   end
 
+  it "should register with whitelist" do
+    input = LogStash::Plugin.lookup("input", "kafka").new(white_list_kafka_config)
+    expect {input.register}.to_not raise_error
+  end
+
+  it "should fail with multiple topic configs" do
+    input = LogStash::Plugin.lookup("input", "kafka").new(empty_config)
+    expect {input.register}.to raise_error
+  end
+
+  it "should fail without topic configs" do
+    input = LogStash::Plugin.lookup("input", "kafka").new(bad_kafka_config)
+    expect {input.register}.to raise_error
+  end
+
   it 'should populate kafka config with default values' do
     kafka = LogStash::Inputs::TestKafka.new(kafka_config)
     insist {kafka.zk_connect} == 'localhost:2181'
@@ -66,6 +95,8 @@ describe 'inputs/kafka' do
     insist { e['kafka']['topic'] } == 'test'
     insist { e['kafka']['consumer_group'] } == 'logstash'
     insist { e['kafka']['msg_size'] } == 13
+    insist { e['kafka']['partition'] } == 0
+    insist { e['kafka']['key'] } == nil
   end
 
 end

metadata

@@ -1,17 +1,18 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-kafka
 version: !ruby/object:Gem::Version
-  version: 0.1.9
+  version: 0.1.10
 platform: ruby
 authors:
 - Elasticsearch
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-02-13 00:00:00.000000000 Z
+date: 2015-02-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
+  name: logstash
+  version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
@@ -19,10 +20,7 @@ dependencies:
     - - <
       - !ruby/object:Gem::Version
         version: 2.0.0
-  name: logstash
-  prerelease: false
-  type: :runtime
-  version_requirements: !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
@@ -30,104 +28,112 @@ dependencies:
     - - <
       - !ruby/object:Gem::Version
         version: 2.0.0
+  prerelease: false
+  type: :runtime
 - !ruby/object:Gem::Dependency
+  name: logstash-codec-json
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  name: logstash-codec-json
   prerelease: false
   type: :runtime
+- !ruby/object:Gem::Dependency
+  name: logstash-codec-plain
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  name: logstash-codec-plain
   prerelease: false
   type: :runtime
+- !ruby/object:Gem::Dependency
+  name: jar-dependencies
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
+        version: 0.1.7
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
         version: 0.1.7
-  name: jar-dependencies
   prerelease: false
   type: :runtime
+- !ruby/object:Gem::Dependency
+  name: ruby-maven
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.1.7
-- !ruby/object:Gem::Dependency
+        version: 3.1.1.0.8
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
         version: 3.1.1.0.8
-  name: ruby-maven
   prerelease: false
   type: :runtime
+- !ruby/object:Gem::Dependency
+  name: maven-tools
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 3.1.1.0.8
-- !ruby/object:Gem::Dependency
+        version: 1.0.7
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
         version: 1.0.7
-  name: maven-tools
   prerelease: false
   type: :runtime
+- !ruby/object:Gem::Dependency
+  name: jruby-kafka
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - '>='
       - !ruby/object:Gem::Version
-        version: 1.0.7
-- !ruby/object:Gem::Dependency
+        version: 1.1.0
+    - - <
+      - !ruby/object:Gem::Version
+        version: 2.0.0
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: 0.2.1
-  name: jruby-kafka
+        version: 1.1.0
+    - - <
+      - !ruby/object:Gem::Version
+        version: 2.0.0
   prerelease: false
   type: :runtime
+- !ruby/object:Gem::Dependency
+  name: logstash-devutils
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: 0.2.1
-- !ruby/object:Gem::Dependency
+        version: '0'
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  name: logstash-devutils
   prerelease: false
   type: :development
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: '0'
 description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
 email: info@elasticsearch.com
 executables: []
@@ -164,9 +170,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
   - - '>='
     - !ruby/object:Gem::Version
       version: '0'
-requirements:
-- jar 'org.apache.kafka:kafka_2.9.2', '0.8.1.1'
-- jar 'org.slf4j:slf4j-log4j12', '1.7.10'
+requirements: []
 rubyforge_project:
 rubygems_version: 2.1.9
 signing_key: