logstash-kafka 0.7.3-java → 0.7.4-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d7b82a85fc1104b64c4f624d47552563c6c525a8
-  data.tar.gz: d724c2364a9953ed92b7ac544b8df547bdbb8f35
+  metadata.gz: a520ddbf6cb6a908962b69b3fbbb89de78134f29
+  data.tar.gz: 19634df5edd8eabdd8e1dc8599eeff9c9ee15b24
 SHA512:
-  metadata.gz: 0daf018fbc3a6559a1c808e48a330fb9ba8472573b4417c6f3e3fcd1c08df4a691c7c7c65ac4c440b755cbb55fd2bd23b56a477f39873384e5122c8d9fa484b1
-  data.tar.gz: e133e09305b42af3815edd60f216f046b62acbd44eefc630e757269d149c8f919d26dd4c89667f5fd6897ac145c8c606c3a1dea2edef9ccfc269d904930b5424
+  metadata.gz: 8ba628001dadd30db6a48b6fa2de474009ccb5e9c83edf457ddf7f0ec74d5905686884ab90cdbefd6f2573657e3a00b19fba68c2b7cc9312a836f8730c7fc5db
+  data.tar.gz: b6db481e43f90f4cb708d3c3d129ef689c51f3282e003a68578145d22d02f296c6860b0070b0eac0f2c266141e5a3957e45dc8e8d0c7546cc9aa82edd607dc7f

data/lib/logstash/outputs/kafka.rb

@@ -64,7 +64,7 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
   # throughput) but open the possibility of a failure of the client machine dropping unsent data.
   config :producer_type, :validate => %w( sync async ), :default => 'sync'
   # The serializer class for keys (defaults to the same as for messages if nothing is given)
-  config :key_serializer_class, :validate => :string, :default => nil
+  config :key_serializer_class, :validate => :string, :default => 'kafka.serializer.StringEncoder'
   # This property will cause the producer to automatically retry a failed send request. This
   # property specifies the number of retries when such failures occur. Note that setting a
   # non-zero value here can lead to duplicates in the case of network errors that cause a message
@@ -101,30 +101,40 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
   config :send_buffer_bytes, :validate => :number, :default => 100 * 1024
   # The client id is a user-specified string sent in each request to help trace calls. It should
   # logically identify the application making the request.
-  config :client_id, :validate => :string, :default => ''
+  config :client_id, :validate => :string, :default => ""
+  # # Provides a way to specify a partition key as a string. To specify a partition key for
+  # Kafka, configure a format that will produce the key as a string. Defaults key_serializer to 
+  # kafka.serializer.StringEncoder to match. For example, to partition by host:
+  #     output {
+  #       kafka {
+  #           partition_key_format => "%{host}"
+  #       }
+  #     }
+  config :partition_key_format, :validate => :string, :default => nil
 
   public
   def register
     LogStash::Logger.setup_log4j(@logger)
     options = {
-        :broker_list => @broker_list,
-        :compression_codec => @compression_codec,
-        :compressed_topics => @compressed_topics,
-        :request_required_acks => @request_required_acks,
-        :serializer_class => @serializer_class,
-        :partitioner_class => @partitioner_class,
-        :request_timeout_ms => @request_timeout_ms,
-        :producer_type => @producer_type,
-        :key_serializer_class => @key_serializer_class,
-        :message_send_max_retries => @message_send_max_retries,
-        :retry_backoff_ms => @retry_backoff_ms,
-        :topic_metadata_refresh_interval_ms => @topic_metadata_refresh_interval_ms,
-        :queue_buffering_max_ms => @queue_buffering_max_ms,
-        :queue_buffering_max_messages => @queue_buffering_max_messages,
-        :queue_enqueue_timeout_ms => @queue_enqueue_timeout_ms,
-        :batch_num_messages => @batch_num_messages,
-        :send_buffer_bytes => @send_buffer_bytes,
-        :client_id => @client_id
+      :broker_list => @broker_list,
+      :compression_codec => @compression_codec,
+      :compressed_topics => @compressed_topics,
+      :request_required_acks => @request_required_acks,
+      :serializer_class => @serializer_class,
+      :partitioner_class => @partitioner_class,
+      :request_timeout_ms => @request_timeout_ms,
+      :producer_type => @producer_type,
+      :key_serializer_class => @key_serializer_class,
+      :message_send_max_retries => @message_send_max_retries,
+      :retry_backoff_ms => @retry_backoff_ms,
+      :topic_metadata_refresh_interval_ms => @topic_metadata_refresh_interval_ms,
+      :queue_buffering_max_ms => @queue_buffering_max_ms,
+      :queue_buffering_max_messages => @queue_buffering_max_messages,
+      :queue_enqueue_timeout_ms => @queue_enqueue_timeout_ms,
+      :batch_num_messages => @batch_num_messages,
+      :send_buffer_bytes => @send_buffer_bytes,
+      :client_id => @client_id,
+      :partition_key_format => @partition_key_format
     }
     @producer = Kafka::Producer.new(options)
     @producer.connect
@@ -133,7 +143,7 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
 
     @codec.on_event do |event, data|
       begin
-        @producer.send_msg(event.sprintf(@topic_id),nil,data)
+        @producer.send_msg(event.sprintf(@topic_id),@partition_key,data)
       rescue LogStash::ShutdownSignal
         @logger.info('Kafka producer got shutdown signal')
       rescue => e
@@ -149,9 +159,11 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
       finished
       return
     end
+    @partition_key = if @partition_key_format.nil? then nil else event.sprintf(@partition_key_format) end
     @codec.encode(event)
+    @partition_key = nil
   end
-
+    
   def teardown
     @producer.close
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-kafka
 version: !ruby/object:Gem::Version
-  version: 0.7.3
+  version: 0.7.4
 platform: java
 authors:
 - Joseph Lawson
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-02-19 00:00:00.000000000 Z
+date: 2015-02-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement