logstash-integration-kafka 10.0.1-java → 10.1.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c46d0081cceeb47ad2a3718ab295b38350edbedce2448d6423257b9696d5f4ff
-  data.tar.gz: 258b4f5bd38c5e4acfa848a23292dd54db45b0bc77dbf473821eb4caa228294c
+  metadata.gz: 254abccf066d63d45cf0660dafa06b603c97fb5557c1f978ecc41b41078c6ead
+  data.tar.gz: a6bcb799f703db46d80a4412f35809b7a7d13bcdf8eaf91e12ef06befc700a93
 SHA512:
-  metadata.gz: a8ee6bb607a1636544679a647f1ed20d0ed5d022dcad5af08297b016cd19dc5a01251b2a00e9c5fe984df6bb5c4d9ec831113e94ce3a6c84503caaf966d12491
-  data.tar.gz: 8d315bdf003dbb5b4c9bd0af488426c3f8a84d5aebf9084728fda9f62ac2f8a269c563750f4be5cf4ab51befb372c80f4d91717396c9a8e9a3682e2c314ecbee
+  metadata.gz: 9551a410f21e1015e56ebd2d2881d75c1eb3d13e5a3aff609e98ac4111376764bcc1978612bb9b239b2335757e129897a94be69bac02d996dfbf31d50ffc9614
+  data.tar.gz: 106b3fa2816631035f132a4771f4b5776fd9a79811305200d37a1b539a3ba1600079e8b9d9c1ec718ff9a2cecdea9115fb31b2ccd425edfe9097114da7ca752f

data/CHANGELOG.md

@@ -1,7 +1,16 @@
+## 10.1.0
+  - updated kafka client (and its dependencies) to version 2.4.1 ([#16](https://github.com/logstash-plugins/logstash-integration-kafka/pull/16))
+  - added the input `client_rack` parameter to enable support for follower fetching
+  - added the output `partitioner` parameter for tuning partitioning strategy
+  - Refactor: normalized error logging a bit - make sure exception type is logged
+  - Fix: properly handle empty ssl_endpoint_identification_algorithm [#8](https://github.com/logstash-plugins/logstash-integration-kafka/pull/8)
+  - Refactor : made `partition_assignment_strategy` option easier to configure by accepting simple values from an enumerated set instead of requiring lengthy class paths ([#25](https://github.com/logstash-plugins/logstash-integration-kafka/pull/25))
+
 ## 10.0.1
   - Fix links in changelog pointing to stand-alone plugin changelogs.
   - Refactor: scope java_import to plugin class
 
+
 ## 10.0.0
   - Initial release of the Kafka Integration Plugin, which combines
     previously-separate Kafka plugins and shared dependencies into a single

data/CONTRIBUTORS

@@ -11,6 +11,7 @@ Contributors:
 * João Duarte (jsvd)
 * Kurt Hurtado (kurtado)
 * Ry Biesemeyer (yaauie)
+* Rob Cowart (robcowart)
 
 Note: If you've sent us patches, bug reports, or otherwise contributed to
 Logstash, and you aren't on the list above and want to be, please let us know

data/LICENSE

@@ -1,13 +1,202 @@
-Copyright (c) 2012-2018 Elasticsearch <http://www.elastic.co>
 
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
 
-    http://www.apache.org/licenses/LICENSE-2.0
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
 
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2020 Elastic and contributors
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

data/docs/input-kafka.asciidoc

@@ -71,8 +71,7 @@ inserted into your original event, you'll have to use the `mutate` filter to man
 
 This plugin supports these configuration options plus the <<plugins-{type}s-{plugin}-common-options>> described later.
 
-NOTE: Some of these options map to a Kafka option. See the
-https://kafka.apache.org/documentation for more details.
+NOTE: Some of these options map to a Kafka option. See the https://kafka.apache.org/documentation for more details.
 
 [cols="<,<,<",options="header",]
 |=======================================================================
@@ -122,6 +121,7 @@ https://kafka.apache.org/documentation for more details.
 | <<plugins-{type}s-{plugin}-topics>> |<<array,array>>|No
 | <<plugins-{type}s-{plugin}-topics_pattern>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-value_deserializer_class>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-client_rack>> |<<string,string>>|No
 |=======================================================================
 
 Also see <<plugins-{type}s-{plugin}-common-options>> for a list of options supported by all
@@ -374,10 +374,16 @@ we haven't seen any partition leadership changes to proactively discover any new
   * Value type is <<string,string>>
   * There is no default value for this setting.
 
-The class name of the partition assignment strategy that the client uses to
-distribute partition ownership amongst consumer instances. Maps to
-the Kafka `partition.assignment.strategy` setting, which defaults to
-`org.apache.kafka.clients.consumer.RangeAssignor`.
+The name of the partition assignment strategy that the client uses to distribute
+partition ownership amongst consumer instances, supported options are:
+
+* `range`
+* `round_robin`
+* `sticky`
+* `cooperative_sticky`
+
+These map to Kafka's corresponding https://kafka.apache.org/24/javadoc/org/apache/kafka/clients/consumer/ConsumerPartitionAssignor.html[`ConsumerPartitionAssignor`]
+implementations.
 
 [id="plugins-{type}s-{plugin}-poll_timeout_ms"]
 ===== `poll_timeout_ms` 
@@ -578,6 +584,17 @@ The topics configuration will be ignored when using this configuration.
 Java Class used to deserialize the record's value
 
 
+[id="plugins-{type}s-{plugin}-client_rack"]
+===== `client_rack` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+A rack identifier for the Kafka consumer. 
+Used to select the physically closest rack for the consumer to read from.
+The setting corresponds with Kafka's `broker.rack` configuration.
+
+NOTE: Only available for Kafka 2.4.0 and higher; see https://cwiki.apache.org/confluence/display/KAFKA/KIP-392%3A+Allow+consumers+to+fetch+from+closest+replica[KIP-392].
 
 [id="plugins-{type}s-{plugin}-common-options"]
 include::{include_path}/{type}.asciidoc[]

data/docs/output-kafka.asciidoc

@@ -73,6 +73,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-message_key>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-metadata_fetch_timeout_ms>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-metadata_max_age_ms>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-partitioner>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-receive_buffer_bytes>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-reconnect_backoff_ms>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-request_timeout_ms>> |<<string,string>>|No
@@ -248,6 +249,21 @@ the timeout setting for initial metadata request to fetch topic metadata.
 
 the max time in milliseconds before a metadata refresh is forced.
 
+[id="plugins-{type}s-{plugin}-partitioner"]
+===== `partitioner`
+
+* Value type is <<string,string>>
+* There is no default value for this setting.
+
+The default behavior is to hash the `message_key` of an event to get the partition.
+When no message key is present, the plugin picks a partition in a round-robin fashion.
+
+Available options for choosing a partitioning strategy are as follows:
+
+* `default` use the default partitioner as described above
+* `round_robin` distributes writes to all partitions equally, regardless of `message_key`
+* `uniform_sticky` sticks to a partition for the duration of a batch than randomly picks a new one
+
 [id="plugins-{type}s-{plugin}-receive_buffer_bytes"]
 ===== `receive_buffer_bytes` 
 

data/lib/logstash-integration-kafka_jars.rb

@@ -1,8 +1,8 @@
 # AUTOGENERATED BY THE GRADLE SCRIPT. DO NOT EDIT.
 
 require 'jar_dependencies'
-require_jar('org.apache.kafka', 'kafka-clients', '2.3.0')
-require_jar('com.github.luben', 'zstd-jni', '1.4.2-1')
-require_jar('org.slf4j', 'slf4j-api', '1.7.26')
+require_jar('org.apache.kafka', 'kafka-clients', '2.4.1')
+require_jar('com.github.luben', 'zstd-jni', '1.4.3-1')
+require_jar('org.slf4j', 'slf4j-api', '1.7.28')
 require_jar('org.lz4', 'lz4-java', '1.6.0')
 require_jar('org.xerial.snappy', 'snappy-java', '1.1.7.3')

data/lib/logstash/inputs/kafka.rb

@@ -128,8 +128,10 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
   # The period of time in milliseconds after which we force a refresh of metadata even if
   # we haven't seen any partition leadership changes to proactively discover any new brokers or partitions
   config :metadata_max_age_ms, :validate => :string
-  # The class name of the partition assignment strategy that the client will use to distribute
-  # partition ownership amongst consumer instances
+  # The name of the partition assignment strategy that the client uses to distribute
+  # partition ownership amongst consumer instances, supported options are `range`,
+  # `round_robin`, `sticky` and `cooperative_sticky`
+  # (for backwards compatibility setting the class name directly is supported).
   config :partition_assignment_strategy, :validate => :string
   # The size of the TCP receive buffer (SO_RCVBUF) to use when reading data.
   config :receive_buffer_bytes, :validate => :string
@@ -159,6 +161,11 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
   config :topics_pattern, :validate => :string
   # Time kafka consumer will wait to receive new messages from topics
   config :poll_timeout_ms, :validate => :number, :default => 100
+  # The rack id string to pass to the server when making requests. This is used 
+  # as a selector for a rack, region, or datacenter. Corresponds to the broker.rack parameter
+  # in the broker configuration. 
+  # Only has an effect in combination with brokers with Kafka 2.4+ with the broker.rack setting. Ignored otherwise. 
+  config :client_rack, :validate => :string
   # The truststore type.
   config :ssl_truststore_type, :validate => :string
   # The JKS truststore path to validate the Kafka broker's certificate.
@@ -305,7 +312,7 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
       props.put(kafka::MAX_POLL_RECORDS_CONFIG, max_poll_records) unless max_poll_records.nil?
       props.put(kafka::MAX_POLL_INTERVAL_MS_CONFIG, max_poll_interval_ms) unless max_poll_interval_ms.nil?
       props.put(kafka::METADATA_MAX_AGE_CONFIG, metadata_max_age_ms) unless metadata_max_age_ms.nil?
-      props.put(kafka::PARTITION_ASSIGNMENT_STRATEGY_CONFIG, partition_assignment_strategy) unless partition_assignment_strategy.nil?
+      props.put(kafka::PARTITION_ASSIGNMENT_STRATEGY_CONFIG, partition_assignment_strategy_class) unless partition_assignment_strategy.nil?
       props.put(kafka::RECEIVE_BUFFER_CONFIG, receive_buffer_bytes) unless receive_buffer_bytes.nil?
       props.put(kafka::RECONNECT_BACKOFF_MS_CONFIG, reconnect_backoff_ms) unless reconnect_backoff_ms.nil?
       props.put(kafka::REQUEST_TIMEOUT_MS_CONFIG, request_timeout_ms) unless request_timeout_ms.nil?
@@ -313,6 +320,7 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
       props.put(kafka::SEND_BUFFER_CONFIG, send_buffer_bytes) unless send_buffer_bytes.nil?
       props.put(kafka::SESSION_TIMEOUT_MS_CONFIG, session_timeout_ms) unless session_timeout_ms.nil?
       props.put(kafka::VALUE_DESERIALIZER_CLASS_CONFIG, value_deserializer_class)
+      props.put(kafka::CLIENT_RACK_CONFIG, client_rack) unless client_rack.nil? 
 
       props.put("security.protocol", security_protocol) unless security_protocol.nil?
 
@@ -334,6 +342,24 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
     end
   end
 
+  def partition_assignment_strategy_class
+    case partition_assignment_strategy
+    when 'range'
+      'org.apache.kafka.clients.consumer.RangeAssignor'
+    when 'round_robin'
+      'org.apache.kafka.clients.consumer.RoundRobinAssignor'
+    when 'sticky'
+      'org.apache.kafka.clients.consumer.StickyAssignor'
+    when 'cooperative_sticky'
+      'org.apache.kafka.clients.consumer.CooperativeStickyAssignor'
+    else
+      unless partition_assignment_strategy.index('.')
+        raise LogStash::ConfigurationError, "unsupported partition_assignment_strategy: #{partition_assignment_strategy.inspect}"
+      end
+      partition_assignment_strategy # assume a fully qualified class-name
+    end
+  end
+
   def set_trustore_keystore_config(props)
     props.put("ssl.truststore.type", ssl_truststore_type) unless ssl_truststore_type.nil?
     props.put("ssl.truststore.location", ssl_truststore_location) unless ssl_truststore_location.nil?

data/lib/logstash/outputs/kafka.rb

@@ -101,6 +101,8 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
   config :metadata_fetch_timeout_ms, :validate => :number, :default => 60000
   # the max time in milliseconds before a metadata refresh is forced.
   config :metadata_max_age_ms, :validate => :number, :default => 300000
+  # Partitioner to use - can be `default`, `uniform_sticky`, `round_robin` or a fully qualified class name of a custom partitioner.
+  config :partitioner, :validate => :string
   # The size of the TCP receive buffer to use when reading data
   config :receive_buffer_bytes, :validate => :number, :default => 32768
   # The amount of time to wait before attempting to reconnect to a given host when a connection fails.
@@ -184,7 +186,7 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
         raise ConfigurationError, "A negative retry count (#{@retries}) is not valid. Must be a value >= 0"
       end
 
-      @logger.warn("Kafka output is configured with finite retry. This instructs Logstash to LOSE DATA after a set number of send attempts fails. If you do not want to lose data if Kafka is down, then you must remove the retry setting.", :retries => @retries)
+      logger.warn("Kafka output is configured with finite retry. This instructs Logstash to LOSE DATA after a set number of send attempts fails. If you do not want to lose data if Kafka is down, then you must remove the retry setting.", :retries => @retries)
     end
 
 
@@ -202,8 +204,6 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
     end
   end
 
-  # def register
-
   def prepare(record)
     # This output is threadsafe, so we need to keep a batch per thread.
     @thread_batch_map[Thread.current].add(record)
@@ -269,7 +269,7 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
           result = future.get()
         rescue => e
           # TODO(sissel): Add metric to count failures, possibly by exception type.
-          logger.warn("KafkaProducer.send() failed: #{e}", :exception => e)
+          logger.warn("producer send failed", :exception => e.class, :message => e.message)
           failures << batch[i]
         end
       end
@@ -303,10 +303,9 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
     end
     prepare(record)
   rescue LogStash::ShutdownSignal
-    @logger.debug('Kafka producer got shutdown signal')
+    logger.debug('producer received shutdown signal')
   rescue => e
-    @logger.warn('kafka producer threw exception, restarting',
-                 :exception => e)
+    logger.warn('producer threw exception, restarting', :exception => e.class, :message => e.message)
   end
 
   def create_producer
@@ -324,6 +323,10 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
       props.put(kafka::LINGER_MS_CONFIG, linger_ms.to_s)
       props.put(kafka::MAX_REQUEST_SIZE_CONFIG, max_request_size.to_s)
       props.put(kafka::METADATA_MAX_AGE_CONFIG, metadata_max_age_ms) unless metadata_max_age_ms.nil?
+      unless partitioner.nil?
+        props.put(kafka::PARTITIONER_CLASS_CONFIG, partitioner = partitioner_class)
+        logger.debug('producer configured using partitioner', :partitioner_class => partitioner)
+      end
       props.put(kafka::RECEIVE_BUFFER_CONFIG, receive_buffer_bytes.to_s) unless receive_buffer_bytes.nil?
       props.put(kafka::RECONNECT_BACKOFF_MS_CONFIG, reconnect_backoff_ms) unless reconnect_backoff_ms.nil?
       props.put(kafka::REQUEST_TIMEOUT_MS_CONFIG, request_timeout_ms) unless request_timeout_ms.nil?
@@ -353,13 +356,31 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
     end
   end
 
+  def partitioner_class
+    case partitioner
+    when 'round_robin'
+      'org.apache.kafka.clients.producer.RoundRobinPartitioner'
+    when 'uniform_sticky'
+      'org.apache.kafka.clients.producer.UniformStickyPartitioner'
+    when 'default'
+      'org.apache.kafka.clients.producer.internals.DefaultPartitioner'
+    else
+      unless partitioner.index('.')
+        raise LogStash::ConfigurationError, "unsupported partitioner: #{partitioner.inspect}"
+      end
+      partitioner # assume a fully qualified class-name
+    end
+  end
+
   def set_trustore_keystore_config(props)
-    if ssl_truststore_location.nil?
-      raise LogStash::ConfigurationError, "ssl_truststore_location must be set when SSL is enabled"
+    unless ssl_endpoint_identification_algorithm.to_s.strip.empty?
+      if ssl_truststore_location.nil?
+        raise LogStash::ConfigurationError, "ssl_truststore_location must be set when SSL is enabled"
+      end
+      props.put("ssl.truststore.type", ssl_truststore_type) unless ssl_truststore_type.nil?
+      props.put("ssl.truststore.location", ssl_truststore_location)
+      props.put("ssl.truststore.password", ssl_truststore_password.value) unless ssl_truststore_password.nil?
     end
-    props.put("ssl.truststore.type", ssl_truststore_type) unless ssl_truststore_type.nil?
-    props.put("ssl.truststore.location", ssl_truststore_location)
-    props.put("ssl.truststore.password", ssl_truststore_password.value) unless ssl_truststore_password.nil?
 
     # Client auth stuff
     props.put("ssl.keystore.type", ssl_keystore_type) unless ssl_keystore_type.nil?
@@ -370,15 +391,15 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
   end
 
   def set_sasl_config(props)
-    java.lang.System.setProperty("java.security.auth.login.config",jaas_path) unless jaas_path.nil?
-    java.lang.System.setProperty("java.security.krb5.conf",kerberos_config) unless kerberos_config.nil?
+    java.lang.System.setProperty("java.security.auth.login.config", jaas_path) unless jaas_path.nil?
+    java.lang.System.setProperty("java.security.krb5.conf", kerberos_config) unless kerberos_config.nil?
 
     props.put("sasl.mechanism",sasl_mechanism)
     if sasl_mechanism == "GSSAPI" && sasl_kerberos_service_name.nil?
       raise LogStash::ConfigurationError, "sasl_kerberos_service_name must be specified when SASL mechanism is GSSAPI"
     end
 
-    props.put("sasl.kerberos.service.name",sasl_kerberos_service_name) unless sasl_kerberos_service_name.nil?
+    props.put("sasl.kerberos.service.name", sasl_kerberos_service_name) unless sasl_kerberos_service_name.nil?
     props.put("sasl.jaas.config", sasl_jaas_config) unless sasl_jaas_config.nil?
   end
 

data/logstash-integration-kafka.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-integration-kafka'
-  s.version         = '10.0.1'
+  s.version         = '10.1.0'
   s.licenses        = ['Apache-2.0']
   s.summary         = "Integration with Kafka - input and output plugins"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline "+
@@ -49,6 +49,6 @@ Gem::Specification.new do |s|
 
   s.add_development_dependency 'logstash-devutils'
   s.add_development_dependency 'rspec-wait'
-  s.add_development_dependency 'poseidon'
+  s.add_development_dependency 'ruby-kafka'
   s.add_development_dependency 'snappy'
 end

data/spec/integration/inputs/kafka_spec.rb

@@ -1,7 +1,6 @@
 # encoding: utf-8
 require "logstash/devutils/rspec/spec_helper"
 require "logstash/inputs/kafka"
-require "digest"
 require "rspec/wait"
 
 # Please run kafka_test_setup.sh prior to executing this integration test.
@@ -12,159 +11,129 @@ describe "inputs/kafka", :integration => true do
   let(:group_id_3) {rand(36**8).to_s(36)}
   let(:group_id_4) {rand(36**8).to_s(36)}
   let(:group_id_5) {rand(36**8).to_s(36)}
-  let(:plain_config) { { 'topics' => ['logstash_integration_topic_plain'], 'codec' => 'plain', 'group_id' => group_id_1, 'auto_offset_reset' => 'earliest'} }
-  let(:multi_consumer_config) { plain_config.merge({"group_id" => group_id_4, "client_id" => "spec", "consumer_threads" => 3}) }
-  let(:snappy_config) { { 'topics' => ['logstash_integration_topic_snappy'], 'codec' => 'plain', 'group_id' => group_id_1, 'auto_offset_reset' => 'earliest'} }
-  let(:lz4_config) { { 'topics' => ['logstash_integration_topic_lz4'], 'codec' => 'plain', 'group_id' => group_id_1, 'auto_offset_reset' => 'earliest'} }
-  let(:pattern_config) { { 'topics_pattern' => 'logstash_integration_topic_.*', 'group_id' => group_id_2, 'codec' => 'plain', 'auto_offset_reset' => 'earliest'} }
-  let(:decorate_config) { { 'topics' => ['logstash_integration_topic_plain'], 'codec' => 'plain', 'group_id' => group_id_3, 'auto_offset_reset' => 'earliest', 'decorate_events' => true} }
-  let(:manual_commit_config) { { 'topics' => ['logstash_integration_topic_plain'], 'codec' => 'plain', 'group_id' => group_id_5, 'auto_offset_reset' => 'earliest', 'enable_auto_commit' => 'false'} }
+  let(:group_id_6) {rand(36**8).to_s(36)}
+  let(:plain_config) do
+    { 'topics' => ['logstash_integration_topic_plain'], 'codec' => 'plain', 'group_id' => group_id_1,
+      'auto_offset_reset' => 'earliest' }
+  end
+  let(:multi_consumer_config) do
+    plain_config.merge({"group_id" => group_id_4, "client_id" => "spec", "consumer_threads" => 3})
+  end
+  let(:snappy_config) do
+    { 'topics' => ['logstash_integration_topic_snappy'], 'codec' => 'plain', 'group_id' => group_id_1,
+      'auto_offset_reset' => 'earliest' }
+  end
+  let(:lz4_config) do
+    { 'topics' => ['logstash_integration_topic_lz4'], 'codec' => 'plain', 'group_id' => group_id_1,
+      'auto_offset_reset' => 'earliest' }
+  end
+  let(:pattern_config) do
+    { 'topics_pattern' => 'logstash_integration_topic_.*', 'group_id' => group_id_2, 'codec' => 'plain',
+      'auto_offset_reset' => 'earliest' }
+  end
+  let(:decorate_config) do
+    { 'topics' => ['logstash_integration_topic_plain'], 'codec' => 'plain', 'group_id' => group_id_3,
+      'auto_offset_reset' => 'earliest', 'decorate_events' => true }
+  end
+  let(:manual_commit_config) do
+    { 'topics' => ['logstash_integration_topic_plain'], 'codec' => 'plain', 'group_id' => group_id_5,
+      'auto_offset_reset' => 'earliest', 'enable_auto_commit' => 'false' }
+  end
   let(:timeout_seconds) { 30 }
   let(:num_events) { 103 }
 
   describe "#kafka-topics" do
-    def thread_it(kafka_input, queue)
-      Thread.new do
-        begin
-          kafka_input.run(queue)
-        end
-      end
-    end
 
     it "should consume all messages from plain 3-partition topic" do
-      kafka_input = LogStash::Inputs::Kafka.new(plain_config)
-      queue = Queue.new
-      t = thread_it(kafka_input, queue)
-      begin
-        t.run
-        wait(timeout_seconds).for {queue.length}.to eq(num_events)
-        expect(queue.length).to eq(num_events)
-      ensure
-        t.kill
-        t.join(30_000)
-      end
+      queue = consume_messages(plain_config, timeout: timeout_seconds, event_count: num_events)
+      expect(queue.length).to eq(num_events)
     end
 
     it "should consume all messages from snappy 3-partition topic" do
-      kafka_input = LogStash::Inputs::Kafka.new(snappy_config)
-      queue = Queue.new
-      t = thread_it(kafka_input, queue)
-      begin
-        t.run
-        wait(timeout_seconds).for {queue.length}.to eq(num_events)
-        expect(queue.length).to eq(num_events)
-      ensure
-        t.kill
-        t.join(30_000)
-      end
+      queue = consume_messages(snappy_config, timeout: timeout_seconds, event_count: num_events)
+      expect(queue.length).to eq(num_events)
     end
 
     it "should consume all messages from lz4 3-partition topic" do
-      kafka_input = LogStash::Inputs::Kafka.new(lz4_config)
-      queue = Queue.new
-      t = thread_it(kafka_input, queue)
-      begin
-        t.run
-        wait(timeout_seconds).for {queue.length}.to eq(num_events)
-        expect(queue.length).to eq(num_events)
-      ensure
-        t.kill
-        t.join(30_000)
-      end
+      queue = consume_messages(lz4_config, timeout: timeout_seconds, event_count: num_events)
+      expect(queue.length).to eq(num_events)
     end
 
     it "should consumer all messages with multiple consumers" do
-      kafka_input = LogStash::Inputs::Kafka.new(multi_consumer_config)
-      queue = Queue.new
-      t = thread_it(kafka_input, queue)
-      begin
-        t.run
-        wait(timeout_seconds).for {queue.length}.to eq(num_events)
+      consume_messages(multi_consumer_config, timeout: timeout_seconds, event_count: num_events) do |queue, kafka_input|
         expect(queue.length).to eq(num_events)
         kafka_input.kafka_consumers.each_with_index do |consumer, i|
           expect(consumer.metrics.keys.first.tags["client-id"]).to eq("spec-#{i}")
         end
-      ensure
-        t.kill
-        t.join(30_000)
       end
     end
   end
 
-  describe "#kafka-topics-pattern" do
-    def thread_it(kafka_input, queue)
-      Thread.new do
-        begin
-          kafka_input.run(queue)
-        end
-      end
-    end
-
+  context "#kafka-topics-pattern" do
     it "should consume all messages from all 3 topics" do
-      kafka_input = LogStash::Inputs::Kafka.new(pattern_config)
-      queue = Queue.new
-      t = thread_it(kafka_input, queue)
-      begin
-        t.run
-        wait(timeout_seconds).for {queue.length}.to eq(3*num_events)
-        expect(queue.length).to eq(3*num_events)
-      ensure
-        t.kill
-        t.join(30_000)
-      end
+      total_events = num_events * 3
+      queue = consume_messages(pattern_config, timeout: timeout_seconds, event_count: total_events)
+      expect(queue.length).to eq(total_events)
     end
   end
 
-  describe "#kafka-decorate" do
-    def thread_it(kafka_input, queue)
-      Thread.new do
-        begin
-          kafka_input.run(queue)
-        end
-      end
-    end
-
+  context "#kafka-decorate" do
     it "should show the right topic and group name in decorated kafka section" do
       start = LogStash::Timestamp.now.time.to_i
-      kafka_input = LogStash::Inputs::Kafka.new(decorate_config)
-      queue = Queue.new
-      t = thread_it(kafka_input, queue)
-      begin
-        t.run
-        wait(timeout_seconds).for {queue.length}.to eq(num_events)
+      consume_messages(decorate_config, timeout: timeout_seconds, event_count: num_events) do |queue, _|
         expect(queue.length).to eq(num_events)
         event = queue.shift
         expect(event.get("[@metadata][kafka][topic]")).to eq("logstash_integration_topic_plain")
         expect(event.get("[@metadata][kafka][consumer_group]")).to eq(group_id_3)
         expect(event.get("[@metadata][kafka][timestamp]")).to be >= start
-      ensure
-        t.kill
-        t.join(30_000)
       end
     end
   end
 
-  describe "#kafka-offset-commit" do
-    def thread_it(kafka_input, queue)
-      Thread.new do
-        begin
-          kafka_input.run(queue)
+  context "#kafka-offset-commit" do
+    it "should manually commit offsets" do
+      queue = consume_messages(manual_commit_config, timeout: timeout_seconds, event_count: num_events)
+      expect(queue.length).to eq(num_events)
+    end
+  end
+
+  context 'setting partition_assignment_strategy' do
+    let(:test_topic) { 'logstash_integration_partitioner_topic' }
+    let(:consumer_config) do
+      plain_config.merge(
+          "topics" => [test_topic],
+          'group_id' => group_id_6,
+          "client_id" => "partition_assignment_strategy-spec",
+          "consumer_threads" => 2,
+          "partition_assignment_strategy" => partition_assignment_strategy
+      )
+    end
+    let(:partition_assignment_strategy) { nil }
+
+    # NOTE: just verify setting works, as its a bit cumbersome to do in a unit spec
+    [ 'range', 'round_robin', 'sticky', 'org.apache.kafka.clients.consumer.CooperativeStickyAssignor' ].each do |partition_assignment_strategy|
+      describe partition_assignment_strategy do
+        let(:partition_assignment_strategy) { partition_assignment_strategy }
+        it 'consumes data' do
+          consume_messages(consumer_config, timeout: false, event_count: 0)
         end
       end
     end
+  end
 
-    it "should manually commit offsets" do
-      kafka_input = LogStash::Inputs::Kafka.new(manual_commit_config)
-      queue = Queue.new
-      t = thread_it(kafka_input, queue)
-      begin
-        t.run
-        wait(timeout_seconds).for {queue.length}.to eq(num_events)
-        expect(queue.length).to eq(num_events)
-      ensure
-        t.kill
-        t.join(30_000)
-      end
+  private
+
+  def consume_messages(config, queue: Queue.new, timeout:, event_count:)
+    kafka_input = LogStash::Inputs::Kafka.new(config)
+    t = Thread.new { kafka_input.run(queue) }
+    begin
+      t.run
+      wait(timeout).for { queue.length }.to eq(event_count) unless timeout.eql?(false)
+      block_given? ? yield(queue, kafka_input) : queue
+    ensure
+      t.kill
+      t.join(30_000)
     end
   end
+
 end

data/spec/integration/outputs/kafka_spec.rb

@@ -3,26 +3,29 @@
 require "logstash/devutils/rspec/spec_helper"
 require 'logstash/outputs/kafka'
 require 'json'
-require 'poseidon'
+require 'kafka'
 
 describe "outputs/kafka", :integration => true do
   let(:kafka_host) { 'localhost' }
   let(:kafka_port) { 9092 }
   let(:num_events) { 10 }
+
   let(:base_config) { {'client_id' => 'kafkaoutputspec'} }
-  let(:event) { LogStash::Event.new({'message' => '183.60.215.50 - - [11/Sep/2014:22:00:00 +0000] "GET /scripts/netcat-webserver HTTP/1.1" 200 182 "-" "Mozilla/5.0 (compatible; EasouSpider; +http://www.easou.com/search/spider.html)"', '@timestamp' => LogStash::Timestamp.at(0) }) }
+  let(:message_content) do
+    '"GET /scripts/netcat-webserver HTTP/1.1" 200 182 "-" "Mozilla/5.0 (compatible; EasouSpider; +http://www.easou.com/search/spider.html)"'
+  end
+  let(:event) do
+    LogStash::Event.new({ 'message' =>
+      '183.60.215.50 - - [11/Sep/2014:22:00:00 +0000] ' + message_content,
+      '@timestamp' => LogStash::Timestamp.at(0)
+    })
+  end
 
+  let(:kafka_client) { Kafka.new ["#{kafka_host}:#{kafka_port}"] }
 
   context 'when outputting messages serialized as String' do
     let(:test_topic) { 'logstash_integration_topic1' }
     let(:num_events) { 3 }
-    let(:consumer) do
-      Poseidon::PartitionConsumer.new("my_test_consumer", kafka_host, kafka_port,
-                                      test_topic, 0, :earliest_offset)
-    end
-    subject do
-      consumer.fetch
-    end
 
     before :each do
       config = base_config.merge({"topic_id" => test_topic})
@@ -30,8 +33,10 @@ describe "outputs/kafka", :integration => true do
     end
 
     it 'should have data integrity' do
-      expect(subject.size).to eq(num_events)
-      subject.each do |m|
+      messages = fetch_messages(test_topic)
+
+      expect(messages.size).to eq(num_events)
+      messages.each do |m|
         expect(m.value).to eq(event.to_s)
       end
     end
@@ -41,13 +46,6 @@ describe "outputs/kafka", :integration => true do
   context 'when outputting messages serialized as Byte Array' do
     let(:test_topic) { 'topic1b' }
     let(:num_events) { 3 }
-    let(:consumer) do
-      Poseidon::PartitionConsumer.new("my_test_consumer", kafka_host, kafka_port,
-                                      test_topic, 0, :earliest_offset)
-    end
-    subject do
-      consumer.fetch
-    end
 
     before :each do
       config = base_config.merge(
@@ -60,8 +58,10 @@ describe "outputs/kafka", :integration => true do
     end
 
     it 'should have data integrity' do
-      expect(subject.size).to eq(num_events)
-      subject.each do |m|
+      messages = fetch_messages(test_topic)
+
+      expect(messages.size).to eq(num_events)
+      messages.each do |m|
         expect(m.value).to eq(event.to_s)
       end
     end
@@ -71,14 +71,6 @@ describe "outputs/kafka", :integration => true do
   context 'when setting message_key' do
     let(:num_events) { 10 }
     let(:test_topic) { 'logstash_integration_topic2' }
-    let!(:consumer0) do
-      Poseidon::PartitionConsumer.new("my_test_consumer", kafka_host, kafka_port,
-                                      test_topic, 0, :earliest_offset)
-    end
-    let!(:consumer1) do
-      Poseidon::PartitionConsumer.new("my_test_consumer", kafka_host, kafka_port,
-                                      test_topic, 1, :earliest_offset)
-    end
 
     before :each do
       config = base_config.merge({"topic_id" => test_topic, "message_key" => "static_key"})
@@ -86,19 +78,14 @@ describe "outputs/kafka", :integration => true do
     end
 
     it 'should send all events to one partition' do
-      expect(consumer0.fetch.size == num_events || consumer1.fetch.size == num_events).to be true
+      data0 = fetch_messages(test_topic, partition: 0)
+      data1 = fetch_messages(test_topic, partition: 1)
+      expect(data0.size == num_events || data1.size == num_events).to be true
     end
   end
 
   context 'when using gzip compression' do
     let(:test_topic) { 'logstash_integration_gzip_topic' }
-    let!(:consumer) do
-      Poseidon::PartitionConsumer.new("my_test_consumer", kafka_host, kafka_port,
-                                      test_topic, 0, :earliest_offset)
-    end
-    subject do
-      consumer.fetch
-    end
 
     before :each do
       config = base_config.merge({"topic_id" => test_topic, "compression_type" => "gzip"})
@@ -106,8 +93,10 @@ describe "outputs/kafka", :integration => true do
     end
 
     it 'should have data integrity' do
-      expect(subject.size).to eq(num_events)
-      subject.each do |m|
+      messages = fetch_messages(test_topic)
+
+      expect(messages.size).to eq(num_events)
+      messages.each do |m|
         expect(m.value).to eq(event.to_s)
       end
     end
@@ -115,13 +104,6 @@ describe "outputs/kafka", :integration => true do
 
   context 'when using snappy compression' do
     let(:test_topic) { 'logstash_integration_snappy_topic' }
-    let!(:consumer) do
-      Poseidon::PartitionConsumer.new("my_test_consumer", kafka_host, kafka_port,
-                                      test_topic, 0, :earliest_offset)
-    end
-    subject do
-      consumer.fetch
-    end
 
     before :each do
       config = base_config.merge({"topic_id" => test_topic, "compression_type" => "snappy"})
@@ -129,8 +111,10 @@ describe "outputs/kafka", :integration => true do
     end
 
     it 'should have data integrity' do
-      expect(subject.size).to eq(num_events)
-      subject.each do |m|
+      messages = fetch_messages(test_topic)
+
+      expect(messages.size).to eq(num_events)
+      messages.each do |m|
         expect(m.value).to eq(event.to_s)
       end
     end
@@ -143,52 +127,85 @@ describe "outputs/kafka", :integration => true do
       config = base_config.merge({"topic_id" => test_topic, "compression_type" => "lz4"})
       load_kafka_data(config)
     end
+
+    # NOTE: depends on extlz4 gem which is using a C-extension
+    # it 'should have data integrity' do
+    #   messages = fetch_messages(test_topic)
+    #
+    #   expect(messages.size).to eq(num_events)
+    #   messages.each do |m|
+    #     expect(m.value).to eq(event.to_s)
+    #   end
+    # end
   end
 
   context 'when using multi partition topic' do
-    let(:num_events) { 10 }
+    let(:num_events) { 100 } # ~ more than (batch.size) 16,384 bytes
     let(:test_topic) { 'logstash_integration_topic3' }
-    let!(:consumer0) do
-      Poseidon::PartitionConsumer.new("my_test_consumer", kafka_host, kafka_port,
-                                      test_topic, 0, :earliest_offset)
-    end
-    let!(:consumer1) do
-      Poseidon::PartitionConsumer.new("my_test_consumer", kafka_host, kafka_port,
-                                      test_topic, 1, :earliest_offset)
+
+    before :each do
+      config = base_config.merge("topic_id" => test_topic, "partitioner" => 'org.apache.kafka.clients.producer.UniformStickyPartitioner')
+      load_kafka_data(config) do # let's have a bit more (diverse) dataset
+        num_events.times.collect do
+          LogStash::Event.new.tap do |e|
+            e.set('message', event.get('message').sub('183.60.215.50') { "#{rand(126)+1}.#{rand(126)+1}.#{rand(126)+1}.#{rand(126)+1}" })
+          end
+        end
+      end
     end
 
-    let!(:consumer2) do
-      Poseidon::PartitionConsumer.new("my_test_consumer", kafka_host, kafka_port,
-                                      test_topic, 2, :earliest_offset)
+    it 'should distribute events to all partitions' do
+      consumer0_records = fetch_messages(test_topic, partition: 0)
+      consumer1_records = fetch_messages(test_topic, partition: 1)
+      consumer2_records = fetch_messages(test_topic, partition: 2)
+
+      all_records = consumer0_records + consumer1_records + consumer2_records
+      expect(all_records.size).to eq(num_events * 2)
+      all_records.each do |m|
+        expect(m.value).to include message_content
+      end
+
+      expect(consumer0_records.size).to be > 1
+      expect(consumer1_records.size).to be > 1
+      expect(consumer2_records.size).to be > 1
     end
+  end
+
+  context 'setting partitioner' do
+    let(:test_topic) { 'logstash_integration_partitioner_topic' }
+    let(:partitioner) { nil }
 
     before :each do
-      config = base_config.merge({"topic_id" => test_topic})
+      @messages_offset = fetch_messages_from_all_partitions
+
+      config = base_config.merge("topic_id" => test_topic, 'partitioner' => partitioner)
       load_kafka_data(config)
     end
 
-    it 'should distribute events to all partition' do
-      consumer0_records = consumer0.fetch
-      consumer1_records = consumer1.fetch
-      consumer2_records = consumer2.fetch
-
-      expect(consumer0_records.size > 1 &&
-        consumer1_records.size > 1 &&
-          consumer2_records.size > 1).to be true
-
-      all_records = consumer0_records + consumer1_records + consumer2_records
-      expect(all_records.size).to eq(num_events)
-      all_records.each do |m|
-        expect(m.value).to eq(event.to_s)
+    [ 'default', 'round_robin', 'uniform_sticky' ].each do |partitioner|
+      describe partitioner do
+        let(:partitioner) { partitioner }
+        it 'loads data' do
+          expect(fetch_messages_from_all_partitions - @messages_offset).to eql num_events
+        end
       end
     end
+
+    def fetch_messages_from_all_partitions
+      3.times.map { |i| fetch_messages(test_topic, partition: i).size }.sum
+    end
   end
 
   def load_kafka_data(config)
     kafka = LogStash::Outputs::Kafka.new(config)
     kafka.register
     kafka.multi_receive(num_events.times.collect { event })
+    kafka.multi_receive(Array(yield)) if block_given?
     kafka.close
   end
 
+  def fetch_messages(topic, partition: 0, offset: :earliest)
+    kafka_client.fetch_messages(topic: topic, partition: partition, offset: offset)
+  end
+
 end

data/spec/unit/inputs/kafka_spec.rb

@@ -34,6 +34,18 @@ describe LogStash::Inputs::Kafka do
   subject { LogStash::Inputs::Kafka.new(config) }
 
   it "should register" do
-    expect {subject.register}.to_not raise_error
+    expect { subject.register }.to_not raise_error
+  end
+
+  context 'with client_rack' do
+    let(:config) { super.merge('client_rack' => 'EU-R1') }
+
+    it "sets broker rack parameter" do
+      expect(org.apache.kafka.clients.consumer.KafkaConsumer).
+          to receive(:new).with(hash_including('client.rack' => 'EU-R1')).
+              and_return kafka_client = double('kafka-consumer')
+
+      expect( subject.send(:create_consumer, 'sample_client-0') ).to be kafka_client
+    end
   end
 end

data/spec/unit/outputs/kafka_spec.rb

@@ -97,7 +97,7 @@ describe "outputs/kafka" do
       let(:sendcount) { failcount + 1 }
 
       it "should retry until successful" do
-        count = 0;
+        count = 0
 
         expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send)
               .exactly(sendcount).times
@@ -189,4 +189,25 @@ describe "outputs/kafka" do
       end
     end
   end
+
+  context 'when ssl endpoint identification disabled' do
+
+    let(:config) do
+      simple_kafka_config.merge('ssl_endpoint_identification_algorithm' => '', 'security_protocol' => 'SSL')
+    end
+
+    subject { LogStash::Outputs::Kafka.new(config) }
+
+    it 'does not configure truststore' do
+      expect(org.apache.kafka.clients.producer.KafkaProducer).
+          to receive(:new).with(hash_excluding('ssl.truststore.location' => anything))
+      subject.register
+    end
+
+    it 'sets empty ssl.endpoint.identification.algorithm' do
+      expect(org.apache.kafka.clients.producer.KafkaProducer).
+          to receive(:new).with(hash_including('ssl.endpoint.identification.algorithm' => ''))
+      subject.register
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-integration-kafka
 version: !ruby/object:Gem::Version
-  version: 10.0.1
+  version: 10.1.0
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-03-12 00:00:00.000000000 Z
+date: 2020-03-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -140,7 +140,7 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: poseidon
+  name: ruby-kafka
   prerelease: false
   type: :development
   version_requirements: !ruby/object:Gem::Requirement
@@ -188,10 +188,10 @@ files:
 - spec/integration/outputs/kafka_spec.rb
 - spec/unit/inputs/kafka_spec.rb
 - spec/unit/outputs/kafka_spec.rb
-- vendor/jar-dependencies/com/github/luben/zstd-jni/1.4.2-1/zstd-jni-1.4.2-1.jar
-- vendor/jar-dependencies/org/apache/kafka/kafka-clients/2.3.0/kafka-clients-2.3.0.jar
+- vendor/jar-dependencies/com/github/luben/zstd-jni/1.4.3-1/zstd-jni-1.4.3-1.jar
+- vendor/jar-dependencies/org/apache/kafka/kafka-clients/2.4.1/kafka-clients-2.4.1.jar
 - vendor/jar-dependencies/org/lz4/lz4-java/1.6.0/lz4-java-1.6.0.jar
-- vendor/jar-dependencies/org/slf4j/slf4j-api/1.7.26/slf4j-api-1.7.26.jar
+- vendor/jar-dependencies/org/slf4j/slf4j-api/1.7.28/slf4j-api-1.7.28.jar
 - vendor/jar-dependencies/org/xerial/snappy/snappy-java/1.1.7.3/snappy-java-1.1.7.3.jar
 homepage: http://www.elastic.co/guide/en/logstash/current/index.html
 licenses: