logstash-integration-kafka 10.9.0-java → 11.3.2-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: df9c89cdfcc2db6702409ec45ffb8d9f8f2b9274304889df1527e0697ccfcf95
-  data.tar.gz: e907ad2e277d27c8cdbe98ebd203af86a3505a85574d64ace4402607c075a69e
+  metadata.gz: cd3536a455fdf7a64f5882e81c0648022d572660d4dc45841249374c1cd7e406
+  data.tar.gz: 26281f8584e29961625c8e385cfd599fd43444cbedd84d191b5542f603d00701
 SHA512:
-  metadata.gz: bac93eb957af9028a6efc6e31a66c94818ae61333fa738daa6606abdd325dbea206fea2cd905d891f2b341a7bc983f8eaf5a5471015ac9548bc902f941b4a0d9
-  data.tar.gz: 456739a2409ef5a42f007a23c8d0dbfceb3518e8e65b528c0f48266f2a219c2415a83a507fdab3ba028cbc5493d645c080ce191a0d39d7c1787557abded9a0e1
+  metadata.gz: 226f3c894edb182246d36bc283f7fc8d5d8471d9d411b98054a3770f7e414ca321ddbe401012c53ff5bbf4fd74da0340193153ebf7204a74e03802dc8c3df8ad
+  data.tar.gz: 12a5454b826df30697e4f505e96893687202b8cd17536599a5832a518653274218a278d0139c3150f7c48806e823251f91f3b583ee9b60f828104b5e052fea1b

data/CHANGELOG.md

@@ -1,6 +1,44 @@
+## 11.3.2
+  - Fix: update Avro library [#150](https://api.github.com/repos/logstash-plugins/logstash-integration-kafka/pulls/150)
+
+## 11.3.1
+  - Fix: update snappy dependency [#148](https://github.com/logstash-plugins/logstash-integration-kafka/pull/148)
+
+## 11.3.0
+  - Bump kafka client to 3.4.1 [#145](https://github.com/logstash-plugins/logstash-integration-kafka/pull/145)
+
+## 11.2.1
+ - Fix nil exception to empty headers of record during event metadata assignment [#140](https://github.com/logstash-plugins/logstash-integration-kafka/pull/140)
+
+## 11.2.0
+ - Added TLS truststore and keystore settings specifically to access the schema registry [#137](https://github.com/logstash-plugins/logstash-integration-kafka/pull/137)
+
+## 11.1.0
+ - Added config `group_instance_id` to use the Kafka's consumer static membership feature [#135](https://github.com/logstash-plugins/logstash-integration-kafka/pull/135)
+
+## 11.0.0
+  - Changed Kafka client to 3.3.1, requires Logstash >= 8.3.0. 
+  - Deprecated `default` value for setting `client_dns_lookup` forcing to `use_all_dns_ips` when explicitly used [#130](https://github.com/logstash-plugins/logstash-integration-kafka/pull/130)
+  - Changed the consumer's poll from using the one that blocks on metadata retrieval to the one that doesn't [#136](https://github.com/logstash-plugins/logstash-integration-kafka/pull/133)
+
+## 10.12.1
+  - Fix: update Avro library on 10.x [#149](https://api.github.com/repos/logstash-plugins/logstash-integration-kafka/pulls/149)
+
+## 10.12.0
+  - bump kafka client to 2.8.1 [#115](https://github.com/logstash-plugins/logstash-integration-kafka/pull/115)
+
+## 10.11.0
+  - Feat: added connections_max_idle_ms setting for output [#118](https://github.com/logstash-plugins/logstash-integration-kafka/pull/118)
+  - Refactor: mixins to follow shared mixin module naming
+
+## 10.10.1
+  - Update CHANGELOG.md [#114](https://api.github.com/repos/logstash-plugins/logstash-integration-kafka/pulls/114)
+
+## 10.10.0
+  - Added config setting to enable 'zstd' compression in the Kafka output [#112](https://github.com/logstash-plugins/logstash-integration-kafka/pull/112)
+
 ## 10.9.0
   - Refactor: leverage codec when using schema registry [#106](https://github.com/logstash-plugins/logstash-integration-kafka/pull/106)
-  
     Previously using `schema_registry_url` parsed the payload as JSON even if `codec => 'plain'` was set, this is no longer the case.  
 
 ## 10.8.2
@@ -87,7 +125,6 @@
   - Fix links in changelog pointing to stand-alone plugin changelogs.
   - Refactor: scope java_import to plugin class
 
-
 ## 10.0.0
   - Initial release of the Kafka Integration Plugin, which combines
     previously-separate Kafka plugins and shared dependencies into a single

data/DEVELOPER.md

@@ -62,7 +62,7 @@ See http://kafka.apache.org/documentation.html#producerconfigs for details about
         kafka {
             topic_id => ... # string (required), The topic to produce the messages to
             broker_list => ... # string (optional), default: "localhost:9092", This is for bootstrapping and the producer will only use it for getting metadata
-            compression_codec => ... # string (optional), one of ["none", "gzip", "snappy"], default: "none"
+            compression_codec => ... # string (optional), one of ["none", "gzip", "snappy", "lz4", "zstd"], default: "none"
             compressed_topics => ... # string (optional), default: "", This parameter allows you to set whether compression should be turned on for particular
             request_required_acks => ... # number (optional), one of [-1, 0, 1], default: 0, This value controls when a produce request is considered completed
             serializer_class => ... # string, (optional) default: "kafka.serializer.StringEncoder", The serializer class for messages. The default encoder takes a byte[] and returns the same byte[]

data/docs/index.asciidoc

@@ -1,7 +1,7 @@
 :plugin: kafka
 :type: integration
 :no_codec:
-:kafka_client: 2.5.1
+:kafka_client: 3.4.1
 
 ///////////////////////////////////////////
 START - GENERATED VARIABLES, DO NOT EDIT!

data/docs/input-kafka.asciidoc

@@ -2,8 +2,8 @@
 :plugin: kafka
 :type: input
 :default_codec: plain
-:kafka_client: 2.5
-:kafka_client_doc: 25
+:kafka_client: 3.4
+:kafka_client_doc: 34
 
 ///////////////////////////////////////////
 START - GENERATED VARIABLES, DO NOT EDIT!
@@ -113,6 +113,7 @@ See the https://kafka.apache.org/{kafka_client_doc}/documentation for more detai
 | <<plugins-{type}s-{plugin}-fetch_max_wait_ms>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-fetch_min_bytes>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-group_id>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-group_instance_id>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-heartbeat_interval_ms>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-isolation_level>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-jaas_path>> |a valid filesystem path|No
@@ -134,6 +135,12 @@ See the https://kafka.apache.org/{kafka_client_doc}/documentation for more detai
 | <<plugins-{type}s-{plugin}-schema_registry_key>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-schema_registry_proxy>> |<<uri,uri>>|No
 | <<plugins-{type}s-{plugin}-schema_registry_secret>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-schema_registry_ssl_keystore_location>> |a valid filesystem path|No
+| <<plugins-{type}s-{plugin}-schema_registry_ssl_keystore_password>> |<<password,password>>|No
+| <<plugins-{type}s-{plugin}-schema_registry_ssl_keystore_type>> |<<string,string>>, one of `["jks", "PKCS12"]`|No
+| <<plugins-{type}s-{plugin}-schema_registry_ssl_truststore_location>> |a valid filesystem path|No
+| <<plugins-{type}s-{plugin}-schema_registry_ssl_truststore_password>> |<<password,password>>|No
+| <<plugins-{type}s-{plugin}-schema_registry_ssl_truststore_type>> |<<string,string>>, one of `["jks", "PKCS12"]`|No
 | <<plugins-{type}s-{plugin}-schema_registry_url>> |<<uri,uri>>|No
 | <<plugins-{type}s-{plugin}-schema_registry_validation>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-security_protocol>> |<<string,string>>, one of `["PLAINTEXT", "SSL", "SASL_PLAINTEXT", "SASL_SSL"]`|No
@@ -143,10 +150,10 @@ See the https://kafka.apache.org/{kafka_client_doc}/documentation for more detai
 | <<plugins-{type}s-{plugin}-ssl_key_password>> |<<password,password>>|No
 | <<plugins-{type}s-{plugin}-ssl_keystore_location>> |a valid filesystem path|No
 | <<plugins-{type}s-{plugin}-ssl_keystore_password>> |<<password,password>>|No
-| <<plugins-{type}s-{plugin}-ssl_keystore_type>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-ssl_keystore_type>> |<<string,string>>, one of `["jks", "PKCS12"]`|No
 | <<plugins-{type}s-{plugin}-ssl_truststore_location>> |a valid filesystem path|No
 | <<plugins-{type}s-{plugin}-ssl_truststore_password>> |<<password,password>>|No
-| <<plugins-{type}s-{plugin}-ssl_truststore_type>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-ssl_truststore_type>> |<<string,string>>, one of `["jks", "PKCS12"]`|No
 | <<plugins-{type}s-{plugin}-topics>> |<<array,array>>|No
 | <<plugins-{type}s-{plugin}-topics_pattern>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-value_deserializer_class>> |<<string,string>>|No
@@ -211,6 +218,12 @@ IP addresses for a hostname, they will all be attempted to connect to before fai
 connection. If the value is `resolve_canonical_bootstrap_servers_only` each entry will be 
 resolved and expanded into a list of canonical names.
 
+[NOTE]
+====
+Starting from Kafka 3 `default` value for `client.dns.lookup` value has been removed.
+If explicitly configured it fallbacks to `use_all_dns_ips`.
+====
+
 [id="plugins-{type}s-{plugin}-client_id"]
 ===== `client_id` 
 
@@ -338,6 +351,28 @@ NOTE: In cases when multiple inputs are being used in a single pipeline, reading
 it's essential to set a different `group_id => ...` for each input. Setting a unique `client_id => ...`
 is also recommended.
 
+[id="plugins-{type}s-{plugin}-group_instance_id"]
+===== `group_instance_id`
+
+* Value type is <<string,string>>
+* There is no default value for this setting.
+
+The static membership identifier for this Logstash Kafka consumer. Static membership feature was introduced in
+https://cwiki.apache.org/confluence/display/KAFKA/KIP-345%3A+Introduce+static+membership+protocol+to+reduce+consumer+rebalances[KIP-345],
+available under Kafka property `group.instance.id`.
+Its purpose is to avoid rebalances in situations in which a lot of data
+has to be forwarded after a consumer goes offline.
+This feature mitigates cases where the service state is heavy and the rebalance of one topic partition from instance
+A to B would cause a huge amount of data to be transferred.
+A client that goes offline/online frequently can avoid frequent and heavy rebalances by using this option.
+
+NOTE: The `group_instance_id` setting must be unique across all the clients belonging to the same <<plugins-{type}s-{plugin}-group_id>>.
+Otherwise, another client connecting with same `group.instance.id` value would cause the oldest instance to be disconnected.
+You can set this value to use information such as a hostname, an IP, or anything that uniquely identifies the client application.
+
+NOTE: In cases when multiple threads are configured and `consumer_threads` is greater than one, a suffix is appended to
+the `group_instance_id` to avoid collisions.
+
 [id="plugins-{type}s-{plugin}-heartbeat_interval_ms"]
 ===== `heartbeat_interval_ms` 
 
@@ -569,6 +604,54 @@ Set the address of a forward HTTP proxy. An empty string is treated as if proxy
 
 Set the password for basic authorization to access remote Schema Registry.
 
+[id="plugins-{type}s-{plugin}-schema_registry_ssl_keystore_location"]
+===== `schema_registry_ssl_keystore_location`
+
+* Value type is <<path,path>>
+* There is no default value for this setting.
+
+If schema registry client authentication is required, this setting stores the keystore path.
+
+[id="plugins-{type}s-{plugin}-schema_registry_ssl_keystore_password"]
+===== `schema_registry_ssl_keystore_password`
+
+* Value type is <<password,password>>
+* There is no default value for this setting.
+
+If schema registry authentication is required, this setting stores the keystore password.
+
+[id="plugins-{type}s-{plugin}-schema_registry_ssl_keystore_type"]
+===== `schema_registry_ssl_keystore_type`
+
+* Value type is <<string,string>>
+* There is no default value for this setting.
+
+The format of the keystore file. It must be either `jks` or `PKCS12`.
+
+[id="plugins-{type}s-{plugin}-schema_registry_ssl_truststore_location"]
+===== `schema_registry_ssl_truststore_location`
+
+* Value type is <<path,path>>
+* There is no default value for this setting.
+
+The truststore path to validate the schema registry's certificate.
+
+[id="plugins-{type}s-{plugin}-schema_registry_ssl_truststore_password"]
+===== `schema_registry_ssl_truststore_password`
+
+* Value type is <<password,password>>
+* There is no default value for this setting.
+
+The schema registry truststore password.
+
+[id="plugins-{type}s-{plugin}-schema_registry_ssl_truststore_type"]
+===== `schema_registry_ssl_truststore_type`
+
+* Value type is <<string,string>>
+* There is no default value for this setting.
+
+The format of the schema registry's truststore file. It must be either `jks` or `PKCS12`.
+
 [id="plugins-{type}s-{plugin}-schema_registry_url"]
 ===== `schema_registry_url`
 
@@ -659,7 +742,7 @@ If client authentication is required, this setting stores the keystore password
   * Value type is <<string,string>>
   * There is no default value for this setting.
 
-The keystore type.
+The format of the keystore file. It must be either `jks` or `PKCS12`.
 
 [id="plugins-{type}s-{plugin}-ssl_truststore_location"]
 ===== `ssl_truststore_location` 
@@ -683,7 +766,7 @@ The truststore password.
   * Value type is <<string,string>>
   * There is no default value for this setting.
 
-The truststore type.
+The format of the truststore file. It must be either `jks` or `PKCS12`.
 
 [id="plugins-{type}s-{plugin}-topics"]
 ===== `topics` 
@@ -699,8 +782,13 @@ A list of topics to subscribe to, defaults to ["logstash"].
   * Value type is <<string,string>>
   * There is no default value for this setting.
 
-A topic regex pattern to subscribe to. 
-The topics configuration will be ignored when using this configuration.
+A topic regular expression pattern to subscribe to.
+
+Filtering by a regular expression is done by retrieving the full list of topic names from the broker and applying the pattern locally. When used with brokers with a lot of topics this operation could be very slow, especially if there are a lot of consumers.
+
+NOTE:  When the broker has some topics configured with ACL rules and they miss the DESCRIBE permission, then the subscription
+happens but on the broker side it is logged that the subscription of some topics was denied to the configured user.
+
 
 [id="plugins-{type}s-{plugin}-value_deserializer_class"]
 ===== `value_deserializer_class` 

data/docs/output-kafka.asciidoc

@@ -2,8 +2,8 @@
 :plugin: kafka
 :type: output
 :default_codec: plain
-:kafka_client: 2.5
-:kafka_client_doc: 25
+:kafka_client: 3.4
+:kafka_client_doc: 34
 
 ///////////////////////////////////////////
 START - GENERATED VARIABLES, DO NOT EDIT!
@@ -84,7 +84,8 @@ See the https://kafka.apache.org/{kafka_client_doc}/documentation for more detai
 | <<plugins-{type}s-{plugin}-buffer_memory>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-client_dns_lookup>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-client_id>> |<<string,string>>|No
-| <<plugins-{type}s-{plugin}-compression_type>> |<<string,string>>, one of `["none", "gzip", "snappy", "lz4"]`|No
+| <<plugins-{type}s-{plugin}-compression_type>> |<<string,string>>, one of `["none", "gzip", "snappy", "lz4", "zstd"]`|No
+| <<plugins-{type}s-{plugin}-connections_max_idle_ms>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-jaas_path>> |a valid filesystem path|No
 | <<plugins-{type}s-{plugin}-kerberos_config>> |a valid filesystem path|No
 | <<plugins-{type}s-{plugin}-key_serializer>> |<<string,string>>|No
@@ -180,6 +181,12 @@ all IP addresses returned for a hostname before failing the connection.
 If set to `resolve_canonical_bootstrap_servers_only`, each entry will be 
 resolved and expanded into a list of canonical names.
 
+[NOTE]
+====
+Starting from Kafka 3 `default` value for `client.dns.lookup` value has been removed.
+If explicitly configured it fallbacks to `use_all_dns_ips`.
+====
+
 [id="plugins-{type}s-{plugin}-client_id"]
 ===== `client_id` 
 
@@ -193,11 +200,19 @@ ip/port by allowing a logical application name to be included with the request
 [id="plugins-{type}s-{plugin}-compression_type"]
 ===== `compression_type` 
 
-  * Value can be any of: `none`, `gzip`, `snappy`, `lz4`
+  * Value can be any of: `none`, `gzip`, `snappy`, `lz4`, `zstd`
   * Default value is `"none"`
 
 The compression type for all data generated by the producer.
-The default is none (i.e. no compression). Valid values are none, gzip, snappy, or lz4.
+The default is none (meaning no compression). Valid values are none, gzip, snappy, lz4, or zstd.
+
+[id="plugins-{type}s-{plugin}-connections_max_idle_ms"]
+===== `connections_max_idle_ms`
+
+  * Value type is <<number,number>>
+  * Default value is `540000` milliseconds (9 minutes).
+
+Close idle connections after the number of milliseconds specified by this config.
 
 [id="plugins-{type}s-{plugin}-jaas_path"]
 ===== `jaas_path` 

data/lib/logstash/inputs/kafka.rb

@@ -2,12 +2,11 @@ require 'logstash/namespace'
 require 'logstash/inputs/base'
 require 'stud/interval'
 require 'java'
-require 'logstash-integration-kafka_jars.rb'
-require 'logstash/plugin_mixins/kafka_support'
-require 'manticore'
 require "json"
 require "logstash/json"
-require_relative '../plugin_mixins/common'
+require 'logstash-integration-kafka_jars.rb'
+require 'logstash/plugin_mixins/kafka/common'
+require 'logstash/plugin_mixins/kafka/avro_schema_registry'
 require 'logstash/plugin_mixins/deprecation_logger_support'
 
 # This input will read events from a Kafka topic. It uses the 0.10 version of
@@ -57,8 +56,8 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
 
   DEFAULT_DESERIALIZER_CLASS = "org.apache.kafka.common.serialization.StringDeserializer"
 
-  include LogStash::PluginMixins::KafkaSupport
-  include ::LogStash::PluginMixins::KafkaAvroSchemaRegistry
+  include LogStash::PluginMixins::Kafka::Common
+  include LogStash::PluginMixins::Kafka::AvroSchemaRegistry
   include LogStash::PluginMixins::DeprecationLoggerSupport
 
   config_name 'kafka'
@@ -93,13 +92,12 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
   # IP addresses for a hostname, they will all be attempted to connect to before failing the 
   # connection. If the value is `resolve_canonical_bootstrap_servers_only` each entry will be 
   # resolved and expanded into a list of canonical names.
-  config :client_dns_lookup, :validate => ["default", "use_all_dns_ips", "resolve_canonical_bootstrap_servers_only"], :default => "default"
+  # Starting from Kafka 3 `default` value for `client.dns.lookup` value has been removed. If explicitly configured it fallbacks to `use_all_dns_ips`.
+  config :client_dns_lookup, :validate => ["default", "use_all_dns_ips", "resolve_canonical_bootstrap_servers_only"], :default => "use_all_dns_ips"
   # The id string to pass to the server when making requests. The purpose of this
   # is to be able to track the source of requests beyond just ip/port by allowing
   # a logical application name to be included.
   config :client_id, :validate => :string, :default => "logstash"
-  # Close idle connections after the number of milliseconds specified by this config.
-  config :connections_max_idle_ms, :validate => :number, :default => 540_000 # (9m) Kafka default
   # Ideally you should have as many threads as the number of partitions for a perfect
   # balance — more threads than partitions means that some threads will be idle
   config :consumer_threads, :validate => :number, :default => 1
@@ -126,6 +124,11 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
   # that happens to be made up of multiple processors. Messages in a topic will be distributed to all
   # Logstash instances with the same `group_id`
   config :group_id, :validate => :string, :default => "logstash"
+  # Set a static group instance id used in static membership feature to avoid rebalancing when a
+  # consumer goes offline. If set and `consumer_threads` is greater than 1 then for each
+  # consumer crated by each thread an artificial suffix is appended to the user provided `group_instance_id`
+  # to avoid clashing.
+  config :group_instance_id, :validate => :string
   # The expected time between heartbeats to the consumer coordinator. Heartbeats are used to ensure 
   # that the consumer's session stays active and to facilitate rebalancing when new
   # consumers join or leave the group. The value must be set lower than
@@ -138,7 +141,7 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
   # been aborted. Non-transactional messages will be returned unconditionally in either mode.
   config :isolation_level, :validate => ["read_uncommitted", "read_committed"], :default => "read_uncommitted" # Kafka default
   # Java Class used to deserialize the record's key
-  config :key_deserializer_class, :validate => :string, :default => "org.apache.kafka.common.serialization.StringDeserializer"
+  config :key_deserializer_class, :validate => :string, :default => DEFAULT_DESERIALIZER_CLASS
   # The maximum delay between invocations of poll() when using consumer group management. This places 
   # an upper bound on the amount of time that the consumer can be idle before fetching more records. 
   # If poll() is not called before expiration of this timeout, then the consumer is considered failed and 
@@ -152,9 +155,6 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
   config :max_partition_fetch_bytes, :validate => :number, :default => 1_048_576 # (1MB) Kafka default
   # The maximum number of records returned in a single call to poll().
   config :max_poll_records, :validate => :number, :default => 500 # Kafka default
-  # The period of time in milliseconds after which we force a refresh of metadata even if
-  # we haven't seen any partition leadership changes to proactively discover any new brokers or partitions
-  config :metadata_max_age_ms, :validate => :number, :default => 300_000 # (5m) Kafka default
   # The name of the partition assignment strategy that the client uses to distribute
   # partition ownership amongst consumer instances, supported options are `range`,
   # `round_robin`, `sticky` and `cooperative_sticky`
@@ -167,10 +167,6 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
   # This avoids repeatedly connecting to a host in a tight loop.
   # This backoff applies to all connection attempts by the client to a broker.
   config :reconnect_backoff_ms, :validate => :number, :default => 50 # Kafka default
-  # The configuration controls the maximum amount of time the client will wait for the response of a request.
-  # If the response is not received before the timeout elapses the client will resend the request if necessary
-  # or fail the request if retries are exhausted.
-  config :request_timeout_ms, :validate => :number, :default => 40_000 # Kafka default
   # The amount of time to wait before attempting to retry a failed fetch request
   # to a given topic partition. This avoids repeated fetching-and-failing in a tight loop.
   config :retry_backoff_ms, :validate => :number, :default => 100 # Kafka default
@@ -267,6 +263,7 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
   def register
     @runner_threads = []
     @metadata_mode = extract_metadata_level(@decorate_events)
+    reassign_dns_lookup
     @pattern ||= java.util.regex.Pattern.compile(@topics_pattern) unless @topics_pattern.nil?
     check_schema_registry_parameters
   end
@@ -295,7 +292,10 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
 
   public
   def run(logstash_queue)
-    @runner_consumers = consumer_threads.times.map { |i| subscribe(create_consumer("#{client_id}-#{i}")) }
+    @runner_consumers = consumer_threads.times.map do |i|
+      thread_group_instance_id = consumer_threads > 1 && group_instance_id ? "#{group_instance_id}-#{i}" : group_instance_id
+      subscribe(create_consumer("#{client_id}-#{i}", thread_group_instance_id))
+    end
     @runner_threads = @runner_consumers.map.with_index { |consumer, i| thread_runner(logstash_queue, consumer,
                                                                                      "kafka-input-worker-#{client_id}-#{i}") }
     @runner_threads.each(&:start)
@@ -339,10 +339,13 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
   def do_poll(consumer)
     records = []
     begin
-      records = consumer.poll(poll_timeout_ms)
+      records = consumer.poll(java.time.Duration.ofMillis(poll_timeout_ms))
     rescue org.apache.kafka.common.errors.WakeupException => e
       logger.debug("Wake up from poll", :kafka_error_message => e)
       raise e unless stop?
+    rescue org.apache.kafka.common.errors.FencedInstanceIdException => e
+      logger.error("Another consumer with same group.instance.id has connected", :original_error_message => e.message)
+      raise e unless stop?
     rescue => e
       logger.error("Unable to poll Kafka consumer",
                    :kafka_error_message => e,
@@ -370,7 +373,9 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
       event.set("[@metadata][kafka][timestamp]", record.timestamp)
     end
     if @metadata_mode.include?(:headers)
-      record.headers.each do |header|
+      record.headers
+            .select{|h| header_with_value(h) }
+            .each do |header|
         s = String.from_java_bytes(header.value)
         s.force_encoding(Encoding::UTF_8)
         if s.valid_encoding?
@@ -397,7 +402,7 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
   end
 
   private
-  def create_consumer(client_id)
+  def create_consumer(client_id, group_instance_id)
     begin
       props = java.util.Properties.new
       kafka = org.apache.kafka.clients.consumer.ConsumerConfig
@@ -415,6 +420,7 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
       props.put(kafka::FETCH_MAX_WAIT_MS_CONFIG, fetch_max_wait_ms.to_s) unless fetch_max_wait_ms.nil?
       props.put(kafka::FETCH_MIN_BYTES_CONFIG, fetch_min_bytes.to_s) unless fetch_min_bytes.nil?
       props.put(kafka::GROUP_ID_CONFIG, group_id)
+      props.put(kafka::GROUP_INSTANCE_ID_CONFIG, group_instance_id) unless group_instance_id.nil?
       props.put(kafka::HEARTBEAT_INTERVAL_MS_CONFIG, heartbeat_interval_ms.to_s) unless heartbeat_interval_ms.nil?
       props.put(kafka::ISOLATION_LEVEL_CONFIG, isolation_level)
       props.put(kafka::KEY_DESERIALIZER_CLASS_CONFIG, key_deserializer_class)
@@ -456,6 +462,17 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
         set_trustore_keystore_config(props)
         set_sasl_config(props)
       end
+      if schema_registry_ssl_truststore_location
+        props.put('schema.registry.ssl.truststore.location', schema_registry_ssl_truststore_location)
+        props.put('schema.registry.ssl.truststore.password', schema_registry_ssl_truststore_password.value)
+        props.put('schema.registry.ssl.truststore.type', schema_registry_ssl_truststore_type)
+      end
+
+      if schema_registry_ssl_keystore_location
+        props.put('schema.registry.ssl.keystore.location', schema_registry_ssl_keystore_location)
+        props.put('schema.registry.ssl.keystore.password', schema_registry_ssl_keystore_password.value)
+        props.put('schema.registry.ssl.keystore.type', schema_registry_ssl_keystore_type)
+      end
 
       org.apache.kafka.clients.consumer.KafkaConsumer.new(props)
     rescue => e
@@ -484,4 +501,8 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
     end
   end
 
+  def header_with_value(header)
+    !header.nil? && !header.value.nil? && !header.key.nil?
+  end
+
 end #class LogStash::Inputs::Kafka

data/lib/logstash/outputs/kafka.rb

@@ -2,7 +2,7 @@ require 'logstash/namespace'
 require 'logstash/outputs/base'
 require 'java'
 require 'logstash-integration-kafka_jars.rb'
-require 'logstash/plugin_mixins/kafka_support'
+require 'logstash/plugin_mixins/kafka/common'
 
 # Write events to a Kafka topic. This uses the Kafka Producer API to write messages to a topic on
 # the broker.
@@ -51,7 +51,7 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
 
   java_import org.apache.kafka.clients.producer.ProducerRecord
 
-  include LogStash::PluginMixins::KafkaSupport
+  include LogStash::PluginMixins::Kafka::Common
 
   declare_threadsafe!
 
@@ -80,13 +80,14 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
   # The total bytes of memory the producer can use to buffer records waiting to be sent to the server.
   config :buffer_memory, :validate => :number, :default => 33_554_432 # (32M) Kafka default
   # The compression type for all data generated by the producer.
-  # The default is none (i.e. no compression). Valid values are none, gzip, or snappy.
-  config :compression_type, :validate => ["none", "gzip", "snappy", "lz4"], :default => "none"
+  # The default is none (i.e. no compression). Valid values are none, gzip, snappy, lz4 or zstd.
+  config :compression_type, :validate => ["none", "gzip", "snappy", "lz4", "zstd"], :default => "none"
   # How DNS lookups should be done. If set to `use_all_dns_ips`, when the lookup returns multiple 
   # IP addresses for a hostname, they will all be attempted to connect to before failing the 
   # connection. If the value is `resolve_canonical_bootstrap_servers_only` each entry will be 
   # resolved and expanded into a list of canonical names.
-  config :client_dns_lookup, :validate => ["default", "use_all_dns_ips", "resolve_canonical_bootstrap_servers_only"], :default => "default"
+  # Starting from Kafka 3 `default` value for `client.dns.lookup` value has been removed. If explicitly configured it fallbacks to `use_all_dns_ips`.
+  config :client_dns_lookup, :validate => ["default", "use_all_dns_ips", "resolve_canonical_bootstrap_servers_only"], :default => "use_all_dns_ips"
   # The id string to pass to the server when making requests.
   # The purpose of this is to be able to track the source of requests beyond just
   # ip/port by allowing a logical application name to be included with the request
@@ -107,19 +108,12 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
   config :message_key, :validate => :string
   # the timeout setting for initial metadata request to fetch topic metadata.
   config :metadata_fetch_timeout_ms, :validate => :number, :default => 60_000
-  # the max time in milliseconds before a metadata refresh is forced.
-  config :metadata_max_age_ms, :validate => :number, :default => 300_000 # (5m) Kafka default
   # Partitioner to use - can be `default`, `uniform_sticky`, `round_robin` or a fully qualified class name of a custom partitioner.
   config :partitioner, :validate => :string
   # The size of the TCP receive buffer to use when reading data
   config :receive_buffer_bytes, :validate => :number, :default => 32_768 # (32KB) Kafka default
   # The amount of time to wait before attempting to reconnect to a given host when a connection fails.
   config :reconnect_backoff_ms, :validate => :number, :default => 50 # Kafka default
-  # The configuration controls the maximum amount of time the client will wait
-  # for the response of a request. If the response is not received before the timeout
-  # elapses the client will resend the request if necessary or fail the request if
-  # retries are exhausted.
-  config :request_timeout_ms, :validate => :number, :default => 40_000 # (40s) Kafka default
   # The default retry behavior is to retry until successful. To prevent data loss,
   # the use of this setting is discouraged.
   #
@@ -197,6 +191,7 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
       logger.warn("Kafka output is configured with finite retry. This instructs Logstash to LOSE DATA after a set number of send attempts fails. If you do not want to lose data if Kafka is down, then you must remove the retry setting.", :retries => @retries)
     end
 
+    reassign_dns_lookup
 
     @producer = create_producer
     if value_serializer == 'org.apache.kafka.common.serialization.StringSerializer'