logstash-integration-kafka 10.8.2-java → 10.11.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2e11499f791fd2bd71102ddb6f8e52fcbd9791feecf4107c7ebd27dc9334cc34
-  data.tar.gz: 779cd381a6e53155f61457229df4fc827ceda02ef4e069d4eab0240dafa4ab83
+  metadata.gz: 2833139d65282055a4e924ccfb50b19341a61af69e0834ef318a62f1459f3a16
+  data.tar.gz: fbfd7e6b94e9a74109b44b838df344ed4d7447d56d624f24e642235eb62b00e3
 SHA512:
-  metadata.gz: 5577048fe1adee0c28b012c0bf6eb0ff594b7be4760f7b7ebdf9072473fb0a1806e24997dd7b0659c21f205396a4d7eccb10bc4e789541c35fb7e32610562632
-  data.tar.gz: 7ee0e7428d63957a57e927a3f0408bcdddf94efad944c5cf0acbac3c1031be55b6751ad0290ea134e5201f10f9e26f2804c6d68150e925f651c9db27847bab5e
+  metadata.gz: b35d99bf553fbfd3b78f3f7de62b427b18001853573c5160090bebba5dfb78f4d17aa9f879a94a98805f949b1ed76a3fa82e9c1a7b95815f223a5d04104f5bce
+  data.tar.gz: 0e8edf4f07b00443c39ac55523fb5663395ba0ae524dfedc5181a56b5160c2c33fe110e24bccbb060b43c6be0aa2211b50422d041fa1dea59561a26efc79d687

data/CHANGELOG.md

@@ -1,3 +1,17 @@
+## 10.11.0
+  - Feat: added connections_max_idle_ms setting for output [#118](https://github.com/logstash-plugins/logstash-integration-kafka/pull/118)
+  - Refactor: mixins to follow shared mixin module naming
+
+## 10.10.1
+  - Update CHANGELOG.md [#114](https://api.github.com/repos/logstash-plugins/logstash-integration-kafka/pulls/114)
+
+## 10.10.0
+  - Added config setting to enable 'zstd' compression in the Kafka output [#112](https://github.com/logstash-plugins/logstash-integration-kafka/pull/112)
+
+## 10.9.0
+  - Refactor: leverage codec when using schema registry [#106](https://github.com/logstash-plugins/logstash-integration-kafka/pull/106)
+    Previously using `schema_registry_url` parsed the payload as JSON even if `codec => 'plain'` was set, this is no longer the case.  
+
 ## 10.8.2
   - [DOC] Updates description of `enable_auto_commit=false` to clarify that the commit happens after data is fetched AND written to the queue [#90](https://github.com/logstash-plugins/logstash-integration-kafka/pull/90)
   - Fix: update to Gradle 7 [#104](https://github.com/logstash-plugins/logstash-integration-kafka/pull/104)
@@ -82,7 +96,6 @@
   - Fix links in changelog pointing to stand-alone plugin changelogs.
   - Refactor: scope java_import to plugin class
 
-
 ## 10.0.0
   - Initial release of the Kafka Integration Plugin, which combines
     previously-separate Kafka plugins and shared dependencies into a single

data/DEVELOPER.md

@@ -62,7 +62,7 @@ See http://kafka.apache.org/documentation.html#producerconfigs for details about
         kafka {
             topic_id => ... # string (required), The topic to produce the messages to
             broker_list => ... # string (optional), default: "localhost:9092", This is for bootstrapping and the producer will only use it for getting metadata
-            compression_codec => ... # string (optional), one of ["none", "gzip", "snappy"], default: "none"
+            compression_codec => ... # string (optional), one of ["none", "gzip", "snappy", "lz4", "zstd"], default: "none"
             compressed_topics => ... # string (optional), default: "", This parameter allows you to set whether compression should be turned on for particular
             request_required_acks => ... # number (optional), one of [-1, 0, 1], default: 0, This value controls when a produce request is considered completed
             serializer_class => ... # string, (optional) default: "kafka.serializer.StringEncoder", The serializer class for messages. The default encoder takes a byte[] and returns the same byte[]

data/docs/output-kafka.asciidoc

@@ -84,7 +84,8 @@ See the https://kafka.apache.org/{kafka_client_doc}/documentation for more detai
 | <<plugins-{type}s-{plugin}-buffer_memory>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-client_dns_lookup>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-client_id>> |<<string,string>>|No
-| <<plugins-{type}s-{plugin}-compression_type>> |<<string,string>>, one of `["none", "gzip", "snappy", "lz4"]`|No
+| <<plugins-{type}s-{plugin}-compression_type>> |<<string,string>>, one of `["none", "gzip", "snappy", "lz4", "zstd"]`|No
+| <<plugins-{type}s-{plugin}-connections_max_idle_ms>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-jaas_path>> |a valid filesystem path|No
 | <<plugins-{type}s-{plugin}-kerberos_config>> |a valid filesystem path|No
 | <<plugins-{type}s-{plugin}-key_serializer>> |<<string,string>>|No
@@ -193,11 +194,19 @@ ip/port by allowing a logical application name to be included with the request
 [id="plugins-{type}s-{plugin}-compression_type"]
 ===== `compression_type` 
 
-  * Value can be any of: `none`, `gzip`, `snappy`, `lz4`
+  * Value can be any of: `none`, `gzip`, `snappy`, `lz4`, `zstd`
   * Default value is `"none"`
 
 The compression type for all data generated by the producer.
-The default is none (i.e. no compression). Valid values are none, gzip, snappy, or lz4.
+The default is none (meaning no compression). Valid values are none, gzip, snappy, lz4, or zstd.
+
+[id="plugins-{type}s-{plugin}-connections_max_idle_ms"]
+===== `connections_max_idle_ms`
+
+  * Value type is <<number,number>>
+  * Default value is `540000` milliseconds (9 minutes).
+
+Close idle connections after the number of milliseconds specified by this config.
 
 [id="plugins-{type}s-{plugin}-jaas_path"]
 ===== `jaas_path` 

data/lib/logstash/inputs/kafka.rb

@@ -2,12 +2,11 @@ require 'logstash/namespace'
 require 'logstash/inputs/base'
 require 'stud/interval'
 require 'java'
-require 'logstash-integration-kafka_jars.rb'
-require 'logstash/plugin_mixins/kafka_support'
-require 'manticore'
 require "json"
 require "logstash/json"
-require_relative '../plugin_mixins/common'
+require 'logstash-integration-kafka_jars.rb'
+require 'logstash/plugin_mixins/kafka/common'
+require 'logstash/plugin_mixins/kafka/avro_schema_registry'
 require 'logstash/plugin_mixins/deprecation_logger_support'
 
 # This input will read events from a Kafka topic. It uses the 0.10 version of
@@ -57,13 +56,18 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
 
   DEFAULT_DESERIALIZER_CLASS = "org.apache.kafka.common.serialization.StringDeserializer"
 
-  include LogStash::PluginMixins::KafkaSupport
-  include ::LogStash::PluginMixins::KafkaAvroSchemaRegistry
+  include LogStash::PluginMixins::Kafka::Common
+  include LogStash::PluginMixins::Kafka::AvroSchemaRegistry
   include LogStash::PluginMixins::DeprecationLoggerSupport
 
   config_name 'kafka'
 
-  default :codec, 'plain'
+  # default :codec, 'plain' or 'json' depending whether schema registry is used
+  #
+  # @override LogStash::Inputs::Base - removing the `:default => :plain`
+  config :codec, :validate => :codec
+  # NOTE: isn't necessary due the params['codec'] = ... done in #initialize
+  # having the `nil` default explicit makes the behavior more noticeable.
 
   # The frequency in milliseconds that the consumer offsets are committed to Kafka.
   config :auto_commit_interval_ms, :validate => :number, :default => 5000 # Kafka default
@@ -93,8 +97,6 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
   # is to be able to track the source of requests beyond just ip/port by allowing
   # a logical application name to be included.
   config :client_id, :validate => :string, :default => "logstash"
-  # Close idle connections after the number of milliseconds specified by this config.
-  config :connections_max_idle_ms, :validate => :number, :default => 540_000 # (9m) Kafka default
   # Ideally you should have as many threads as the number of partitions for a perfect
   # balance — more threads than partitions means that some threads will be idle
   config :consumer_threads, :validate => :number, :default => 1
@@ -147,9 +149,6 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
   config :max_partition_fetch_bytes, :validate => :number, :default => 1_048_576 # (1MB) Kafka default
   # The maximum number of records returned in a single call to poll().
   config :max_poll_records, :validate => :number, :default => 500 # Kafka default
-  # The period of time in milliseconds after which we force a refresh of metadata even if
-  # we haven't seen any partition leadership changes to proactively discover any new brokers or partitions
-  config :metadata_max_age_ms, :validate => :number, :default => 300_000 # (5m) Kafka default
   # The name of the partition assignment strategy that the client uses to distribute
   # partition ownership amongst consumer instances, supported options are `range`,
   # `round_robin`, `sticky` and `cooperative_sticky`
@@ -162,10 +161,6 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
   # This avoids repeatedly connecting to a host in a tight loop.
   # This backoff applies to all connection attempts by the client to a broker.
   config :reconnect_backoff_ms, :validate => :number, :default => 50 # Kafka default
-  # The configuration controls the maximum amount of time the client will wait for the response of a request.
-  # If the response is not received before the timeout elapses the client will resend the request if necessary
-  # or fail the request if retries are exhausted.
-  config :request_timeout_ms, :validate => :number, :default => 40_000 # Kafka default
   # The amount of time to wait before attempting to retry a failed fetch request
   # to a given topic partition. This avoids repeated fetching-and-failing in a tight loop.
   config :retry_backoff_ms, :validate => :number, :default => 100 # Kafka default
@@ -249,6 +244,15 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
 
   attr_reader :metadata_mode
 
+  # @overload based on schema registry change the codec default
+  def initialize(params = {})
+    unless params.key?('codec')
+      params['codec'] = params.key?('schema_registry_url') ? 'json' : 'plain'
+    end
+
+    super(params)
+  end
+
   public
   def register
     @runner_threads = []
@@ -341,22 +345,11 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
   def handle_record(record, codec_instance, queue)
     codec_instance.decode(record.value.to_s) do |event|
       decorate(event)
-      maybe_apply_schema(event, record)
       maybe_set_metadata(event, record)
       queue << event
     end
   end
 
-  def maybe_apply_schema(event, record)
-    if schema_registry_url
-      json = LogStash::Json.load(record.value.to_s)
-      json.each do |k, v|
-        event.set(k, v)
-      end
-      event.remove("message")
-    end
-  end
-
   def maybe_set_metadata(event, record)
     if @metadata_mode.include?(:record_props)
       event.set("[@metadata][kafka][topic]", record.topic)

data/lib/logstash/outputs/kafka.rb

@@ -2,7 +2,7 @@ require 'logstash/namespace'
 require 'logstash/outputs/base'
 require 'java'
 require 'logstash-integration-kafka_jars.rb'
-require 'logstash/plugin_mixins/kafka_support'
+require 'logstash/plugin_mixins/kafka/common'
 
 # Write events to a Kafka topic. This uses the Kafka Producer API to write messages to a topic on
 # the broker.
@@ -51,7 +51,7 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
 
   java_import org.apache.kafka.clients.producer.ProducerRecord
 
-  include LogStash::PluginMixins::KafkaSupport
+  include LogStash::PluginMixins::Kafka::Common
 
   declare_threadsafe!
 
@@ -80,8 +80,8 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
   # The total bytes of memory the producer can use to buffer records waiting to be sent to the server.
   config :buffer_memory, :validate => :number, :default => 33_554_432 # (32M) Kafka default
   # The compression type for all data generated by the producer.
-  # The default is none (i.e. no compression). Valid values are none, gzip, or snappy.
-  config :compression_type, :validate => ["none", "gzip", "snappy", "lz4"], :default => "none"
+  # The default is none (i.e. no compression). Valid values are none, gzip, snappy, lz4 or zstd.
+  config :compression_type, :validate => ["none", "gzip", "snappy", "lz4", "zstd"], :default => "none"
   # How DNS lookups should be done. If set to `use_all_dns_ips`, when the lookup returns multiple 
   # IP addresses for a hostname, they will all be attempted to connect to before failing the 
   # connection. If the value is `resolve_canonical_bootstrap_servers_only` each entry will be 
@@ -107,19 +107,12 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
   config :message_key, :validate => :string
   # the timeout setting for initial metadata request to fetch topic metadata.
   config :metadata_fetch_timeout_ms, :validate => :number, :default => 60_000
-  # the max time in milliseconds before a metadata refresh is forced.
-  config :metadata_max_age_ms, :validate => :number, :default => 300_000 # (5m) Kafka default
   # Partitioner to use - can be `default`, `uniform_sticky`, `round_robin` or a fully qualified class name of a custom partitioner.
   config :partitioner, :validate => :string
   # The size of the TCP receive buffer to use when reading data
   config :receive_buffer_bytes, :validate => :number, :default => 32_768 # (32KB) Kafka default
   # The amount of time to wait before attempting to reconnect to a given host when a connection fails.
   config :reconnect_backoff_ms, :validate => :number, :default => 50 # Kafka default
-  # The configuration controls the maximum amount of time the client will wait
-  # for the response of a request. If the response is not received before the timeout
-  # elapses the client will resend the request if necessary or fail the request if
-  # retries are exhausted.
-  config :request_timeout_ms, :validate => :number, :default => 40_000 # (40s) Kafka default
   # The default retry behavior is to retry until successful. To prevent data loss,
   # the use of this setting is discouraged.
   #

data/lib/logstash/plugin_mixins/kafka/avro_schema_registry.rb

@@ -0,0 +1,108 @@
+require 'manticore'
+
+module LogStash module PluginMixins module Kafka
+  module AvroSchemaRegistry
+
+    def self.included(base)
+      base.extend(self)
+      base.setup_schema_registry_config
+    end
+
+    def setup_schema_registry_config
+      # Option to set key to access Schema Registry.
+      config :schema_registry_key, :validate => :string
+
+      # Option to set secret to access Schema Registry.
+      config :schema_registry_secret, :validate => :password
+
+      # Option to set the endpoint of the Schema Registry.
+      # This option permit the usage of Avro Kafka deserializer which retrieve the schema of the Avro message from an
+      # instance of schema registry. If this option has value `value_deserializer_class` nor `topics_pattern` could be valued
+      config :schema_registry_url, :validate => :uri
+
+      # Option to set the proxy of the Schema Registry.
+      # This option permits to define a proxy to be used to reach the schema registry service instance.
+      config :schema_registry_proxy, :validate => :uri
+
+      # Option to skip validating the schema registry during registration. This can be useful when using
+      # certificate based auth
+      config :schema_registry_validation, :validate => ['auto', 'skip'], :default => 'auto'
+    end
+
+    def check_schema_registry_parameters
+      if @schema_registry_url
+        check_for_schema_registry_conflicts
+        @schema_registry_proxy_host, @schema_registry_proxy_port  = split_proxy_into_host_and_port(schema_registry_proxy)
+        check_for_key_and_secret
+        check_for_schema_registry_connectivity_and_subjects if schema_registry_validation?
+      end
+    end
+
+    def schema_registry_validation?
+      return false if schema_registry_validation.to_s == 'skip'
+      return false if using_kerberos? # pre-validation doesn't support kerberos
+
+      true
+    end
+
+    def using_kerberos?
+      security_protocol == "SASL_PLAINTEXT" || security_protocol == "SASL_SSL"
+    end
+
+    private
+    def check_for_schema_registry_conflicts
+      if @value_deserializer_class != LogStash::Inputs::Kafka::DEFAULT_DESERIALIZER_CLASS
+        raise LogStash::ConfigurationError, 'Option schema_registry_url prohibit the customization of value_deserializer_class'
+      end
+      if @topics_pattern && !@topics_pattern.empty?
+       raise LogStash::ConfigurationError, 'Option schema_registry_url prohibit the customization of topics_pattern'
+      end
+    end
+
+    private
+    def check_for_schema_registry_connectivity_and_subjects
+      options = {}
+      if schema_registry_proxy && !schema_registry_proxy.empty?
+        options[:proxy] = schema_registry_proxy.to_s
+      end
+      if schema_registry_key and !schema_registry_key.empty?
+        options[:auth] = {:user => schema_registry_key, :password => schema_registry_secret.value}
+      end
+      client = Manticore::Client.new(options)
+      begin
+        response = client.get(@schema_registry_url.uri.to_s + '/subjects').body
+      rescue Manticore::ManticoreException => e
+        raise LogStash::ConfigurationError.new("Schema registry service doesn't respond, error: #{e.message}")
+      end
+      registered_subjects = JSON.parse response
+      expected_subjects = @topics.map { |t| "#{t}-value"}
+      if (expected_subjects & registered_subjects).size != expected_subjects.size
+        undefined_topic_subjects = expected_subjects - registered_subjects
+        raise LogStash::ConfigurationError, "The schema registry does not contain definitions for required topic subjects: #{undefined_topic_subjects}"
+      end
+    end
+
+    def split_proxy_into_host_and_port(proxy_uri)
+      return nil unless proxy_uri && !proxy_uri.empty?
+
+      port = proxy_uri.port
+
+      host_spec = ""
+      host_spec << proxy_uri.scheme || "http"
+      host_spec << "://"
+      host_spec << "#{proxy_uri.userinfo}@" if proxy_uri.userinfo
+      host_spec << proxy_uri.host
+
+      [host_spec, port]
+    end
+
+    def check_for_key_and_secret
+      if schema_registry_key and !schema_registry_key.empty?
+        if !schema_registry_secret or schema_registry_secret.value.empty?
+          raise LogStash::ConfigurationError, "Setting `schema_registry_secret` is required when `schema_registry_key` is provided."
+        end
+      end
+    end
+
+  end
+end end end

data/lib/logstash/plugin_mixins/kafka/common.rb

@@ -0,0 +1,47 @@
+module LogStash module PluginMixins module Kafka
+  module Common
+
+    def self.included(base)
+      # COMMON CONFIGURATION SUPPORTED BY BOTH PRODUCER/CONSUMER
+
+      # Close idle connections after the number of milliseconds specified by this config.
+      base.config :connections_max_idle_ms, :validate => :number, :default => 540_000 # (9m) Kafka default
+
+      # The period of time in milliseconds after which we force a refresh of metadata even if
+      # we haven't seen any partition leadership changes to proactively discover any new brokers or partitions
+      base.config :metadata_max_age_ms, :validate => :number, :default => 300_000 # (5m) Kafka default
+
+      # The configuration controls the maximum amount of time the client will wait for the response of a request.
+      # If the response is not received before the timeout elapses the client will resend the request if necessary
+      # or fail the request if retries are exhausted.
+      base.config :request_timeout_ms, :validate => :number, :default => 40_000 # Kafka default
+    end
+
+    def set_trustore_keystore_config(props)
+      props.put("ssl.truststore.type", ssl_truststore_type) unless ssl_truststore_type.nil?
+      props.put("ssl.truststore.location", ssl_truststore_location) unless ssl_truststore_location.nil?
+      props.put("ssl.truststore.password", ssl_truststore_password.value) unless ssl_truststore_password.nil?
+
+      # Client auth stuff
+      props.put("ssl.keystore.type", ssl_keystore_type) unless ssl_keystore_type.nil?
+      props.put("ssl.key.password", ssl_key_password.value) unless ssl_key_password.nil?
+      props.put("ssl.keystore.location", ssl_keystore_location) unless ssl_keystore_location.nil?
+      props.put("ssl.keystore.password", ssl_keystore_password.value) unless ssl_keystore_password.nil?
+      props.put("ssl.endpoint.identification.algorithm", ssl_endpoint_identification_algorithm) unless ssl_endpoint_identification_algorithm.nil?
+    end
+
+    def set_sasl_config(props)
+      java.lang.System.setProperty("java.security.auth.login.config", jaas_path) unless jaas_path.nil?
+      java.lang.System.setProperty("java.security.krb5.conf", kerberos_config) unless kerberos_config.nil?
+
+      props.put("sasl.mechanism", sasl_mechanism)
+      if sasl_mechanism == "GSSAPI" && sasl_kerberos_service_name.nil?
+        raise LogStash::ConfigurationError, "sasl_kerberos_service_name must be specified when SASL mechanism is GSSAPI"
+      end
+
+      props.put("sasl.kerberos.service.name", sasl_kerberos_service_name) unless sasl_kerberos_service_name.nil?
+      props.put("sasl.jaas.config", sasl_jaas_config) unless sasl_jaas_config.nil?
+    end
+
+  end
+end end end

data/logstash-integration-kafka.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-integration-kafka'
-  s.version         = '10.8.2'
+  s.version         = '10.11.0'
   s.licenses        = ['Apache-2.0']
   s.summary         = "Integration with Kafka - input and output plugins"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline "+
@@ -50,6 +50,7 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'logstash-mixin-deprecation_logger_support', '~>1.0'
 
   s.add_development_dependency 'logstash-devutils'
+  s.add_development_dependency 'logstash-codec-line'
   s.add_development_dependency 'rspec-wait'
   s.add_development_dependency 'digest-crc', '~> 0.5.1' # 0.6.0 started using a C-ext
   s.add_development_dependency 'ruby-kafka' # depends on digest-crc

data/spec/integration/inputs/kafka_spec.rb

@@ -16,38 +16,38 @@ describe "inputs/kafka", :integration => true do
   let(:group_id_5) {rand(36**8).to_s(36)}
   let(:group_id_6) {rand(36**8).to_s(36)}
   let(:plain_config) do
-    { 'topics' => ['logstash_integration_topic_plain'], 'codec' => 'plain', 'group_id' => group_id_1,
+    { 'topics' => ['logstash_integration_topic_plain'], 'group_id' => group_id_1,
       'auto_offset_reset' => 'earliest' }
   end
   let(:multi_consumer_config) do
     plain_config.merge({"group_id" => group_id_4, "client_id" => "spec", "consumer_threads" => 3})
   end
   let(:snappy_config) do
-    { 'topics' => ['logstash_integration_topic_snappy'], 'codec' => 'plain', 'group_id' => group_id_1,
+    { 'topics' => ['logstash_integration_topic_snappy'], 'group_id' => group_id_1,
       'auto_offset_reset' => 'earliest' }
   end
   let(:lz4_config) do
-    { 'topics' => ['logstash_integration_topic_lz4'], 'codec' => 'plain', 'group_id' => group_id_1,
+    { 'topics' => ['logstash_integration_topic_lz4'], 'group_id' => group_id_1,
       'auto_offset_reset' => 'earliest' }
   end
   let(:pattern_config) do
-    { 'topics_pattern' => 'logstash_integration_topic_.*', 'group_id' => group_id_2, 'codec' => 'plain',
+    { 'topics_pattern' => 'logstash_integration_topic_.*', 'group_id' => group_id_2,
       'auto_offset_reset' => 'earliest' }
   end
   let(:decorate_config) do
-    { 'topics' => ['logstash_integration_topic_plain'], 'codec' => 'plain', 'group_id' => group_id_3,
+    { 'topics' => ['logstash_integration_topic_plain'], 'group_id' => group_id_3,
       'auto_offset_reset' => 'earliest', 'decorate_events' => 'true' }
   end
   let(:decorate_headers_config) do
-    { 'topics' => ['logstash_integration_topic_plain_with_headers'], 'codec' => 'plain', 'group_id' => group_id_3,
+    { 'topics' => ['logstash_integration_topic_plain_with_headers'], 'group_id' => group_id_3,
       'auto_offset_reset' => 'earliest', 'decorate_events' => 'extended' }
   end
   let(:decorate_bad_headers_config) do
-    { 'topics' => ['logstash_integration_topic_plain_with_headers_badly'], 'codec' => 'plain', 'group_id' => group_id_3,
+    { 'topics' => ['logstash_integration_topic_plain_with_headers_badly'], 'group_id' => group_id_3,
       'auto_offset_reset' => 'earliest', 'decorate_events' => 'extended' }
   end
   let(:manual_commit_config) do
-    { 'topics' => ['logstash_integration_topic_plain'], 'codec' => 'plain', 'group_id' => group_id_5,
+    { 'topics' => ['logstash_integration_topic_plain'], 'group_id' => group_id_5,
       'auto_offset_reset' => 'earliest', 'enable_auto_commit' => 'false' }
   end
   let(:timeout_seconds) { 30 }
@@ -352,10 +352,7 @@ describe "Deserializing with the schema registry", :integration => true do
 
     let(:base_config) do
       {
-          'topics' => [avro_topic_name],
-          'codec' => 'plain',
-          'group_id' => group_id_1,
-          'auto_offset_reset' => 'earliest'
+          'topics' => [avro_topic_name], 'group_id' => group_id_1, 'auto_offset_reset' => 'earliest'
       }
     end
 

data/spec/integration/outputs/kafka_spec.rb

@@ -28,7 +28,8 @@ describe "outputs/kafka", :integration => true do
     let(:num_events) { 3 }
 
     before :each do
-      config = base_config.merge({"topic_id" => test_topic})
+      # NOTE: the connections_max_idle_ms is irrelevant just testing that configuration works ...
+      config = base_config.merge({"topic_id" => test_topic, "connections_max_idle_ms" => 540_000})
       load_kafka_data(config)
     end
 
@@ -139,6 +140,25 @@ describe "outputs/kafka", :integration => true do
     # end
   end
 
+  context 'when using zstd compression' do
+    let(:test_topic) { 'logstash_integration_zstd_topic' }
+
+    before :each do
+      config = base_config.merge({"topic_id" => test_topic, "compression_type" => "zstd"})
+      load_kafka_data(config)
+    end
+
+    # NOTE: depends on zstd-ruby gem which is using a C-extension
+    # it 'should have data integrity' do
+    #   messages = fetch_messages(test_topic)
+    #
+    #   expect(messages.size).to eq(num_events)
+    #   messages.each do |m|
+    #     expect(m.value).to eq(event.to_s)
+    #   end
+    # end
+  end
+
   context 'when using multi partition topic' do
     let(:num_events) { 100 } # ~ more than (batch.size) 16,384 bytes
     let(:test_topic) { 'logstash_integration_topic3' }

data/spec/unit/inputs/kafka_spec.rb

@@ -177,7 +177,23 @@ describe LogStash::Inputs::Kafka do
     end
   end
 
-  describe "schema registry parameter verification" do
+  it 'uses plain codec by default' do
+    expect( subject.codec ).to respond_to :decode
+    expect( subject.codec.class ).to be LogStash::Codecs::Plain
+  end
+
+  context 'with codec option' do
+
+    let(:config) { super().merge 'codec' => 'line' }
+
+    it 'uses specified codec' do
+      expect( subject.codec ).to respond_to :decode
+      expect( subject.codec.class ).to be LogStash::Codecs::Line
+    end
+
+  end
+
+  describe "schema registry" do
     let(:base_config) do {
           'schema_registry_url' => 'http://localhost:8081',
           'topics' => ['logstash'],
@@ -186,7 +202,7 @@ describe LogStash::Inputs::Kafka do
     end
 
     context "schema_registry_url" do
-     let(:config) { base_config }
+      let(:config) { base_config }
 
       it "conflict with value_deserializer_class should fail" do
         config['value_deserializer_class'] = 'my.fantasy.Deserializer'
@@ -197,6 +213,11 @@ describe LogStash::Inputs::Kafka do
         config['topics_pattern'] = 'topic_.*'
         expect { subject.register }.to raise_error LogStash::ConfigurationError, /Option schema_registry_url prohibit the customization of topics_pattern/
       end
+
+      it 'switches default codec to json' do
+        expect( subject.codec ).to respond_to :decode
+        expect( subject.codec.class ).to be LogStash::Codecs::JSON
+      end
     end
 
     context 'when kerberos auth is used' do
@@ -204,9 +225,8 @@ describe LogStash::Inputs::Kafka do
         context "with #{protocol}" do
           ['auto', 'skip'].each do |vsr|
             context "when validata_schema_registry is #{vsr}" do
-              let(:config) { base_config.merge({'security_protocol' => protocol,
-                                                'schema_registry_validation' => vsr})
-              }
+              let(:config) { base_config.merge({'security_protocol' => protocol, 'schema_registry_validation' => vsr}) }
+
               it 'skips verification' do
                 expect(subject).not_to receive(:check_for_schema_registry_connectivity_and_subjects)
                 expect { subject.register }.not_to raise_error

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-integration-kafka
 version: !ruby/object:Gem::Version
-  version: 10.8.2
+  version: 10.11.0
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-11-04 00:00:00.000000000 Z
+date: 2022-04-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -154,6 +154,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-codec-line
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -231,8 +245,8 @@ files:
 - lib/logstash-integration-kafka_jars.rb
 - lib/logstash/inputs/kafka.rb
 - lib/logstash/outputs/kafka.rb
-- lib/logstash/plugin_mixins/common.rb
-- lib/logstash/plugin_mixins/kafka_support.rb
+- lib/logstash/plugin_mixins/kafka/avro_schema_registry.rb
+- lib/logstash/plugin_mixins/kafka/common.rb
 - logstash-integration-kafka.gemspec
 - spec/check_docs_spec.rb
 - spec/fixtures/jaas.config

data/lib/logstash/plugin_mixins/common.rb

@@ -1,107 +0,0 @@
-module LogStash
-  module PluginMixins
-    module KafkaAvroSchemaRegistry
-
-      def self.included(base)
-        base.extend(self)
-        base.setup_schema_registry_config
-      end
-
-      def setup_schema_registry_config
-        # Option to set key to access Schema Registry.
-        config :schema_registry_key, :validate => :string
-
-        # Option to set secret to access Schema Registry.
-        config :schema_registry_secret, :validate => :password
-
-        # Option to set the endpoint of the Schema Registry.
-        # This option permit the usage of Avro Kafka deserializer which retrieve the schema of the Avro message from an
-        # instance of schema registry. If this option has value `value_deserializer_class` nor `topics_pattern` could be valued
-        config :schema_registry_url, :validate => :uri
-
-        # Option to set the proxy of the Schema Registry.
-        # This option permits to define a proxy to be used to reach the schema registry service instance.
-        config :schema_registry_proxy, :validate => :uri
-
-        # Option to skip validating the schema registry during registration. This can be useful when using
-        # certificate based auth
-        config :schema_registry_validation, :validate => ['auto', 'skip'], :default => 'auto'
-      end
-
-      def check_schema_registry_parameters
-        if @schema_registry_url
-          check_for_schema_registry_conflicts
-          @schema_registry_proxy_host, @schema_registry_proxy_port  = split_proxy_into_host_and_port(schema_registry_proxy)
-          check_for_key_and_secret
-          check_for_schema_registry_connectivity_and_subjects if schema_registry_validation?
-        end
-      end
-
-      def schema_registry_validation?
-        return false if schema_registry_validation.to_s == 'skip'
-        return false if using_kerberos? # pre-validation doesn't support kerberos
-        
-        true
-      end
-
-      def using_kerberos?
-        security_protocol == "SASL_PLAINTEXT" || security_protocol == "SASL_SSL"
-      end
-
-      private
-      def check_for_schema_registry_conflicts
-        if @value_deserializer_class != LogStash::Inputs::Kafka::DEFAULT_DESERIALIZER_CLASS
-          raise LogStash::ConfigurationError, 'Option schema_registry_url prohibit the customization of value_deserializer_class'
-        end
-        if @topics_pattern && !@topics_pattern.empty?
-         raise LogStash::ConfigurationError, 'Option schema_registry_url prohibit the customization of topics_pattern'
-        end
-      end
-
-      private
-      def check_for_schema_registry_connectivity_and_subjects
-        options = {}
-        if schema_registry_proxy && !schema_registry_proxy.empty?
-          options[:proxy] = schema_registry_proxy.to_s
-        end
-        if schema_registry_key and !schema_registry_key.empty?
-          options[:auth] = {:user => schema_registry_key, :password => schema_registry_secret.value}
-        end
-        client = Manticore::Client.new(options)
-        begin
-          response = client.get(@schema_registry_url.uri.to_s + '/subjects').body
-        rescue Manticore::ManticoreException => e
-          raise LogStash::ConfigurationError.new("Schema registry service doesn't respond, error: #{e.message}")
-        end
-        registered_subjects = JSON.parse response
-        expected_subjects = @topics.map { |t| "#{t}-value"}
-        if (expected_subjects & registered_subjects).size != expected_subjects.size
-          undefined_topic_subjects = expected_subjects - registered_subjects
-          raise LogStash::ConfigurationError, "The schema registry does not contain definitions for required topic subjects: #{undefined_topic_subjects}"
-        end
-      end
-
-      def split_proxy_into_host_and_port(proxy_uri)
-        return nil unless proxy_uri && !proxy_uri.empty?
-
-        port = proxy_uri.port
-
-        host_spec = ""
-        host_spec << proxy_uri.scheme || "http"
-        host_spec << "://"
-        host_spec << "#{proxy_uri.userinfo}@" if proxy_uri.userinfo
-        host_spec << proxy_uri.host
-
-        [host_spec, port]
-      end
-
-      def check_for_key_and_secret
-        if schema_registry_key and !schema_registry_key.empty?
-          if !schema_registry_secret or schema_registry_secret.value.empty?
-            raise LogStash::ConfigurationError, "Setting `schema_registry_secret` is required when `schema_registry_key` is provided."
-          end
-        end
-      end
-    end
-  end
-end

data/lib/logstash/plugin_mixins/kafka_support.rb

@@ -1,29 +0,0 @@
-module LogStash module PluginMixins module KafkaSupport
-
-  def set_trustore_keystore_config(props)
-    props.put("ssl.truststore.type", ssl_truststore_type) unless ssl_truststore_type.nil?
-    props.put("ssl.truststore.location", ssl_truststore_location) unless ssl_truststore_location.nil?
-    props.put("ssl.truststore.password", ssl_truststore_password.value) unless ssl_truststore_password.nil?
-
-    # Client auth stuff
-    props.put("ssl.keystore.type", ssl_keystore_type) unless ssl_keystore_type.nil?
-    props.put("ssl.key.password", ssl_key_password.value) unless ssl_key_password.nil?
-    props.put("ssl.keystore.location", ssl_keystore_location) unless ssl_keystore_location.nil?
-    props.put("ssl.keystore.password", ssl_keystore_password.value) unless ssl_keystore_password.nil?
-    props.put("ssl.endpoint.identification.algorithm", ssl_endpoint_identification_algorithm) unless ssl_endpoint_identification_algorithm.nil?
-  end
-
-  def set_sasl_config(props)
-    java.lang.System.setProperty("java.security.auth.login.config", jaas_path) unless jaas_path.nil?
-    java.lang.System.setProperty("java.security.krb5.conf", kerberos_config) unless kerberos_config.nil?
-
-    props.put("sasl.mechanism", sasl_mechanism)
-    if sasl_mechanism == "GSSAPI" && sasl_kerberos_service_name.nil?
-      raise LogStash::ConfigurationError, "sasl_kerberos_service_name must be specified when SASL mechanism is GSSAPI"
-    end
-
-    props.put("sasl.kerberos.service.name", sasl_kerberos_service_name) unless sasl_kerberos_service_name.nil?
-    props.put("sasl.jaas.config", sasl_jaas_config) unless sasl_jaas_config.nil?
-  end
-
-end end end