logstash-integration-kafka 10.9.0-java → 10.12.1-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: df9c89cdfcc2db6702409ec45ffb8d9f8f2b9274304889df1527e0697ccfcf95
-  data.tar.gz: e907ad2e277d27c8cdbe98ebd203af86a3505a85574d64ace4402607c075a69e
+  metadata.gz: 53d305a9b4c08e49bde08c31edf4005a58b596ce74c03e86327a477bf5a4410c
+  data.tar.gz: 8f717699318bac4ad63f61722de0ac4a61fcc8686269bb26ddee22c8aaa4cddc
 SHA512:
-  metadata.gz: bac93eb957af9028a6efc6e31a66c94818ae61333fa738daa6606abdd325dbea206fea2cd905d891f2b341a7bc983f8eaf5a5471015ac9548bc902f941b4a0d9
-  data.tar.gz: 456739a2409ef5a42f007a23c8d0dbfceb3518e8e65b528c0f48266f2a219c2415a83a507fdab3ba028cbc5493d645c080ce191a0d39d7c1787557abded9a0e1
+  metadata.gz: 0bed7cd4a84af389ee1ffb23d7dd068113268a65c79ecc4fbbffa07f2d3f6a50a61c380873e24b25e2c555f33d7d1e9a34b00e10fdf6a03cab9e09f264ea8684
+  data.tar.gz: 5bf4ff031097a0fe5617760cf5238f125ee00cd029a67bc9d58e4387e7af6d816ac48057dd7c3e37e305adc0159f907d264cfb42177d929c4e9310736aa3cf5d

data/CHANGELOG.md

@@ -1,6 +1,21 @@
+## 10.12.1
+  - Fix: update Avro library on 10.x [#149](https://api.github.com/repos/logstash-plugins/logstash-integration-kafka/pulls/149)
+
+## 10.12.0
+  - bump kafka client to 2.8.1 [#115](https://api.github.com/repos/logstash-plugins/logstash-integration-kafka/pulls/115)
+
+## 10.11.0
+  - Feat: added connections_max_idle_ms setting for output [#118](https://github.com/logstash-plugins/logstash-integration-kafka/pull/118)
+  - Refactor: mixins to follow shared mixin module naming
+
+## 10.10.1
+  - Update CHANGELOG.md [#114](https://api.github.com/repos/logstash-plugins/logstash-integration-kafka/pulls/114)
+
+## 10.10.0
+  - Added config setting to enable 'zstd' compression in the Kafka output [#112](https://github.com/logstash-plugins/logstash-integration-kafka/pull/112)
+
 ## 10.9.0
   - Refactor: leverage codec when using schema registry [#106](https://github.com/logstash-plugins/logstash-integration-kafka/pull/106)
-  
     Previously using `schema_registry_url` parsed the payload as JSON even if `codec => 'plain'` was set, this is no longer the case.  
 
 ## 10.8.2
@@ -87,7 +102,6 @@
   - Fix links in changelog pointing to stand-alone plugin changelogs.
   - Refactor: scope java_import to plugin class
 
-
 ## 10.0.0
   - Initial release of the Kafka Integration Plugin, which combines
     previously-separate Kafka plugins and shared dependencies into a single

data/DEVELOPER.md

@@ -62,7 +62,7 @@ See http://kafka.apache.org/documentation.html#producerconfigs for details about
         kafka {
             topic_id => ... # string (required), The topic to produce the messages to
             broker_list => ... # string (optional), default: "localhost:9092", This is for bootstrapping and the producer will only use it for getting metadata
-            compression_codec => ... # string (optional), one of ["none", "gzip", "snappy"], default: "none"
+            compression_codec => ... # string (optional), one of ["none", "gzip", "snappy", "lz4", "zstd"], default: "none"
             compressed_topics => ... # string (optional), default: "", This parameter allows you to set whether compression should be turned on for particular
             request_required_acks => ... # number (optional), one of [-1, 0, 1], default: 0, This value controls when a produce request is considered completed
             serializer_class => ... # string, (optional) default: "kafka.serializer.StringEncoder", The serializer class for messages. The default encoder takes a byte[] and returns the same byte[]

data/docs/index.asciidoc

@@ -1,7 +1,7 @@
 :plugin: kafka
 :type: integration
 :no_codec:
-:kafka_client: 2.5.1
+:kafka_client: 2.8.1
 
 ///////////////////////////////////////////
 START - GENERATED VARIABLES, DO NOT EDIT!

data/docs/input-kafka.asciidoc

@@ -2,7 +2,7 @@
 :plugin: kafka
 :type: input
 :default_codec: plain
-:kafka_client: 2.5
+:kafka_client: 2.8
 :kafka_client_doc: 25
 
 ///////////////////////////////////////////

data/docs/output-kafka.asciidoc

@@ -2,7 +2,7 @@
 :plugin: kafka
 :type: output
 :default_codec: plain
-:kafka_client: 2.5
+:kafka_client: 2.8
 :kafka_client_doc: 25
 
 ///////////////////////////////////////////
@@ -84,7 +84,8 @@ See the https://kafka.apache.org/{kafka_client_doc}/documentation for more detai
 | <<plugins-{type}s-{plugin}-buffer_memory>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-client_dns_lookup>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-client_id>> |<<string,string>>|No
-| <<plugins-{type}s-{plugin}-compression_type>> |<<string,string>>, one of `["none", "gzip", "snappy", "lz4"]`|No
+| <<plugins-{type}s-{plugin}-compression_type>> |<<string,string>>, one of `["none", "gzip", "snappy", "lz4", "zstd"]`|No
+| <<plugins-{type}s-{plugin}-connections_max_idle_ms>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-jaas_path>> |a valid filesystem path|No
 | <<plugins-{type}s-{plugin}-kerberos_config>> |a valid filesystem path|No
 | <<plugins-{type}s-{plugin}-key_serializer>> |<<string,string>>|No
@@ -193,11 +194,19 @@ ip/port by allowing a logical application name to be included with the request
 [id="plugins-{type}s-{plugin}-compression_type"]
 ===== `compression_type` 
 
-  * Value can be any of: `none`, `gzip`, `snappy`, `lz4`
+  * Value can be any of: `none`, `gzip`, `snappy`, `lz4`, `zstd`
   * Default value is `"none"`
 
 The compression type for all data generated by the producer.
-The default is none (i.e. no compression). Valid values are none, gzip, snappy, or lz4.
+The default is none (meaning no compression). Valid values are none, gzip, snappy, lz4, or zstd.
+
+[id="plugins-{type}s-{plugin}-connections_max_idle_ms"]
+===== `connections_max_idle_ms`
+
+  * Value type is <<number,number>>
+  * Default value is `540000` milliseconds (9 minutes).
+
+Close idle connections after the number of milliseconds specified by this config.
 
 [id="plugins-{type}s-{plugin}-jaas_path"]
 ===== `jaas_path` 

data/lib/logstash/inputs/kafka.rb

@@ -2,12 +2,11 @@ require 'logstash/namespace'
 require 'logstash/inputs/base'
 require 'stud/interval'
 require 'java'
-require 'logstash-integration-kafka_jars.rb'
-require 'logstash/plugin_mixins/kafka_support'
-require 'manticore'
 require "json"
 require "logstash/json"
-require_relative '../plugin_mixins/common'
+require 'logstash-integration-kafka_jars.rb'
+require 'logstash/plugin_mixins/kafka/common'
+require 'logstash/plugin_mixins/kafka/avro_schema_registry'
 require 'logstash/plugin_mixins/deprecation_logger_support'
 
 # This input will read events from a Kafka topic. It uses the 0.10 version of
@@ -57,8 +56,8 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
 
   DEFAULT_DESERIALIZER_CLASS = "org.apache.kafka.common.serialization.StringDeserializer"
 
-  include LogStash::PluginMixins::KafkaSupport
-  include ::LogStash::PluginMixins::KafkaAvroSchemaRegistry
+  include LogStash::PluginMixins::Kafka::Common
+  include LogStash::PluginMixins::Kafka::AvroSchemaRegistry
   include LogStash::PluginMixins::DeprecationLoggerSupport
 
   config_name 'kafka'
@@ -98,8 +97,6 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
   # is to be able to track the source of requests beyond just ip/port by allowing
   # a logical application name to be included.
   config :client_id, :validate => :string, :default => "logstash"
-  # Close idle connections after the number of milliseconds specified by this config.
-  config :connections_max_idle_ms, :validate => :number, :default => 540_000 # (9m) Kafka default
   # Ideally you should have as many threads as the number of partitions for a perfect
   # balance — more threads than partitions means that some threads will be idle
   config :consumer_threads, :validate => :number, :default => 1
@@ -152,9 +149,6 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
   config :max_partition_fetch_bytes, :validate => :number, :default => 1_048_576 # (1MB) Kafka default
   # The maximum number of records returned in a single call to poll().
   config :max_poll_records, :validate => :number, :default => 500 # Kafka default
-  # The period of time in milliseconds after which we force a refresh of metadata even if
-  # we haven't seen any partition leadership changes to proactively discover any new brokers or partitions
-  config :metadata_max_age_ms, :validate => :number, :default => 300_000 # (5m) Kafka default
   # The name of the partition assignment strategy that the client uses to distribute
   # partition ownership amongst consumer instances, supported options are `range`,
   # `round_robin`, `sticky` and `cooperative_sticky`
@@ -167,10 +161,6 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
   # This avoids repeatedly connecting to a host in a tight loop.
   # This backoff applies to all connection attempts by the client to a broker.
   config :reconnect_backoff_ms, :validate => :number, :default => 50 # Kafka default
-  # The configuration controls the maximum amount of time the client will wait for the response of a request.
-  # If the response is not received before the timeout elapses the client will resend the request if necessary
-  # or fail the request if retries are exhausted.
-  config :request_timeout_ms, :validate => :number, :default => 40_000 # Kafka default
   # The amount of time to wait before attempting to retry a failed fetch request
   # to a given topic partition. This avoids repeated fetching-and-failing in a tight loop.
   config :retry_backoff_ms, :validate => :number, :default => 100 # Kafka default

data/lib/logstash/outputs/kafka.rb

@@ -2,7 +2,7 @@ require 'logstash/namespace'
 require 'logstash/outputs/base'
 require 'java'
 require 'logstash-integration-kafka_jars.rb'
-require 'logstash/plugin_mixins/kafka_support'
+require 'logstash/plugin_mixins/kafka/common'
 
 # Write events to a Kafka topic. This uses the Kafka Producer API to write messages to a topic on
 # the broker.
@@ -51,7 +51,7 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
 
   java_import org.apache.kafka.clients.producer.ProducerRecord
 
-  include LogStash::PluginMixins::KafkaSupport
+  include LogStash::PluginMixins::Kafka::Common
 
   declare_threadsafe!
 
@@ -80,8 +80,8 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
   # The total bytes of memory the producer can use to buffer records waiting to be sent to the server.
   config :buffer_memory, :validate => :number, :default => 33_554_432 # (32M) Kafka default
   # The compression type for all data generated by the producer.
-  # The default is none (i.e. no compression). Valid values are none, gzip, or snappy.
-  config :compression_type, :validate => ["none", "gzip", "snappy", "lz4"], :default => "none"
+  # The default is none (i.e. no compression). Valid values are none, gzip, snappy, lz4 or zstd.
+  config :compression_type, :validate => ["none", "gzip", "snappy", "lz4", "zstd"], :default => "none"
   # How DNS lookups should be done. If set to `use_all_dns_ips`, when the lookup returns multiple 
   # IP addresses for a hostname, they will all be attempted to connect to before failing the 
   # connection. If the value is `resolve_canonical_bootstrap_servers_only` each entry will be 
@@ -107,19 +107,12 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
   config :message_key, :validate => :string
   # the timeout setting for initial metadata request to fetch topic metadata.
   config :metadata_fetch_timeout_ms, :validate => :number, :default => 60_000
-  # the max time in milliseconds before a metadata refresh is forced.
-  config :metadata_max_age_ms, :validate => :number, :default => 300_000 # (5m) Kafka default
   # Partitioner to use - can be `default`, `uniform_sticky`, `round_robin` or a fully qualified class name of a custom partitioner.
   config :partitioner, :validate => :string
   # The size of the TCP receive buffer to use when reading data
   config :receive_buffer_bytes, :validate => :number, :default => 32_768 # (32KB) Kafka default
   # The amount of time to wait before attempting to reconnect to a given host when a connection fails.
   config :reconnect_backoff_ms, :validate => :number, :default => 50 # Kafka default
-  # The configuration controls the maximum amount of time the client will wait
-  # for the response of a request. If the response is not received before the timeout
-  # elapses the client will resend the request if necessary or fail the request if
-  # retries are exhausted.
-  config :request_timeout_ms, :validate => :number, :default => 40_000 # (40s) Kafka default
   # The default retry behavior is to retry until successful. To prevent data loss,
   # the use of this setting is discouraged.
   #

data/lib/logstash/plugin_mixins/kafka/avro_schema_registry.rb

@@ -0,0 +1,108 @@
+require 'manticore'
+
+module LogStash module PluginMixins module Kafka
+  module AvroSchemaRegistry
+
+    def self.included(base)
+      base.extend(self)
+      base.setup_schema_registry_config
+    end
+
+    def setup_schema_registry_config
+      # Option to set key to access Schema Registry.
+      config :schema_registry_key, :validate => :string
+
+      # Option to set secret to access Schema Registry.
+      config :schema_registry_secret, :validate => :password
+
+      # Option to set the endpoint of the Schema Registry.
+      # This option permit the usage of Avro Kafka deserializer which retrieve the schema of the Avro message from an
+      # instance of schema registry. If this option has value `value_deserializer_class` nor `topics_pattern` could be valued
+      config :schema_registry_url, :validate => :uri
+
+      # Option to set the proxy of the Schema Registry.
+      # This option permits to define a proxy to be used to reach the schema registry service instance.
+      config :schema_registry_proxy, :validate => :uri
+
+      # Option to skip validating the schema registry during registration. This can be useful when using
+      # certificate based auth
+      config :schema_registry_validation, :validate => ['auto', 'skip'], :default => 'auto'
+    end
+
+    def check_schema_registry_parameters
+      if @schema_registry_url
+        check_for_schema_registry_conflicts
+        @schema_registry_proxy_host, @schema_registry_proxy_port  = split_proxy_into_host_and_port(schema_registry_proxy)
+        check_for_key_and_secret
+        check_for_schema_registry_connectivity_and_subjects if schema_registry_validation?
+      end
+    end
+
+    def schema_registry_validation?
+      return false if schema_registry_validation.to_s == 'skip'
+      return false if using_kerberos? # pre-validation doesn't support kerberos
+
+      true
+    end
+
+    def using_kerberos?
+      security_protocol == "SASL_PLAINTEXT" || security_protocol == "SASL_SSL"
+    end
+
+    private
+    def check_for_schema_registry_conflicts
+      if @value_deserializer_class != LogStash::Inputs::Kafka::DEFAULT_DESERIALIZER_CLASS
+        raise LogStash::ConfigurationError, 'Option schema_registry_url prohibit the customization of value_deserializer_class'
+      end
+      if @topics_pattern && !@topics_pattern.empty?
+       raise LogStash::ConfigurationError, 'Option schema_registry_url prohibit the customization of topics_pattern'
+      end
+    end
+
+    private
+    def check_for_schema_registry_connectivity_and_subjects
+      options = {}
+      if schema_registry_proxy && !schema_registry_proxy.empty?
+        options[:proxy] = schema_registry_proxy.to_s
+      end
+      if schema_registry_key and !schema_registry_key.empty?
+        options[:auth] = {:user => schema_registry_key, :password => schema_registry_secret.value}
+      end
+      client = Manticore::Client.new(options)
+      begin
+        response = client.get(@schema_registry_url.uri.to_s + '/subjects').body
+      rescue Manticore::ManticoreException => e
+        raise LogStash::ConfigurationError.new("Schema registry service doesn't respond, error: #{e.message}")
+      end
+      registered_subjects = JSON.parse response
+      expected_subjects = @topics.map { |t| "#{t}-value"}
+      if (expected_subjects & registered_subjects).size != expected_subjects.size
+        undefined_topic_subjects = expected_subjects - registered_subjects
+        raise LogStash::ConfigurationError, "The schema registry does not contain definitions for required topic subjects: #{undefined_topic_subjects}"
+      end
+    end
+
+    def split_proxy_into_host_and_port(proxy_uri)
+      return nil unless proxy_uri && !proxy_uri.empty?
+
+      port = proxy_uri.port
+
+      host_spec = ""
+      host_spec << proxy_uri.scheme || "http"
+      host_spec << "://"
+      host_spec << "#{proxy_uri.userinfo}@" if proxy_uri.userinfo
+      host_spec << proxy_uri.host
+
+      [host_spec, port]
+    end
+
+    def check_for_key_and_secret
+      if schema_registry_key and !schema_registry_key.empty?
+        if !schema_registry_secret or schema_registry_secret.value.empty?
+          raise LogStash::ConfigurationError, "Setting `schema_registry_secret` is required when `schema_registry_key` is provided."
+        end
+      end
+    end
+
+  end
+end end end

data/lib/logstash/plugin_mixins/kafka/common.rb

@@ -0,0 +1,47 @@
+module LogStash module PluginMixins module Kafka
+  module Common
+
+    def self.included(base)
+      # COMMON CONFIGURATION SUPPORTED BY BOTH PRODUCER/CONSUMER
+
+      # Close idle connections after the number of milliseconds specified by this config.
+      base.config :connections_max_idle_ms, :validate => :number, :default => 540_000 # (9m) Kafka default
+
+      # The period of time in milliseconds after which we force a refresh of metadata even if
+      # we haven't seen any partition leadership changes to proactively discover any new brokers or partitions
+      base.config :metadata_max_age_ms, :validate => :number, :default => 300_000 # (5m) Kafka default
+
+      # The configuration controls the maximum amount of time the client will wait for the response of a request.
+      # If the response is not received before the timeout elapses the client will resend the request if necessary
+      # or fail the request if retries are exhausted.
+      base.config :request_timeout_ms, :validate => :number, :default => 40_000 # Kafka default
+    end
+
+    def set_trustore_keystore_config(props)
+      props.put("ssl.truststore.type", ssl_truststore_type) unless ssl_truststore_type.nil?
+      props.put("ssl.truststore.location", ssl_truststore_location) unless ssl_truststore_location.nil?
+      props.put("ssl.truststore.password", ssl_truststore_password.value) unless ssl_truststore_password.nil?
+
+      # Client auth stuff
+      props.put("ssl.keystore.type", ssl_keystore_type) unless ssl_keystore_type.nil?
+      props.put("ssl.key.password", ssl_key_password.value) unless ssl_key_password.nil?
+      props.put("ssl.keystore.location", ssl_keystore_location) unless ssl_keystore_location.nil?
+      props.put("ssl.keystore.password", ssl_keystore_password.value) unless ssl_keystore_password.nil?
+      props.put("ssl.endpoint.identification.algorithm", ssl_endpoint_identification_algorithm) unless ssl_endpoint_identification_algorithm.nil?
+    end
+
+    def set_sasl_config(props)
+      java.lang.System.setProperty("java.security.auth.login.config", jaas_path) unless jaas_path.nil?
+      java.lang.System.setProperty("java.security.krb5.conf", kerberos_config) unless kerberos_config.nil?
+
+      props.put("sasl.mechanism", sasl_mechanism)
+      if sasl_mechanism == "GSSAPI" && sasl_kerberos_service_name.nil?
+        raise LogStash::ConfigurationError, "sasl_kerberos_service_name must be specified when SASL mechanism is GSSAPI"
+      end
+
+      props.put("sasl.kerberos.service.name", sasl_kerberos_service_name) unless sasl_kerberos_service_name.nil?
+      props.put("sasl.jaas.config", sasl_jaas_config) unless sasl_jaas_config.nil?
+    end
+
+  end
+end end end

data/lib/logstash-integration-kafka_jars.rb

@@ -1,17 +1,17 @@
 # AUTOGENERATED BY THE GRADLE SCRIPT. DO NOT EDIT.
 
 require 'jar_dependencies'
-require_jar('io.confluent', 'kafka-avro-serializer', '5.5.1')
-require_jar('io.confluent', 'kafka-schema-serializer', '5.5.1')
-require_jar('io.confluent', 'common-config', '5.5.1')
-require_jar('org.apache.avro', 'avro', '1.9.2')
-require_jar('io.confluent', 'kafka-schema-registry-client', '5.5.1')
-require_jar('org.apache.kafka', 'kafka_2.12', '2.5.1')
-require_jar('io.confluent', 'common-utils', '5.5.1')
+require_jar('io.confluent', 'kafka-avro-serializer', '6.2.2')
+require_jar('io.confluent', 'kafka-schema-serializer', '6.2.2')
+require_jar('io.confluent', 'common-config', '6.2.2')
+require_jar('org.apache.avro', 'avro', '1.11.3')
+require_jar('io.confluent', 'kafka-schema-registry-client', '6.2.2')
+require_jar('org.apache.kafka', 'kafka_2.12', '2.8.1')
+require_jar('io.confluent', 'common-utils', '6.2.2')
 require_jar('javax.ws.rs', 'javax.ws.rs-api', '2.1.1')
 require_jar('org.glassfish.jersey.core', 'jersey-common', '2.33')
-require_jar('org.apache.kafka', 'kafka-clients', '2.5.1')
-require_jar('com.github.luben', 'zstd-jni', '1.4.4-7')
-require_jar('org.slf4j', 'slf4j-api', '1.7.30')
+require_jar('org.apache.kafka', 'kafka-clients', '2.8.1')
+require_jar('com.github.luben', 'zstd-jni', '1.5.2-2')
+require_jar('org.slf4j', 'slf4j-api', '1.7.36')
 require_jar('org.lz4', 'lz4-java', '1.7.1')
-require_jar('org.xerial.snappy', 'snappy-java', '1.1.7.3')
+require_jar('org.xerial.snappy', 'snappy-java', '1.1.8.4')

data/logstash-integration-kafka.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-integration-kafka'
-  s.version         = '10.9.0'
+  s.version         = '10.12.1'
   s.licenses        = ['Apache-2.0']
   s.summary         = "Integration with Kafka - input and output plugins"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline "+

data/spec/integration/outputs/kafka_spec.rb

@@ -28,7 +28,8 @@ describe "outputs/kafka", :integration => true do
     let(:num_events) { 3 }
 
     before :each do
-      config = base_config.merge({"topic_id" => test_topic})
+      # NOTE: the connections_max_idle_ms is irrelevant just testing that configuration works ...
+      config = base_config.merge({"topic_id" => test_topic, "connections_max_idle_ms" => 540_000})
       load_kafka_data(config)
     end
 
@@ -139,6 +140,25 @@ describe "outputs/kafka", :integration => true do
     # end
   end
 
+  context 'when using zstd compression' do
+    let(:test_topic) { 'logstash_integration_zstd_topic' }
+
+    before :each do
+      config = base_config.merge({"topic_id" => test_topic, "compression_type" => "zstd"})
+      load_kafka_data(config)
+    end
+
+    # NOTE: depends on zstd-ruby gem which is using a C-extension
+    # it 'should have data integrity' do
+    #   messages = fetch_messages(test_topic)
+    #
+    #   expect(messages.size).to eq(num_events)
+    #   messages.each do |m|
+    #     expect(m.value).to eq(event.to_s)
+    #   end
+    # end
+  end
+
   context 'when using multi partition topic' do
     let(:num_events) { 100 } # ~ more than (batch.size) 16,384 bytes
     let(:test_topic) { 'logstash_integration_topic3' }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-integration-kafka
 version: !ruby/object:Gem::Version
-  version: 10.9.0
+  version: 10.12.1
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-12-14 00:00:00.000000000 Z
+date: 2023-10-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -245,8 +245,8 @@ files:
 - lib/logstash-integration-kafka_jars.rb
 - lib/logstash/inputs/kafka.rb
 - lib/logstash/outputs/kafka.rb
-- lib/logstash/plugin_mixins/common.rb
-- lib/logstash/plugin_mixins/kafka_support.rb
+- lib/logstash/plugin_mixins/kafka/avro_schema_registry.rb
+- lib/logstash/plugin_mixins/kafka/common.rb
 - logstash-integration-kafka.gemspec
 - spec/check_docs_spec.rb
 - spec/fixtures/jaas.config
@@ -257,20 +257,20 @@ files:
 - spec/unit/inputs/avro_schema_fixture_payment.asvc
 - spec/unit/inputs/kafka_spec.rb
 - spec/unit/outputs/kafka_spec.rb
-- vendor/jar-dependencies/com/github/luben/zstd-jni/1.4.4-7/zstd-jni-1.4.4-7.jar
-- vendor/jar-dependencies/io/confluent/common-config/5.5.1/common-config-5.5.1.jar
-- vendor/jar-dependencies/io/confluent/common-utils/5.5.1/common-utils-5.5.1.jar
-- vendor/jar-dependencies/io/confluent/kafka-avro-serializer/5.5.1/kafka-avro-serializer-5.5.1.jar
-- vendor/jar-dependencies/io/confluent/kafka-schema-registry-client/5.5.1/kafka-schema-registry-client-5.5.1.jar
-- vendor/jar-dependencies/io/confluent/kafka-schema-serializer/5.5.1/kafka-schema-serializer-5.5.1.jar
+- vendor/jar-dependencies/com/github/luben/zstd-jni/1.5.2-2/zstd-jni-1.5.2-2.jar
+- vendor/jar-dependencies/io/confluent/common-config/6.2.2/common-config-6.2.2.jar
+- vendor/jar-dependencies/io/confluent/common-utils/6.2.2/common-utils-6.2.2.jar
+- vendor/jar-dependencies/io/confluent/kafka-avro-serializer/6.2.2/kafka-avro-serializer-6.2.2.jar
+- vendor/jar-dependencies/io/confluent/kafka-schema-registry-client/6.2.2/kafka-schema-registry-client-6.2.2.jar
+- vendor/jar-dependencies/io/confluent/kafka-schema-serializer/6.2.2/kafka-schema-serializer-6.2.2.jar
 - vendor/jar-dependencies/javax/ws/rs/javax.ws.rs-api/2.1.1/javax.ws.rs-api-2.1.1.jar
-- vendor/jar-dependencies/org/apache/avro/avro/1.9.2/avro-1.9.2.jar
-- vendor/jar-dependencies/org/apache/kafka/kafka-clients/2.5.1/kafka-clients-2.5.1.jar
-- vendor/jar-dependencies/org/apache/kafka/kafka_2.12/2.5.1/kafka_2.12-2.5.1.jar
+- vendor/jar-dependencies/org/apache/avro/avro/1.11.3/avro-1.11.3.jar
+- vendor/jar-dependencies/org/apache/kafka/kafka-clients/2.8.1/kafka-clients-2.8.1.jar
+- vendor/jar-dependencies/org/apache/kafka/kafka_2.12/2.8.1/kafka_2.12-2.8.1.jar
 - vendor/jar-dependencies/org/glassfish/jersey/core/jersey-common/2.33/jersey-common-2.33.jar
 - vendor/jar-dependencies/org/lz4/lz4-java/1.7.1/lz4-java-1.7.1.jar
-- vendor/jar-dependencies/org/slf4j/slf4j-api/1.7.30/slf4j-api-1.7.30.jar
-- vendor/jar-dependencies/org/xerial/snappy/snappy-java/1.1.7.3/snappy-java-1.1.7.3.jar
+- vendor/jar-dependencies/org/slf4j/slf4j-api/1.7.36/slf4j-api-1.7.36.jar
+- vendor/jar-dependencies/org/xerial/snappy/snappy-java/1.1.8.4/snappy-java-1.1.8.4.jar
 homepage: http://www.elastic.co/guide/en/logstash/current/index.html
 licenses:
 - Apache-2.0
@@ -294,7 +294,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.6
+rubygems_version: 3.2.33
 signing_key:
 specification_version: 4
 summary: Integration with Kafka - input and output plugins

data/lib/logstash/plugin_mixins/common.rb

@@ -1,107 +0,0 @@
-module LogStash
-  module PluginMixins
-    module KafkaAvroSchemaRegistry
-
-      def self.included(base)
-        base.extend(self)
-        base.setup_schema_registry_config
-      end
-
-      def setup_schema_registry_config
-        # Option to set key to access Schema Registry.
-        config :schema_registry_key, :validate => :string
-
-        # Option to set secret to access Schema Registry.
-        config :schema_registry_secret, :validate => :password
-
-        # Option to set the endpoint of the Schema Registry.
-        # This option permit the usage of Avro Kafka deserializer which retrieve the schema of the Avro message from an
-        # instance of schema registry. If this option has value `value_deserializer_class` nor `topics_pattern` could be valued
-        config :schema_registry_url, :validate => :uri
-
-        # Option to set the proxy of the Schema Registry.
-        # This option permits to define a proxy to be used to reach the schema registry service instance.
-        config :schema_registry_proxy, :validate => :uri
-
-        # Option to skip validating the schema registry during registration. This can be useful when using
-        # certificate based auth
-        config :schema_registry_validation, :validate => ['auto', 'skip'], :default => 'auto'
-      end
-
-      def check_schema_registry_parameters
-        if @schema_registry_url
-          check_for_schema_registry_conflicts
-          @schema_registry_proxy_host, @schema_registry_proxy_port  = split_proxy_into_host_and_port(schema_registry_proxy)
-          check_for_key_and_secret
-          check_for_schema_registry_connectivity_and_subjects if schema_registry_validation?
-        end
-      end
-
-      def schema_registry_validation?
-        return false if schema_registry_validation.to_s == 'skip'
-        return false if using_kerberos? # pre-validation doesn't support kerberos
-        
-        true
-      end
-
-      def using_kerberos?
-        security_protocol == "SASL_PLAINTEXT" || security_protocol == "SASL_SSL"
-      end
-
-      private
-      def check_for_schema_registry_conflicts
-        if @value_deserializer_class != LogStash::Inputs::Kafka::DEFAULT_DESERIALIZER_CLASS
-          raise LogStash::ConfigurationError, 'Option schema_registry_url prohibit the customization of value_deserializer_class'
-        end
-        if @topics_pattern && !@topics_pattern.empty?
-         raise LogStash::ConfigurationError, 'Option schema_registry_url prohibit the customization of topics_pattern'
-        end
-      end
-
-      private
-      def check_for_schema_registry_connectivity_and_subjects
-        options = {}
-        if schema_registry_proxy && !schema_registry_proxy.empty?
-          options[:proxy] = schema_registry_proxy.to_s
-        end
-        if schema_registry_key and !schema_registry_key.empty?
-          options[:auth] = {:user => schema_registry_key, :password => schema_registry_secret.value}
-        end
-        client = Manticore::Client.new(options)
-        begin
-          response = client.get(@schema_registry_url.uri.to_s + '/subjects').body
-        rescue Manticore::ManticoreException => e
-          raise LogStash::ConfigurationError.new("Schema registry service doesn't respond, error: #{e.message}")
-        end
-        registered_subjects = JSON.parse response
-        expected_subjects = @topics.map { |t| "#{t}-value"}
-        if (expected_subjects & registered_subjects).size != expected_subjects.size
-          undefined_topic_subjects = expected_subjects - registered_subjects
-          raise LogStash::ConfigurationError, "The schema registry does not contain definitions for required topic subjects: #{undefined_topic_subjects}"
-        end
-      end
-
-      def split_proxy_into_host_and_port(proxy_uri)
-        return nil unless proxy_uri && !proxy_uri.empty?
-
-        port = proxy_uri.port
-
-        host_spec = ""
-        host_spec << proxy_uri.scheme || "http"
-        host_spec << "://"
-        host_spec << "#{proxy_uri.userinfo}@" if proxy_uri.userinfo
-        host_spec << proxy_uri.host
-
-        [host_spec, port]
-      end
-
-      def check_for_key_and_secret
-        if schema_registry_key and !schema_registry_key.empty?
-          if !schema_registry_secret or schema_registry_secret.value.empty?
-            raise LogStash::ConfigurationError, "Setting `schema_registry_secret` is required when `schema_registry_key` is provided."
-          end
-        end
-      end
-    end
-  end
-end

data/lib/logstash/plugin_mixins/kafka_support.rb

@@ -1,29 +0,0 @@
-module LogStash module PluginMixins module KafkaSupport
-
-  def set_trustore_keystore_config(props)
-    props.put("ssl.truststore.type", ssl_truststore_type) unless ssl_truststore_type.nil?
-    props.put("ssl.truststore.location", ssl_truststore_location) unless ssl_truststore_location.nil?
-    props.put("ssl.truststore.password", ssl_truststore_password.value) unless ssl_truststore_password.nil?
-
-    # Client auth stuff
-    props.put("ssl.keystore.type", ssl_keystore_type) unless ssl_keystore_type.nil?
-    props.put("ssl.key.password", ssl_key_password.value) unless ssl_key_password.nil?
-    props.put("ssl.keystore.location", ssl_keystore_location) unless ssl_keystore_location.nil?
-    props.put("ssl.keystore.password", ssl_keystore_password.value) unless ssl_keystore_password.nil?
-    props.put("ssl.endpoint.identification.algorithm", ssl_endpoint_identification_algorithm) unless ssl_endpoint_identification_algorithm.nil?
-  end
-
-  def set_sasl_config(props)
-    java.lang.System.setProperty("java.security.auth.login.config", jaas_path) unless jaas_path.nil?
-    java.lang.System.setProperty("java.security.krb5.conf", kerberos_config) unless kerberos_config.nil?
-
-    props.put("sasl.mechanism", sasl_mechanism)
-    if sasl_mechanism == "GSSAPI" && sasl_kerberos_service_name.nil?
-      raise LogStash::ConfigurationError, "sasl_kerberos_service_name must be specified when SASL mechanism is GSSAPI"
-    end
-
-    props.put("sasl.kerberos.service.name", sasl_kerberos_service_name) unless sasl_kerberos_service_name.nil?
-    props.put("sasl.jaas.config", sasl_jaas_config) unless sasl_jaas_config.nil?
-  end
-
-end end end