logstash-integration-kafka 10.10.0-java → 10.11.0-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
 - data/CHANGELOG.md +7 -3
 - data/docs/output-kafka.asciidoc +9 -0
 - data/lib/logstash/inputs/kafka.rb +5 -15
 - data/lib/logstash/outputs/kafka.rb +2 -9
 - data/lib/logstash/plugin_mixins/kafka/avro_schema_registry.rb +108 -0
 - data/lib/logstash/plugin_mixins/kafka/common.rb +47 -0
 - data/logstash-integration-kafka.gemspec +1 -1
 - data/spec/integration/outputs/kafka_spec.rb +2 -1
 - metadata +4 -4
 - data/lib/logstash/plugin_mixins/common.rb +0 -107
 - data/lib/logstash/plugin_mixins/kafka_support.rb +0 -29
 
    
        checksums.yaml
    CHANGED
    
    | 
         @@ -1,7 +1,7 @@ 
     | 
|
| 
       1 
1 
     | 
    
         
             
            ---
         
     | 
| 
       2 
2 
     | 
    
         
             
            SHA256:
         
     | 
| 
       3 
     | 
    
         
            -
              metadata.gz:  
     | 
| 
       4 
     | 
    
         
            -
              data.tar.gz:  
     | 
| 
      
 3 
     | 
    
         
            +
              metadata.gz: 2833139d65282055a4e924ccfb50b19341a61af69e0834ef318a62f1459f3a16
         
     | 
| 
      
 4 
     | 
    
         
            +
              data.tar.gz: fbfd7e6b94e9a74109b44b838df344ed4d7447d56d624f24e642235eb62b00e3
         
     | 
| 
       5 
5 
     | 
    
         
             
            SHA512:
         
     | 
| 
       6 
     | 
    
         
            -
              metadata.gz:  
     | 
| 
       7 
     | 
    
         
            -
              data.tar.gz:  
     | 
| 
      
 6 
     | 
    
         
            +
              metadata.gz: b35d99bf553fbfd3b78f3f7de62b427b18001853573c5160090bebba5dfb78f4d17aa9f879a94a98805f949b1ed76a3fa82e9c1a7b95815f223a5d04104f5bce
         
     | 
| 
      
 7 
     | 
    
         
            +
              data.tar.gz: 0e8edf4f07b00443c39ac55523fb5663395ba0ae524dfedc5181a56b5160c2c33fe110e24bccbb060b43c6be0aa2211b50422d041fa1dea59561a26efc79d687
         
     | 
    
        data/CHANGELOG.md
    CHANGED
    
    | 
         @@ -1,10 +1,15 @@ 
     | 
|
| 
       1 
     | 
    
         
            -
            ## 10. 
     | 
| 
      
 1 
     | 
    
         
            +
            ## 10.11.0
         
     | 
| 
      
 2 
     | 
    
         
            +
              - Feat: added connections_max_idle_ms setting for output [#118](https://github.com/logstash-plugins/logstash-integration-kafka/pull/118)
         
     | 
| 
      
 3 
     | 
    
         
            +
              - Refactor: mixins to follow shared mixin module naming
         
     | 
| 
      
 4 
     | 
    
         
            +
             
     | 
| 
      
 5 
     | 
    
         
            +
            ## 10.10.1
         
     | 
| 
      
 6 
     | 
    
         
            +
              - Update CHANGELOG.md [#114](https://api.github.com/repos/logstash-plugins/logstash-integration-kafka/pulls/114)
         
     | 
| 
       2 
7 
     | 
    
         | 
| 
      
 8 
     | 
    
         
            +
            ## 10.10.0
         
     | 
| 
       3 
9 
     | 
    
         
             
              - Added config setting to enable 'zstd' compression in the Kafka output [#112](https://github.com/logstash-plugins/logstash-integration-kafka/pull/112)
         
     | 
| 
       4 
10 
     | 
    
         | 
| 
       5 
11 
     | 
    
         
             
            ## 10.9.0
         
     | 
| 
       6 
12 
     | 
    
         
             
              - Refactor: leverage codec when using schema registry [#106](https://github.com/logstash-plugins/logstash-integration-kafka/pull/106)
         
     | 
| 
       7 
     | 
    
         
            -
              
         
     | 
| 
       8 
13 
     | 
    
         
             
                Previously using `schema_registry_url` parsed the payload as JSON even if `codec => 'plain'` was set, this is no longer the case.  
         
     | 
| 
       9 
14 
     | 
    
         | 
| 
       10 
15 
     | 
    
         
             
            ## 10.8.2
         
     | 
| 
         @@ -91,7 +96,6 @@ 
     | 
|
| 
       91 
96 
     | 
    
         
             
              - Fix links in changelog pointing to stand-alone plugin changelogs.
         
     | 
| 
       92 
97 
     | 
    
         
             
              - Refactor: scope java_import to plugin class
         
     | 
| 
       93 
98 
     | 
    
         | 
| 
       94 
     | 
    
         
            -
             
     | 
| 
       95 
99 
     | 
    
         
             
            ## 10.0.0
         
     | 
| 
       96 
100 
     | 
    
         
             
              - Initial release of the Kafka Integration Plugin, which combines
         
     | 
| 
       97 
101 
     | 
    
         
             
                previously-separate Kafka plugins and shared dependencies into a single
         
     | 
    
        data/docs/output-kafka.asciidoc
    CHANGED
    
    | 
         @@ -85,6 +85,7 @@ See the https://kafka.apache.org/{kafka_client_doc}/documentation for more detai 
     | 
|
| 
       85 
85 
     | 
    
         
             
            | <<plugins-{type}s-{plugin}-client_dns_lookup>> |<<string,string>>|No
         
     | 
| 
       86 
86 
     | 
    
         
             
            | <<plugins-{type}s-{plugin}-client_id>> |<<string,string>>|No
         
     | 
| 
       87 
87 
     | 
    
         
             
            | <<plugins-{type}s-{plugin}-compression_type>> |<<string,string>>, one of `["none", "gzip", "snappy", "lz4", "zstd"]`|No
         
     | 
| 
      
 88 
     | 
    
         
            +
            | <<plugins-{type}s-{plugin}-connections_max_idle_ms>> |<<number,number>>|No
         
     | 
| 
       88 
89 
     | 
    
         
             
            | <<plugins-{type}s-{plugin}-jaas_path>> |a valid filesystem path|No
         
     | 
| 
       89 
90 
     | 
    
         
             
            | <<plugins-{type}s-{plugin}-kerberos_config>> |a valid filesystem path|No
         
     | 
| 
       90 
91 
     | 
    
         
             
            | <<plugins-{type}s-{plugin}-key_serializer>> |<<string,string>>|No
         
     | 
| 
         @@ -199,6 +200,14 @@ ip/port by allowing a logical application name to be included with the request 
     | 
|
| 
       199 
200 
     | 
    
         
             
            The compression type for all data generated by the producer.
         
     | 
| 
       200 
201 
     | 
    
         
             
            The default is none (meaning no compression). Valid values are none, gzip, snappy, lz4, or zstd.
         
     | 
| 
       201 
202 
     | 
    
         | 
| 
      
 203 
     | 
    
         
            +
            [id="plugins-{type}s-{plugin}-connections_max_idle_ms"]
         
     | 
| 
      
 204 
     | 
    
         
            +
            ===== `connections_max_idle_ms`
         
     | 
| 
      
 205 
     | 
    
         
            +
             
     | 
| 
      
 206 
     | 
    
         
            +
              * Value type is <<number,number>>
         
     | 
| 
      
 207 
     | 
    
         
            +
              * Default value is `540000` milliseconds (9 minutes).
         
     | 
| 
      
 208 
     | 
    
         
            +
             
     | 
| 
      
 209 
     | 
    
         
            +
            Close idle connections after the number of milliseconds specified by this config.
         
     | 
| 
      
 210 
     | 
    
         
            +
             
     | 
| 
       202 
211 
     | 
    
         
             
            [id="plugins-{type}s-{plugin}-jaas_path"]
         
     | 
| 
       203 
212 
     | 
    
         
             
            ===== `jaas_path` 
         
     | 
| 
       204 
213 
     | 
    
         | 
| 
         @@ -2,12 +2,11 @@ require 'logstash/namespace' 
     | 
|
| 
       2 
2 
     | 
    
         
             
            require 'logstash/inputs/base'
         
     | 
| 
       3 
3 
     | 
    
         
             
            require 'stud/interval'
         
     | 
| 
       4 
4 
     | 
    
         
             
            require 'java'
         
     | 
| 
       5 
     | 
    
         
            -
            require 'logstash-integration-kafka_jars.rb'
         
     | 
| 
       6 
     | 
    
         
            -
            require 'logstash/plugin_mixins/kafka_support'
         
     | 
| 
       7 
     | 
    
         
            -
            require 'manticore'
         
     | 
| 
       8 
5 
     | 
    
         
             
            require "json"
         
     | 
| 
       9 
6 
     | 
    
         
             
            require "logstash/json"
         
     | 
| 
       10 
     | 
    
         
            -
             
     | 
| 
      
 7 
     | 
    
         
            +
            require 'logstash-integration-kafka_jars.rb'
         
     | 
| 
      
 8 
     | 
    
         
            +
            require 'logstash/plugin_mixins/kafka/common'
         
     | 
| 
      
 9 
     | 
    
         
            +
            require 'logstash/plugin_mixins/kafka/avro_schema_registry'
         
     | 
| 
       11 
10 
     | 
    
         
             
            require 'logstash/plugin_mixins/deprecation_logger_support'
         
     | 
| 
       12 
11 
     | 
    
         | 
| 
       13 
12 
     | 
    
         
             
            # This input will read events from a Kafka topic. It uses the 0.10 version of
         
     | 
| 
         @@ -57,8 +56,8 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base 
     | 
|
| 
       57 
56 
     | 
    
         | 
| 
       58 
57 
     | 
    
         
             
              DEFAULT_DESERIALIZER_CLASS = "org.apache.kafka.common.serialization.StringDeserializer"
         
     | 
| 
       59 
58 
     | 
    
         | 
| 
       60 
     | 
    
         
            -
              include LogStash::PluginMixins:: 
     | 
| 
       61 
     | 
    
         
            -
              include  
     | 
| 
      
 59 
     | 
    
         
            +
              include LogStash::PluginMixins::Kafka::Common
         
     | 
| 
      
 60 
     | 
    
         
            +
              include LogStash::PluginMixins::Kafka::AvroSchemaRegistry
         
     | 
| 
       62 
61 
     | 
    
         
             
              include LogStash::PluginMixins::DeprecationLoggerSupport
         
     | 
| 
       63 
62 
     | 
    
         | 
| 
       64 
63 
     | 
    
         
             
              config_name 'kafka'
         
     | 
| 
         @@ -98,8 +97,6 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base 
     | 
|
| 
       98 
97 
     | 
    
         
             
              # is to be able to track the source of requests beyond just ip/port by allowing
         
     | 
| 
       99 
98 
     | 
    
         
             
              # a logical application name to be included.
         
     | 
| 
       100 
99 
     | 
    
         
             
              config :client_id, :validate => :string, :default => "logstash"
         
     | 
| 
       101 
     | 
    
         
            -
              # Close idle connections after the number of milliseconds specified by this config.
         
     | 
| 
       102 
     | 
    
         
            -
              config :connections_max_idle_ms, :validate => :number, :default => 540_000 # (9m) Kafka default
         
     | 
| 
       103 
100 
     | 
    
         
             
              # Ideally you should have as many threads as the number of partitions for a perfect
         
     | 
| 
       104 
101 
     | 
    
         
             
              # balance — more threads than partitions means that some threads will be idle
         
     | 
| 
       105 
102 
     | 
    
         
             
              config :consumer_threads, :validate => :number, :default => 1
         
     | 
| 
         @@ -152,9 +149,6 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base 
     | 
|
| 
       152 
149 
     | 
    
         
             
              config :max_partition_fetch_bytes, :validate => :number, :default => 1_048_576 # (1MB) Kafka default
         
     | 
| 
       153 
150 
     | 
    
         
             
              # The maximum number of records returned in a single call to poll().
         
     | 
| 
       154 
151 
     | 
    
         
             
              config :max_poll_records, :validate => :number, :default => 500 # Kafka default
         
     | 
| 
       155 
     | 
    
         
            -
              # The period of time in milliseconds after which we force a refresh of metadata even if
         
     | 
| 
       156 
     | 
    
         
            -
              # we haven't seen any partition leadership changes to proactively discover any new brokers or partitions
         
     | 
| 
       157 
     | 
    
         
            -
              config :metadata_max_age_ms, :validate => :number, :default => 300_000 # (5m) Kafka default
         
     | 
| 
       158 
152 
     | 
    
         
             
              # The name of the partition assignment strategy that the client uses to distribute
         
     | 
| 
       159 
153 
     | 
    
         
             
              # partition ownership amongst consumer instances, supported options are `range`,
         
     | 
| 
       160 
154 
     | 
    
         
             
              # `round_robin`, `sticky` and `cooperative_sticky`
         
     | 
| 
         @@ -167,10 +161,6 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base 
     | 
|
| 
       167 
161 
     | 
    
         
             
              # This avoids repeatedly connecting to a host in a tight loop.
         
     | 
| 
       168 
162 
     | 
    
         
             
              # This backoff applies to all connection attempts by the client to a broker.
         
     | 
| 
       169 
163 
     | 
    
         
             
              config :reconnect_backoff_ms, :validate => :number, :default => 50 # Kafka default
         
     | 
| 
       170 
     | 
    
         
            -
              # The configuration controls the maximum amount of time the client will wait for the response of a request.
         
     | 
| 
       171 
     | 
    
         
            -
              # If the response is not received before the timeout elapses the client will resend the request if necessary
         
     | 
| 
       172 
     | 
    
         
            -
              # or fail the request if retries are exhausted.
         
     | 
| 
       173 
     | 
    
         
            -
              config :request_timeout_ms, :validate => :number, :default => 40_000 # Kafka default
         
     | 
| 
       174 
164 
     | 
    
         
             
              # The amount of time to wait before attempting to retry a failed fetch request
         
     | 
| 
       175 
165 
     | 
    
         
             
              # to a given topic partition. This avoids repeated fetching-and-failing in a tight loop.
         
     | 
| 
       176 
166 
     | 
    
         
             
              config :retry_backoff_ms, :validate => :number, :default => 100 # Kafka default
         
     | 
| 
         @@ -2,7 +2,7 @@ require 'logstash/namespace' 
     | 
|
| 
       2 
2 
     | 
    
         
             
            require 'logstash/outputs/base'
         
     | 
| 
       3 
3 
     | 
    
         
             
            require 'java'
         
     | 
| 
       4 
4 
     | 
    
         
             
            require 'logstash-integration-kafka_jars.rb'
         
     | 
| 
       5 
     | 
    
         
            -
            require 'logstash/plugin_mixins/ 
     | 
| 
      
 5 
     | 
    
         
            +
            require 'logstash/plugin_mixins/kafka/common'
         
     | 
| 
       6 
6 
     | 
    
         | 
| 
       7 
7 
     | 
    
         
             
            # Write events to a Kafka topic. This uses the Kafka Producer API to write messages to a topic on
         
     | 
| 
       8 
8 
     | 
    
         
             
            # the broker.
         
     | 
| 
         @@ -51,7 +51,7 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base 
     | 
|
| 
       51 
51 
     | 
    
         | 
| 
       52 
52 
     | 
    
         
             
              java_import org.apache.kafka.clients.producer.ProducerRecord
         
     | 
| 
       53 
53 
     | 
    
         | 
| 
       54 
     | 
    
         
            -
              include LogStash::PluginMixins:: 
     | 
| 
      
 54 
     | 
    
         
            +
              include LogStash::PluginMixins::Kafka::Common
         
     | 
| 
       55 
55 
     | 
    
         | 
| 
       56 
56 
     | 
    
         
             
              declare_threadsafe!
         
     | 
| 
       57 
57 
     | 
    
         | 
| 
         @@ -107,19 +107,12 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base 
     | 
|
| 
       107 
107 
     | 
    
         
             
              config :message_key, :validate => :string
         
     | 
| 
       108 
108 
     | 
    
         
             
              # the timeout setting for initial metadata request to fetch topic metadata.
         
     | 
| 
       109 
109 
     | 
    
         
             
              config :metadata_fetch_timeout_ms, :validate => :number, :default => 60_000
         
     | 
| 
       110 
     | 
    
         
            -
              # the max time in milliseconds before a metadata refresh is forced.
         
     | 
| 
       111 
     | 
    
         
            -
              config :metadata_max_age_ms, :validate => :number, :default => 300_000 # (5m) Kafka default
         
     | 
| 
       112 
110 
     | 
    
         
             
              # Partitioner to use - can be `default`, `uniform_sticky`, `round_robin` or a fully qualified class name of a custom partitioner.
         
     | 
| 
       113 
111 
     | 
    
         
             
              config :partitioner, :validate => :string
         
     | 
| 
       114 
112 
     | 
    
         
             
              # The size of the TCP receive buffer to use when reading data
         
     | 
| 
       115 
113 
     | 
    
         
             
              config :receive_buffer_bytes, :validate => :number, :default => 32_768 # (32KB) Kafka default
         
     | 
| 
       116 
114 
     | 
    
         
             
              # The amount of time to wait before attempting to reconnect to a given host when a connection fails.
         
     | 
| 
       117 
115 
     | 
    
         
             
              config :reconnect_backoff_ms, :validate => :number, :default => 50 # Kafka default
         
     | 
| 
       118 
     | 
    
         
            -
              # The configuration controls the maximum amount of time the client will wait
         
     | 
| 
       119 
     | 
    
         
            -
              # for the response of a request. If the response is not received before the timeout
         
     | 
| 
       120 
     | 
    
         
            -
              # elapses the client will resend the request if necessary or fail the request if
         
     | 
| 
       121 
     | 
    
         
            -
              # retries are exhausted.
         
     | 
| 
       122 
     | 
    
         
            -
              config :request_timeout_ms, :validate => :number, :default => 40_000 # (40s) Kafka default
         
     | 
| 
       123 
116 
     | 
    
         
             
              # The default retry behavior is to retry until successful. To prevent data loss,
         
     | 
| 
       124 
117 
     | 
    
         
             
              # the use of this setting is discouraged.
         
     | 
| 
       125 
118 
     | 
    
         
             
              #
         
     | 
| 
         @@ -0,0 +1,108 @@ 
     | 
|
| 
      
 1 
     | 
    
         
            +
            require 'manticore'
         
     | 
| 
      
 2 
     | 
    
         
            +
             
     | 
| 
      
 3 
     | 
    
         
            +
            module LogStash module PluginMixins module Kafka
         
     | 
| 
      
 4 
     | 
    
         
            +
              module AvroSchemaRegistry
         
     | 
| 
      
 5 
     | 
    
         
            +
             
     | 
| 
      
 6 
     | 
    
         
            +
                def self.included(base)
         
     | 
| 
      
 7 
     | 
    
         
            +
                  base.extend(self)
         
     | 
| 
      
 8 
     | 
    
         
            +
                  base.setup_schema_registry_config
         
     | 
| 
      
 9 
     | 
    
         
            +
                end
         
     | 
| 
      
 10 
     | 
    
         
            +
             
     | 
| 
      
 11 
     | 
    
         
            +
                def setup_schema_registry_config
         
     | 
| 
      
 12 
     | 
    
         
            +
                  # Option to set key to access Schema Registry.
         
     | 
| 
      
 13 
     | 
    
         
            +
                  config :schema_registry_key, :validate => :string
         
     | 
| 
      
 14 
     | 
    
         
            +
             
     | 
| 
      
 15 
     | 
    
         
            +
                  # Option to set secret to access Schema Registry.
         
     | 
| 
      
 16 
     | 
    
         
            +
                  config :schema_registry_secret, :validate => :password
         
     | 
| 
      
 17 
     | 
    
         
            +
             
     | 
| 
      
 18 
     | 
    
         
            +
                  # Option to set the endpoint of the Schema Registry.
         
     | 
| 
      
 19 
     | 
    
         
            +
                  # This option permit the usage of Avro Kafka deserializer which retrieve the schema of the Avro message from an
         
     | 
| 
      
 20 
     | 
    
         
            +
                  # instance of schema registry. If this option has value `value_deserializer_class` nor `topics_pattern` could be valued
         
     | 
| 
      
 21 
     | 
    
         
            +
                  config :schema_registry_url, :validate => :uri
         
     | 
| 
      
 22 
     | 
    
         
            +
             
     | 
| 
      
 23 
     | 
    
         
            +
                  # Option to set the proxy of the Schema Registry.
         
     | 
| 
      
 24 
     | 
    
         
            +
                  # This option permits to define a proxy to be used to reach the schema registry service instance.
         
     | 
| 
      
 25 
     | 
    
         
            +
                  config :schema_registry_proxy, :validate => :uri
         
     | 
| 
      
 26 
     | 
    
         
            +
             
     | 
| 
      
 27 
     | 
    
         
            +
                  # Option to skip validating the schema registry during registration. This can be useful when using
         
     | 
| 
      
 28 
     | 
    
         
            +
                  # certificate based auth
         
     | 
| 
      
 29 
     | 
    
         
            +
                  config :schema_registry_validation, :validate => ['auto', 'skip'], :default => 'auto'
         
     | 
| 
      
 30 
     | 
    
         
            +
                end
         
     | 
| 
      
 31 
     | 
    
         
            +
             
     | 
| 
      
 32 
     | 
    
         
            +
                def check_schema_registry_parameters
         
     | 
| 
      
 33 
     | 
    
         
            +
                  if @schema_registry_url
         
     | 
| 
      
 34 
     | 
    
         
            +
                    check_for_schema_registry_conflicts
         
     | 
| 
      
 35 
     | 
    
         
            +
                    @schema_registry_proxy_host, @schema_registry_proxy_port  = split_proxy_into_host_and_port(schema_registry_proxy)
         
     | 
| 
      
 36 
     | 
    
         
            +
                    check_for_key_and_secret
         
     | 
| 
      
 37 
     | 
    
         
            +
                    check_for_schema_registry_connectivity_and_subjects if schema_registry_validation?
         
     | 
| 
      
 38 
     | 
    
         
            +
                  end
         
     | 
| 
      
 39 
     | 
    
         
            +
                end
         
     | 
| 
      
 40 
     | 
    
         
            +
             
     | 
| 
      
 41 
     | 
    
         
            +
                def schema_registry_validation?
         
     | 
| 
      
 42 
     | 
    
         
            +
                  return false if schema_registry_validation.to_s == 'skip'
         
     | 
| 
      
 43 
     | 
    
         
            +
                  return false if using_kerberos? # pre-validation doesn't support kerberos
         
     | 
| 
      
 44 
     | 
    
         
            +
             
     | 
| 
      
 45 
     | 
    
         
            +
                  true
         
     | 
| 
      
 46 
     | 
    
         
            +
                end
         
     | 
| 
      
 47 
     | 
    
         
            +
             
     | 
| 
      
 48 
     | 
    
         
            +
                def using_kerberos?
         
     | 
| 
      
 49 
     | 
    
         
            +
                  security_protocol == "SASL_PLAINTEXT" || security_protocol == "SASL_SSL"
         
     | 
| 
      
 50 
     | 
    
         
            +
                end
         
     | 
| 
      
 51 
     | 
    
         
            +
             
     | 
| 
      
 52 
     | 
    
         
            +
                private
         
     | 
| 
      
 53 
     | 
    
         
            +
                def check_for_schema_registry_conflicts
         
     | 
| 
      
 54 
     | 
    
         
            +
                  if @value_deserializer_class != LogStash::Inputs::Kafka::DEFAULT_DESERIALIZER_CLASS
         
     | 
| 
      
 55 
     | 
    
         
            +
                    raise LogStash::ConfigurationError, 'Option schema_registry_url prohibit the customization of value_deserializer_class'
         
     | 
| 
      
 56 
     | 
    
         
            +
                  end
         
     | 
| 
      
 57 
     | 
    
         
            +
                  if @topics_pattern && !@topics_pattern.empty?
         
     | 
| 
      
 58 
     | 
    
         
            +
                   raise LogStash::ConfigurationError, 'Option schema_registry_url prohibit the customization of topics_pattern'
         
     | 
| 
      
 59 
     | 
    
         
            +
                  end
         
     | 
| 
      
 60 
     | 
    
         
            +
                end
         
     | 
| 
      
 61 
     | 
    
         
            +
             
     | 
| 
      
 62 
     | 
    
         
            +
                private
         
     | 
| 
      
 63 
     | 
    
         
            +
                def check_for_schema_registry_connectivity_and_subjects
         
     | 
| 
      
 64 
     | 
    
         
            +
                  options = {}
         
     | 
| 
      
 65 
     | 
    
         
            +
                  if schema_registry_proxy && !schema_registry_proxy.empty?
         
     | 
| 
      
 66 
     | 
    
         
            +
                    options[:proxy] = schema_registry_proxy.to_s
         
     | 
| 
      
 67 
     | 
    
         
            +
                  end
         
     | 
| 
      
 68 
     | 
    
         
            +
                  if schema_registry_key and !schema_registry_key.empty?
         
     | 
| 
      
 69 
     | 
    
         
            +
                    options[:auth] = {:user => schema_registry_key, :password => schema_registry_secret.value}
         
     | 
| 
      
 70 
     | 
    
         
            +
                  end
         
     | 
| 
      
 71 
     | 
    
         
            +
                  client = Manticore::Client.new(options)
         
     | 
| 
      
 72 
     | 
    
         
            +
                  begin
         
     | 
| 
      
 73 
     | 
    
         
            +
                    response = client.get(@schema_registry_url.uri.to_s + '/subjects').body
         
     | 
| 
      
 74 
     | 
    
         
            +
                  rescue Manticore::ManticoreException => e
         
     | 
| 
      
 75 
     | 
    
         
            +
                    raise LogStash::ConfigurationError.new("Schema registry service doesn't respond, error: #{e.message}")
         
     | 
| 
      
 76 
     | 
    
         
            +
                  end
         
     | 
| 
      
 77 
     | 
    
         
            +
                  registered_subjects = JSON.parse response
         
     | 
| 
      
 78 
     | 
    
         
            +
                  expected_subjects = @topics.map { |t| "#{t}-value"}
         
     | 
| 
      
 79 
     | 
    
         
            +
                  if (expected_subjects & registered_subjects).size != expected_subjects.size
         
     | 
| 
      
 80 
     | 
    
         
            +
                    undefined_topic_subjects = expected_subjects - registered_subjects
         
     | 
| 
      
 81 
     | 
    
         
            +
                    raise LogStash::ConfigurationError, "The schema registry does not contain definitions for required topic subjects: #{undefined_topic_subjects}"
         
     | 
| 
      
 82 
     | 
    
         
            +
                  end
         
     | 
| 
      
 83 
     | 
    
         
            +
                end
         
     | 
| 
      
 84 
     | 
    
         
            +
             
     | 
| 
      
 85 
     | 
    
         
            +
                def split_proxy_into_host_and_port(proxy_uri)
         
     | 
| 
      
 86 
     | 
    
         
            +
                  return nil unless proxy_uri && !proxy_uri.empty?
         
     | 
| 
      
 87 
     | 
    
         
            +
             
     | 
| 
      
 88 
     | 
    
         
            +
                  port = proxy_uri.port
         
     | 
| 
      
 89 
     | 
    
         
            +
             
     | 
| 
      
 90 
     | 
    
         
            +
                  host_spec = ""
         
     | 
| 
      
 91 
     | 
    
         
            +
                  host_spec << proxy_uri.scheme || "http"
         
     | 
| 
      
 92 
     | 
    
         
            +
                  host_spec << "://"
         
     | 
| 
      
 93 
     | 
    
         
            +
                  host_spec << "#{proxy_uri.userinfo}@" if proxy_uri.userinfo
         
     | 
| 
      
 94 
     | 
    
         
            +
                  host_spec << proxy_uri.host
         
     | 
| 
      
 95 
     | 
    
         
            +
             
     | 
| 
      
 96 
     | 
    
         
            +
                  [host_spec, port]
         
     | 
| 
      
 97 
     | 
    
         
            +
                end
         
     | 
| 
      
 98 
     | 
    
         
            +
             
     | 
| 
      
 99 
     | 
    
         
            +
                def check_for_key_and_secret
         
     | 
| 
      
 100 
     | 
    
         
            +
                  if schema_registry_key and !schema_registry_key.empty?
         
     | 
| 
      
 101 
     | 
    
         
            +
                    if !schema_registry_secret or schema_registry_secret.value.empty?
         
     | 
| 
      
 102 
     | 
    
         
            +
                      raise LogStash::ConfigurationError, "Setting `schema_registry_secret` is required when `schema_registry_key` is provided."
         
     | 
| 
      
 103 
     | 
    
         
            +
                    end
         
     | 
| 
      
 104 
     | 
    
         
            +
                  end
         
     | 
| 
      
 105 
     | 
    
         
            +
                end
         
     | 
| 
      
 106 
     | 
    
         
            +
             
     | 
| 
      
 107 
     | 
    
         
            +
              end
         
     | 
| 
      
 108 
     | 
    
         
            +
            end end end
         
     | 
| 
         @@ -0,0 +1,47 @@ 
     | 
|
| 
      
 1 
     | 
    
         
            +
            module LogStash module PluginMixins module Kafka
         
     | 
| 
      
 2 
     | 
    
         
            +
              module Common
         
     | 
| 
      
 3 
     | 
    
         
            +
             
     | 
| 
      
 4 
     | 
    
         
            +
                def self.included(base)
         
     | 
| 
      
 5 
     | 
    
         
            +
                  # COMMON CONFIGURATION SUPPORTED BY BOTH PRODUCER/CONSUMER
         
     | 
| 
      
 6 
     | 
    
         
            +
             
     | 
| 
      
 7 
     | 
    
         
            +
                  # Close idle connections after the number of milliseconds specified by this config.
         
     | 
| 
      
 8 
     | 
    
         
            +
                  base.config :connections_max_idle_ms, :validate => :number, :default => 540_000 # (9m) Kafka default
         
     | 
| 
      
 9 
     | 
    
         
            +
             
     | 
| 
      
 10 
     | 
    
         
            +
                  # The period of time in milliseconds after which we force a refresh of metadata even if
         
     | 
| 
      
 11 
     | 
    
         
            +
                  # we haven't seen any partition leadership changes to proactively discover any new brokers or partitions
         
     | 
| 
      
 12 
     | 
    
         
            +
                  base.config :metadata_max_age_ms, :validate => :number, :default => 300_000 # (5m) Kafka default
         
     | 
| 
      
 13 
     | 
    
         
            +
             
     | 
| 
      
 14 
     | 
    
         
            +
                  # The configuration controls the maximum amount of time the client will wait for the response of a request.
         
     | 
| 
      
 15 
     | 
    
         
            +
                  # If the response is not received before the timeout elapses the client will resend the request if necessary
         
     | 
| 
      
 16 
     | 
    
         
            +
                  # or fail the request if retries are exhausted.
         
     | 
| 
      
 17 
     | 
    
         
            +
                  base.config :request_timeout_ms, :validate => :number, :default => 40_000 # Kafka default
         
     | 
| 
      
 18 
     | 
    
         
            +
                end
         
     | 
| 
      
 19 
     | 
    
         
            +
             
     | 
| 
      
 20 
     | 
    
         
            +
                def set_trustore_keystore_config(props)
         
     | 
| 
      
 21 
     | 
    
         
            +
                  props.put("ssl.truststore.type", ssl_truststore_type) unless ssl_truststore_type.nil?
         
     | 
| 
      
 22 
     | 
    
         
            +
                  props.put("ssl.truststore.location", ssl_truststore_location) unless ssl_truststore_location.nil?
         
     | 
| 
      
 23 
     | 
    
         
            +
                  props.put("ssl.truststore.password", ssl_truststore_password.value) unless ssl_truststore_password.nil?
         
     | 
| 
      
 24 
     | 
    
         
            +
             
     | 
| 
      
 25 
     | 
    
         
            +
                  # Client auth stuff
         
     | 
| 
      
 26 
     | 
    
         
            +
                  props.put("ssl.keystore.type", ssl_keystore_type) unless ssl_keystore_type.nil?
         
     | 
| 
      
 27 
     | 
    
         
            +
                  props.put("ssl.key.password", ssl_key_password.value) unless ssl_key_password.nil?
         
     | 
| 
      
 28 
     | 
    
         
            +
                  props.put("ssl.keystore.location", ssl_keystore_location) unless ssl_keystore_location.nil?
         
     | 
| 
      
 29 
     | 
    
         
            +
                  props.put("ssl.keystore.password", ssl_keystore_password.value) unless ssl_keystore_password.nil?
         
     | 
| 
      
 30 
     | 
    
         
            +
                  props.put("ssl.endpoint.identification.algorithm", ssl_endpoint_identification_algorithm) unless ssl_endpoint_identification_algorithm.nil?
         
     | 
| 
      
 31 
     | 
    
         
            +
                end
         
     | 
| 
      
 32 
     | 
    
         
            +
             
     | 
| 
      
 33 
     | 
    
         
            +
                def set_sasl_config(props)
         
     | 
| 
      
 34 
     | 
    
         
            +
                  java.lang.System.setProperty("java.security.auth.login.config", jaas_path) unless jaas_path.nil?
         
     | 
| 
      
 35 
     | 
    
         
            +
                  java.lang.System.setProperty("java.security.krb5.conf", kerberos_config) unless kerberos_config.nil?
         
     | 
| 
      
 36 
     | 
    
         
            +
             
     | 
| 
      
 37 
     | 
    
         
            +
                  props.put("sasl.mechanism", sasl_mechanism)
         
     | 
| 
      
 38 
     | 
    
         
            +
                  if sasl_mechanism == "GSSAPI" && sasl_kerberos_service_name.nil?
         
     | 
| 
      
 39 
     | 
    
         
            +
                    raise LogStash::ConfigurationError, "sasl_kerberos_service_name must be specified when SASL mechanism is GSSAPI"
         
     | 
| 
      
 40 
     | 
    
         
            +
                  end
         
     | 
| 
      
 41 
     | 
    
         
            +
             
     | 
| 
      
 42 
     | 
    
         
            +
                  props.put("sasl.kerberos.service.name", sasl_kerberos_service_name) unless sasl_kerberos_service_name.nil?
         
     | 
| 
      
 43 
     | 
    
         
            +
                  props.put("sasl.jaas.config", sasl_jaas_config) unless sasl_jaas_config.nil?
         
     | 
| 
      
 44 
     | 
    
         
            +
                end
         
     | 
| 
      
 45 
     | 
    
         
            +
             
     | 
| 
      
 46 
     | 
    
         
            +
              end
         
     | 
| 
      
 47 
     | 
    
         
            +
            end end end
         
     | 
| 
         @@ -1,6 +1,6 @@ 
     | 
|
| 
       1 
1 
     | 
    
         
             
            Gem::Specification.new do |s|
         
     | 
| 
       2 
2 
     | 
    
         
             
              s.name            = 'logstash-integration-kafka'
         
     | 
| 
       3 
     | 
    
         
            -
              s.version         = '10. 
     | 
| 
      
 3 
     | 
    
         
            +
              s.version         = '10.11.0'
         
     | 
| 
       4 
4 
     | 
    
         
             
              s.licenses        = ['Apache-2.0']
         
     | 
| 
       5 
5 
     | 
    
         
             
              s.summary         = "Integration with Kafka - input and output plugins"
         
     | 
| 
       6 
6 
     | 
    
         
             
              s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline "+
         
     | 
| 
         @@ -28,7 +28,8 @@ describe "outputs/kafka", :integration => true do 
     | 
|
| 
       28 
28 
     | 
    
         
             
                let(:num_events) { 3 }
         
     | 
| 
       29 
29 
     | 
    
         | 
| 
       30 
30 
     | 
    
         
             
                before :each do
         
     | 
| 
       31 
     | 
    
         
            -
                   
     | 
| 
      
 31 
     | 
    
         
            +
                  # NOTE: the connections_max_idle_ms is irrelevant just testing that configuration works ...
         
     | 
| 
      
 32 
     | 
    
         
            +
                  config = base_config.merge({"topic_id" => test_topic, "connections_max_idle_ms" => 540_000})
         
     | 
| 
       32 
33 
     | 
    
         
             
                  load_kafka_data(config)
         
     | 
| 
       33 
34 
     | 
    
         
             
                end
         
     | 
| 
       34 
35 
     | 
    
         | 
    
        metadata
    CHANGED
    
    | 
         @@ -1,14 +1,14 @@ 
     | 
|
| 
       1 
1 
     | 
    
         
             
            --- !ruby/object:Gem::Specification
         
     | 
| 
       2 
2 
     | 
    
         
             
            name: logstash-integration-kafka
         
     | 
| 
       3 
3 
     | 
    
         
             
            version: !ruby/object:Gem::Version
         
     | 
| 
       4 
     | 
    
         
            -
              version: 10. 
     | 
| 
      
 4 
     | 
    
         
            +
              version: 10.11.0
         
     | 
| 
       5 
5 
     | 
    
         
             
            platform: java
         
     | 
| 
       6 
6 
     | 
    
         
             
            authors:
         
     | 
| 
       7 
7 
     | 
    
         
             
            - Elastic
         
     | 
| 
       8 
8 
     | 
    
         
             
            autorequire:
         
     | 
| 
       9 
9 
     | 
    
         
             
            bindir: bin
         
     | 
| 
       10 
10 
     | 
    
         
             
            cert_chain: []
         
     | 
| 
       11 
     | 
    
         
            -
            date: 2022- 
     | 
| 
      
 11 
     | 
    
         
            +
            date: 2022-04-25 00:00:00.000000000 Z
         
     | 
| 
       12 
12 
     | 
    
         
             
            dependencies:
         
     | 
| 
       13 
13 
     | 
    
         
             
            - !ruby/object:Gem::Dependency
         
     | 
| 
       14 
14 
     | 
    
         
             
              requirement: !ruby/object:Gem::Requirement
         
     | 
| 
         @@ -245,8 +245,8 @@ files: 
     | 
|
| 
       245 
245 
     | 
    
         
             
            - lib/logstash-integration-kafka_jars.rb
         
     | 
| 
       246 
246 
     | 
    
         
             
            - lib/logstash/inputs/kafka.rb
         
     | 
| 
       247 
247 
     | 
    
         
             
            - lib/logstash/outputs/kafka.rb
         
     | 
| 
       248 
     | 
    
         
            -
            - lib/logstash/plugin_mixins/ 
     | 
| 
       249 
     | 
    
         
            -
            - lib/logstash/plugin_mixins/ 
     | 
| 
      
 248 
     | 
    
         
            +
            - lib/logstash/plugin_mixins/kafka/avro_schema_registry.rb
         
     | 
| 
      
 249 
     | 
    
         
            +
            - lib/logstash/plugin_mixins/kafka/common.rb
         
     | 
| 
       250 
250 
     | 
    
         
             
            - logstash-integration-kafka.gemspec
         
     | 
| 
       251 
251 
     | 
    
         
             
            - spec/check_docs_spec.rb
         
     | 
| 
       252 
252 
     | 
    
         
             
            - spec/fixtures/jaas.config
         
     | 
| 
         @@ -1,107 +0,0 @@ 
     | 
|
| 
       1 
     | 
    
         
            -
            module LogStash
         
     | 
| 
       2 
     | 
    
         
            -
              module PluginMixins
         
     | 
| 
       3 
     | 
    
         
            -
                module KafkaAvroSchemaRegistry
         
     | 
| 
       4 
     | 
    
         
            -
             
     | 
| 
       5 
     | 
    
         
            -
                  def self.included(base)
         
     | 
| 
       6 
     | 
    
         
            -
                    base.extend(self)
         
     | 
| 
       7 
     | 
    
         
            -
                    base.setup_schema_registry_config
         
     | 
| 
       8 
     | 
    
         
            -
                  end
         
     | 
| 
       9 
     | 
    
         
            -
             
     | 
| 
       10 
     | 
    
         
            -
                  def setup_schema_registry_config
         
     | 
| 
       11 
     | 
    
         
            -
                    # Option to set key to access Schema Registry.
         
     | 
| 
       12 
     | 
    
         
            -
                    config :schema_registry_key, :validate => :string
         
     | 
| 
       13 
     | 
    
         
            -
             
     | 
| 
       14 
     | 
    
         
            -
                    # Option to set secret to access Schema Registry.
         
     | 
| 
       15 
     | 
    
         
            -
                    config :schema_registry_secret, :validate => :password
         
     | 
| 
       16 
     | 
    
         
            -
             
     | 
| 
       17 
     | 
    
         
            -
                    # Option to set the endpoint of the Schema Registry.
         
     | 
| 
       18 
     | 
    
         
            -
                    # This option permit the usage of Avro Kafka deserializer which retrieve the schema of the Avro message from an
         
     | 
| 
       19 
     | 
    
         
            -
                    # instance of schema registry. If this option has value `value_deserializer_class` nor `topics_pattern` could be valued
         
     | 
| 
       20 
     | 
    
         
            -
                    config :schema_registry_url, :validate => :uri
         
     | 
| 
       21 
     | 
    
         
            -
             
     | 
| 
       22 
     | 
    
         
            -
                    # Option to set the proxy of the Schema Registry.
         
     | 
| 
       23 
     | 
    
         
            -
                    # This option permits to define a proxy to be used to reach the schema registry service instance.
         
     | 
| 
       24 
     | 
    
         
            -
                    config :schema_registry_proxy, :validate => :uri
         
     | 
| 
       25 
     | 
    
         
            -
             
     | 
| 
       26 
     | 
    
         
            -
                    # Option to skip validating the schema registry during registration. This can be useful when using
         
     | 
| 
       27 
     | 
    
         
            -
                    # certificate based auth
         
     | 
| 
       28 
     | 
    
         
            -
                    config :schema_registry_validation, :validate => ['auto', 'skip'], :default => 'auto'
         
     | 
| 
       29 
     | 
    
         
            -
                  end
         
     | 
| 
       30 
     | 
    
         
            -
             
     | 
| 
       31 
     | 
    
         
            -
                  def check_schema_registry_parameters
         
     | 
| 
       32 
     | 
    
         
            -
                    if @schema_registry_url
         
     | 
| 
       33 
     | 
    
         
            -
                      check_for_schema_registry_conflicts
         
     | 
| 
       34 
     | 
    
         
            -
                      @schema_registry_proxy_host, @schema_registry_proxy_port  = split_proxy_into_host_and_port(schema_registry_proxy)
         
     | 
| 
       35 
     | 
    
         
            -
                      check_for_key_and_secret
         
     | 
| 
       36 
     | 
    
         
            -
                      check_for_schema_registry_connectivity_and_subjects if schema_registry_validation?
         
     | 
| 
       37 
     | 
    
         
            -
                    end
         
     | 
| 
       38 
     | 
    
         
            -
                  end
         
     | 
| 
       39 
     | 
    
         
            -
             
     | 
| 
       40 
     | 
    
         
            -
                  def schema_registry_validation?
         
     | 
| 
       41 
     | 
    
         
            -
                    return false if schema_registry_validation.to_s == 'skip'
         
     | 
| 
       42 
     | 
    
         
            -
                    return false if using_kerberos? # pre-validation doesn't support kerberos
         
     | 
| 
       43 
     | 
    
         
            -
                    
         
     | 
| 
       44 
     | 
    
         
            -
                    true
         
     | 
| 
       45 
     | 
    
         
            -
                  end
         
     | 
| 
       46 
     | 
    
         
            -
             
     | 
| 
       47 
     | 
    
         
            -
                  def using_kerberos?
         
     | 
| 
       48 
     | 
    
         
            -
                    security_protocol == "SASL_PLAINTEXT" || security_protocol == "SASL_SSL"
         
     | 
| 
       49 
     | 
    
         
            -
                  end
         
     | 
| 
       50 
     | 
    
         
            -
             
     | 
| 
       51 
     | 
    
         
            -
                  private
         
     | 
| 
       52 
     | 
    
         
            -
                  def check_for_schema_registry_conflicts
         
     | 
| 
       53 
     | 
    
         
            -
                    if @value_deserializer_class != LogStash::Inputs::Kafka::DEFAULT_DESERIALIZER_CLASS
         
     | 
| 
       54 
     | 
    
         
            -
                      raise LogStash::ConfigurationError, 'Option schema_registry_url prohibit the customization of value_deserializer_class'
         
     | 
| 
       55 
     | 
    
         
            -
                    end
         
     | 
| 
       56 
     | 
    
         
            -
                    if @topics_pattern && !@topics_pattern.empty?
         
     | 
| 
       57 
     | 
    
         
            -
                     raise LogStash::ConfigurationError, 'Option schema_registry_url prohibit the customization of topics_pattern'
         
     | 
| 
       58 
     | 
    
         
            -
                    end
         
     | 
| 
       59 
     | 
    
         
            -
                  end
         
     | 
| 
       60 
     | 
    
         
            -
             
     | 
| 
       61 
     | 
    
         
            -
                  private
         
     | 
| 
       62 
     | 
    
         
            -
                  def check_for_schema_registry_connectivity_and_subjects
         
     | 
| 
       63 
     | 
    
         
            -
                    options = {}
         
     | 
| 
       64 
     | 
    
         
            -
                    if schema_registry_proxy && !schema_registry_proxy.empty?
         
     | 
| 
       65 
     | 
    
         
            -
                      options[:proxy] = schema_registry_proxy.to_s
         
     | 
| 
       66 
     | 
    
         
            -
                    end
         
     | 
| 
       67 
     | 
    
         
            -
                    if schema_registry_key and !schema_registry_key.empty?
         
     | 
| 
       68 
     | 
    
         
            -
                      options[:auth] = {:user => schema_registry_key, :password => schema_registry_secret.value}
         
     | 
| 
       69 
     | 
    
         
            -
                    end
         
     | 
| 
       70 
     | 
    
         
            -
                    client = Manticore::Client.new(options)
         
     | 
| 
       71 
     | 
    
         
            -
                    begin
         
     | 
| 
       72 
     | 
    
         
            -
                      response = client.get(@schema_registry_url.uri.to_s + '/subjects').body
         
     | 
| 
       73 
     | 
    
         
            -
                    rescue Manticore::ManticoreException => e
         
     | 
| 
       74 
     | 
    
         
            -
                      raise LogStash::ConfigurationError.new("Schema registry service doesn't respond, error: #{e.message}")
         
     | 
| 
       75 
     | 
    
         
            -
                    end
         
     | 
| 
       76 
     | 
    
         
            -
                    registered_subjects = JSON.parse response
         
     | 
| 
       77 
     | 
    
         
            -
                    expected_subjects = @topics.map { |t| "#{t}-value"}
         
     | 
| 
       78 
     | 
    
         
            -
                    if (expected_subjects & registered_subjects).size != expected_subjects.size
         
     | 
| 
       79 
     | 
    
         
            -
                      undefined_topic_subjects = expected_subjects - registered_subjects
         
     | 
| 
       80 
     | 
    
         
            -
                      raise LogStash::ConfigurationError, "The schema registry does not contain definitions for required topic subjects: #{undefined_topic_subjects}"
         
     | 
| 
       81 
     | 
    
         
            -
                    end
         
     | 
| 
       82 
     | 
    
         
            -
                  end
         
     | 
| 
       83 
     | 
    
         
            -
             
     | 
| 
       84 
     | 
    
         
            -
                  def split_proxy_into_host_and_port(proxy_uri)
         
     | 
| 
       85 
     | 
    
         
            -
                    return nil unless proxy_uri && !proxy_uri.empty?
         
     | 
| 
       86 
     | 
    
         
            -
             
     | 
| 
       87 
     | 
    
         
            -
                    port = proxy_uri.port
         
     | 
| 
       88 
     | 
    
         
            -
             
     | 
| 
       89 
     | 
    
         
            -
                    host_spec = ""
         
     | 
| 
       90 
     | 
    
         
            -
                    host_spec << proxy_uri.scheme || "http"
         
     | 
| 
       91 
     | 
    
         
            -
                    host_spec << "://"
         
     | 
| 
       92 
     | 
    
         
            -
                    host_spec << "#{proxy_uri.userinfo}@" if proxy_uri.userinfo
         
     | 
| 
       93 
     | 
    
         
            -
                    host_spec << proxy_uri.host
         
     | 
| 
       94 
     | 
    
         
            -
             
     | 
| 
       95 
     | 
    
         
            -
                    [host_spec, port]
         
     | 
| 
       96 
     | 
    
         
            -
                  end
         
     | 
| 
       97 
     | 
    
         
            -
             
     | 
| 
       98 
     | 
    
         
            -
                  def check_for_key_and_secret
         
     | 
| 
       99 
     | 
    
         
            -
                    if schema_registry_key and !schema_registry_key.empty?
         
     | 
| 
       100 
     | 
    
         
            -
                      if !schema_registry_secret or schema_registry_secret.value.empty?
         
     | 
| 
       101 
     | 
    
         
            -
                        raise LogStash::ConfigurationError, "Setting `schema_registry_secret` is required when `schema_registry_key` is provided."
         
     | 
| 
       102 
     | 
    
         
            -
                      end
         
     | 
| 
       103 
     | 
    
         
            -
                    end
         
     | 
| 
       104 
     | 
    
         
            -
                  end
         
     | 
| 
       105 
     | 
    
         
            -
                end
         
     | 
| 
       106 
     | 
    
         
            -
              end
         
     | 
| 
       107 
     | 
    
         
            -
            end
         
     | 
| 
         @@ -1,29 +0,0 @@ 
     | 
|
| 
       1 
     | 
    
         
            -
            module LogStash module PluginMixins module KafkaSupport
         
     | 
| 
       2 
     | 
    
         
            -
             
     | 
| 
       3 
     | 
    
         
            -
              def set_trustore_keystore_config(props)
         
     | 
| 
       4 
     | 
    
         
            -
                props.put("ssl.truststore.type", ssl_truststore_type) unless ssl_truststore_type.nil?
         
     | 
| 
       5 
     | 
    
         
            -
                props.put("ssl.truststore.location", ssl_truststore_location) unless ssl_truststore_location.nil?
         
     | 
| 
       6 
     | 
    
         
            -
                props.put("ssl.truststore.password", ssl_truststore_password.value) unless ssl_truststore_password.nil?
         
     | 
| 
       7 
     | 
    
         
            -
             
     | 
| 
       8 
     | 
    
         
            -
                # Client auth stuff
         
     | 
| 
       9 
     | 
    
         
            -
                props.put("ssl.keystore.type", ssl_keystore_type) unless ssl_keystore_type.nil?
         
     | 
| 
       10 
     | 
    
         
            -
                props.put("ssl.key.password", ssl_key_password.value) unless ssl_key_password.nil?
         
     | 
| 
       11 
     | 
    
         
            -
                props.put("ssl.keystore.location", ssl_keystore_location) unless ssl_keystore_location.nil?
         
     | 
| 
       12 
     | 
    
         
            -
                props.put("ssl.keystore.password", ssl_keystore_password.value) unless ssl_keystore_password.nil?
         
     | 
| 
       13 
     | 
    
         
            -
                props.put("ssl.endpoint.identification.algorithm", ssl_endpoint_identification_algorithm) unless ssl_endpoint_identification_algorithm.nil?
         
     | 
| 
       14 
     | 
    
         
            -
              end
         
     | 
| 
       15 
     | 
    
         
            -
             
     | 
| 
       16 
     | 
    
         
            -
              def set_sasl_config(props)
         
     | 
| 
       17 
     | 
    
         
            -
                java.lang.System.setProperty("java.security.auth.login.config", jaas_path) unless jaas_path.nil?
         
     | 
| 
       18 
     | 
    
         
            -
                java.lang.System.setProperty("java.security.krb5.conf", kerberos_config) unless kerberos_config.nil?
         
     | 
| 
       19 
     | 
    
         
            -
             
     | 
| 
       20 
     | 
    
         
            -
                props.put("sasl.mechanism", sasl_mechanism)
         
     | 
| 
       21 
     | 
    
         
            -
                if sasl_mechanism == "GSSAPI" && sasl_kerberos_service_name.nil?
         
     | 
| 
       22 
     | 
    
         
            -
                  raise LogStash::ConfigurationError, "sasl_kerberos_service_name must be specified when SASL mechanism is GSSAPI"
         
     | 
| 
       23 
     | 
    
         
            -
                end
         
     | 
| 
       24 
     | 
    
         
            -
             
     | 
| 
       25 
     | 
    
         
            -
                props.put("sasl.kerberos.service.name", sasl_kerberos_service_name) unless sasl_kerberos_service_name.nil?
         
     | 
| 
       26 
     | 
    
         
            -
                props.put("sasl.jaas.config", sasl_jaas_config) unless sasl_jaas_config.nil?
         
     | 
| 
       27 
     | 
    
         
            -
              end
         
     | 
| 
       28 
     | 
    
         
            -
             
     | 
| 
       29 
     | 
    
         
            -
            end end end
         
     |