logstash-integration-kafka 10.9.0-java → 11.3.2-java

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +39 -2
  3. data/DEVELOPER.md +1 -1
  4. data/docs/index.asciidoc +1 -1
  5. data/docs/input-kafka.asciidoc +96 -8
  6. data/docs/output-kafka.asciidoc +20 -5
  7. data/lib/logstash/inputs/kafka.rb +42 -21
  8. data/lib/logstash/outputs/kafka.rb +7 -12
  9. data/lib/logstash/plugin_mixins/kafka/avro_schema_registry.rb +139 -0
  10. data/lib/logstash/plugin_mixins/kafka/common.rb +55 -0
  11. data/lib/logstash-integration-kafka_jars.rb +9 -14
  12. data/logstash-integration-kafka.gemspec +2 -2
  13. data/spec/integration/inputs/kafka_spec.rb +184 -20
  14. data/spec/integration/outputs/kafka_spec.rb +21 -1
  15. data/spec/unit/inputs/kafka_spec.rb +28 -5
  16. data/spec/unit/outputs/kafka_spec.rb +8 -0
  17. data/vendor/jar-dependencies/com/github/luben/zstd-jni/1.5.5-4/zstd-jni-1.5.5-4.jar +0 -0
  18. data/vendor/jar-dependencies/io/confluent/kafka-avro-serializer/7.4.0/kafka-avro-serializer-7.4.0.jar +0 -0
  19. data/vendor/jar-dependencies/io/confluent/kafka-schema-registry-client/7.4.0/kafka-schema-registry-client-7.4.0.jar +0 -0
  20. data/vendor/jar-dependencies/io/confluent/kafka-schema-serializer/7.4.0/kafka-schema-serializer-7.4.0.jar +0 -0
  21. data/vendor/jar-dependencies/org/apache/avro/avro/1.11.3/avro-1.11.3.jar +0 -0
  22. data/vendor/jar-dependencies/org/apache/kafka/kafka-clients/3.4.1/kafka-clients-3.4.1.jar +0 -0
  23. data/vendor/jar-dependencies/org/lz4/lz4-java/1.8.0/lz4-java-1.8.0.jar +0 -0
  24. data/vendor/jar-dependencies/org/slf4j/slf4j-api/{1.7.30/slf4j-api-1.7.30.jar → 1.7.36/slf4j-api-1.7.36.jar} +0 -0
  25. data/vendor/jar-dependencies/org/xerial/snappy/snappy-java/1.1.10.5/snappy-java-1.1.10.5.jar +0 -0
  26. metadata +16 -21
  27. data/lib/logstash/plugin_mixins/common.rb +0 -107
  28. data/lib/logstash/plugin_mixins/kafka_support.rb +0 -29
  29. data/vendor/jar-dependencies/com/github/luben/zstd-jni/1.4.4-7/zstd-jni-1.4.4-7.jar +0 -0
  30. data/vendor/jar-dependencies/io/confluent/common-config/5.5.1/common-config-5.5.1.jar +0 -0
  31. data/vendor/jar-dependencies/io/confluent/common-utils/5.5.1/common-utils-5.5.1.jar +0 -0
  32. data/vendor/jar-dependencies/io/confluent/kafka-avro-serializer/5.5.1/kafka-avro-serializer-5.5.1.jar +0 -0
  33. data/vendor/jar-dependencies/io/confluent/kafka-schema-registry-client/5.5.1/kafka-schema-registry-client-5.5.1.jar +0 -0
  34. data/vendor/jar-dependencies/io/confluent/kafka-schema-serializer/5.5.1/kafka-schema-serializer-5.5.1.jar +0 -0
  35. data/vendor/jar-dependencies/javax/ws/rs/javax.ws.rs-api/2.1.1/javax.ws.rs-api-2.1.1.jar +0 -0
  36. data/vendor/jar-dependencies/org/apache/avro/avro/1.9.2/avro-1.9.2.jar +0 -0
  37. data/vendor/jar-dependencies/org/apache/kafka/kafka-clients/2.5.1/kafka-clients-2.5.1.jar +0 -0
  38. data/vendor/jar-dependencies/org/apache/kafka/kafka_2.12/2.5.1/kafka_2.12-2.5.1.jar +0 -0
  39. data/vendor/jar-dependencies/org/glassfish/jersey/core/jersey-common/2.33/jersey-common-2.33.jar +0 -0
  40. data/vendor/jar-dependencies/org/lz4/lz4-java/1.7.1/lz4-java-1.7.1.jar +0 -0
  41. data/vendor/jar-dependencies/org/xerial/snappy/snappy-java/1.1.7.3/snappy-java-1.1.7.3.jar +0 -0
@@ -22,6 +22,14 @@ describe "outputs/kafka" do
22
22
  expect(kafka.topic_id).to eql 'test'
23
23
  expect(kafka.key_serializer).to eql 'org.apache.kafka.common.serialization.StringSerializer'
24
24
  end
25
+
26
+ it 'should fallback `client_dns_lookup` to `use_all_dns_ips` when the deprecated `default` is specified' do
27
+ simple_kafka_config["client_dns_lookup"] = 'default'
28
+ kafka = LogStash::Outputs::Kafka.new(simple_kafka_config)
29
+ kafka.register
30
+
31
+ expect(kafka.client_dns_lookup).to eq('use_all_dns_ips')
32
+ end
25
33
  end
26
34
 
27
35
  context 'when outputting messages' do
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-integration-kafka
3
3
  version: !ruby/object:Gem::Version
4
- version: 10.9.0
4
+ version: 11.3.2
5
5
  platform: java
6
6
  authors:
7
7
  - Elastic
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2021-12-14 00:00:00.000000000 Z
11
+ date: 2023-10-16 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
@@ -49,7 +49,7 @@ dependencies:
49
49
  requirements:
50
50
  - - ">="
51
51
  - !ruby/object:Gem::Version
52
- version: 6.5.0
52
+ version: 8.3.0
53
53
  name: logstash-core
54
54
  prerelease: false
55
55
  type: :runtime
@@ -57,7 +57,7 @@ dependencies:
57
57
  requirements:
58
58
  - - ">="
59
59
  - !ruby/object:Gem::Version
60
- version: 6.5.0
60
+ version: 8.3.0
61
61
  - !ruby/object:Gem::Dependency
62
62
  requirement: !ruby/object:Gem::Requirement
63
63
  requirements:
@@ -245,8 +245,8 @@ files:
245
245
  - lib/logstash-integration-kafka_jars.rb
246
246
  - lib/logstash/inputs/kafka.rb
247
247
  - lib/logstash/outputs/kafka.rb
248
- - lib/logstash/plugin_mixins/common.rb
249
- - lib/logstash/plugin_mixins/kafka_support.rb
248
+ - lib/logstash/plugin_mixins/kafka/avro_schema_registry.rb
249
+ - lib/logstash/plugin_mixins/kafka/common.rb
250
250
  - logstash-integration-kafka.gemspec
251
251
  - spec/check_docs_spec.rb
252
252
  - spec/fixtures/jaas.config
@@ -257,20 +257,15 @@ files:
257
257
  - spec/unit/inputs/avro_schema_fixture_payment.asvc
258
258
  - spec/unit/inputs/kafka_spec.rb
259
259
  - spec/unit/outputs/kafka_spec.rb
260
- - vendor/jar-dependencies/com/github/luben/zstd-jni/1.4.4-7/zstd-jni-1.4.4-7.jar
261
- - vendor/jar-dependencies/io/confluent/common-config/5.5.1/common-config-5.5.1.jar
262
- - vendor/jar-dependencies/io/confluent/common-utils/5.5.1/common-utils-5.5.1.jar
263
- - vendor/jar-dependencies/io/confluent/kafka-avro-serializer/5.5.1/kafka-avro-serializer-5.5.1.jar
264
- - vendor/jar-dependencies/io/confluent/kafka-schema-registry-client/5.5.1/kafka-schema-registry-client-5.5.1.jar
265
- - vendor/jar-dependencies/io/confluent/kafka-schema-serializer/5.5.1/kafka-schema-serializer-5.5.1.jar
266
- - vendor/jar-dependencies/javax/ws/rs/javax.ws.rs-api/2.1.1/javax.ws.rs-api-2.1.1.jar
267
- - vendor/jar-dependencies/org/apache/avro/avro/1.9.2/avro-1.9.2.jar
268
- - vendor/jar-dependencies/org/apache/kafka/kafka-clients/2.5.1/kafka-clients-2.5.1.jar
269
- - vendor/jar-dependencies/org/apache/kafka/kafka_2.12/2.5.1/kafka_2.12-2.5.1.jar
270
- - vendor/jar-dependencies/org/glassfish/jersey/core/jersey-common/2.33/jersey-common-2.33.jar
271
- - vendor/jar-dependencies/org/lz4/lz4-java/1.7.1/lz4-java-1.7.1.jar
272
- - vendor/jar-dependencies/org/slf4j/slf4j-api/1.7.30/slf4j-api-1.7.30.jar
273
- - vendor/jar-dependencies/org/xerial/snappy/snappy-java/1.1.7.3/snappy-java-1.1.7.3.jar
260
+ - vendor/jar-dependencies/com/github/luben/zstd-jni/1.5.5-4/zstd-jni-1.5.5-4.jar
261
+ - vendor/jar-dependencies/io/confluent/kafka-avro-serializer/7.4.0/kafka-avro-serializer-7.4.0.jar
262
+ - vendor/jar-dependencies/io/confluent/kafka-schema-registry-client/7.4.0/kafka-schema-registry-client-7.4.0.jar
263
+ - vendor/jar-dependencies/io/confluent/kafka-schema-serializer/7.4.0/kafka-schema-serializer-7.4.0.jar
264
+ - vendor/jar-dependencies/org/apache/avro/avro/1.11.3/avro-1.11.3.jar
265
+ - vendor/jar-dependencies/org/apache/kafka/kafka-clients/3.4.1/kafka-clients-3.4.1.jar
266
+ - vendor/jar-dependencies/org/lz4/lz4-java/1.8.0/lz4-java-1.8.0.jar
267
+ - vendor/jar-dependencies/org/slf4j/slf4j-api/1.7.36/slf4j-api-1.7.36.jar
268
+ - vendor/jar-dependencies/org/xerial/snappy/snappy-java/1.1.10.5/snappy-java-1.1.10.5.jar
274
269
  homepage: http://www.elastic.co/guide/en/logstash/current/index.html
275
270
  licenses:
276
271
  - Apache-2.0
@@ -294,7 +289,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
294
289
  - !ruby/object:Gem::Version
295
290
  version: '0'
296
291
  requirements: []
297
- rubygems_version: 3.1.6
292
+ rubygems_version: 3.2.33
298
293
  signing_key:
299
294
  specification_version: 4
300
295
  summary: Integration with Kafka - input and output plugins
@@ -1,107 +0,0 @@
1
- module LogStash
2
- module PluginMixins
3
- module KafkaAvroSchemaRegistry
4
-
5
- def self.included(base)
6
- base.extend(self)
7
- base.setup_schema_registry_config
8
- end
9
-
10
- def setup_schema_registry_config
11
- # Option to set key to access Schema Registry.
12
- config :schema_registry_key, :validate => :string
13
-
14
- # Option to set secret to access Schema Registry.
15
- config :schema_registry_secret, :validate => :password
16
-
17
- # Option to set the endpoint of the Schema Registry.
18
- # This option permit the usage of Avro Kafka deserializer which retrieve the schema of the Avro message from an
19
- # instance of schema registry. If this option has value `value_deserializer_class` nor `topics_pattern` could be valued
20
- config :schema_registry_url, :validate => :uri
21
-
22
- # Option to set the proxy of the Schema Registry.
23
- # This option permits to define a proxy to be used to reach the schema registry service instance.
24
- config :schema_registry_proxy, :validate => :uri
25
-
26
- # Option to skip validating the schema registry during registration. This can be useful when using
27
- # certificate based auth
28
- config :schema_registry_validation, :validate => ['auto', 'skip'], :default => 'auto'
29
- end
30
-
31
- def check_schema_registry_parameters
32
- if @schema_registry_url
33
- check_for_schema_registry_conflicts
34
- @schema_registry_proxy_host, @schema_registry_proxy_port = split_proxy_into_host_and_port(schema_registry_proxy)
35
- check_for_key_and_secret
36
- check_for_schema_registry_connectivity_and_subjects if schema_registry_validation?
37
- end
38
- end
39
-
40
- def schema_registry_validation?
41
- return false if schema_registry_validation.to_s == 'skip'
42
- return false if using_kerberos? # pre-validation doesn't support kerberos
43
-
44
- true
45
- end
46
-
47
- def using_kerberos?
48
- security_protocol == "SASL_PLAINTEXT" || security_protocol == "SASL_SSL"
49
- end
50
-
51
- private
52
- def check_for_schema_registry_conflicts
53
- if @value_deserializer_class != LogStash::Inputs::Kafka::DEFAULT_DESERIALIZER_CLASS
54
- raise LogStash::ConfigurationError, 'Option schema_registry_url prohibit the customization of value_deserializer_class'
55
- end
56
- if @topics_pattern && !@topics_pattern.empty?
57
- raise LogStash::ConfigurationError, 'Option schema_registry_url prohibit the customization of topics_pattern'
58
- end
59
- end
60
-
61
- private
62
- def check_for_schema_registry_connectivity_and_subjects
63
- options = {}
64
- if schema_registry_proxy && !schema_registry_proxy.empty?
65
- options[:proxy] = schema_registry_proxy.to_s
66
- end
67
- if schema_registry_key and !schema_registry_key.empty?
68
- options[:auth] = {:user => schema_registry_key, :password => schema_registry_secret.value}
69
- end
70
- client = Manticore::Client.new(options)
71
- begin
72
- response = client.get(@schema_registry_url.uri.to_s + '/subjects').body
73
- rescue Manticore::ManticoreException => e
74
- raise LogStash::ConfigurationError.new("Schema registry service doesn't respond, error: #{e.message}")
75
- end
76
- registered_subjects = JSON.parse response
77
- expected_subjects = @topics.map { |t| "#{t}-value"}
78
- if (expected_subjects & registered_subjects).size != expected_subjects.size
79
- undefined_topic_subjects = expected_subjects - registered_subjects
80
- raise LogStash::ConfigurationError, "The schema registry does not contain definitions for required topic subjects: #{undefined_topic_subjects}"
81
- end
82
- end
83
-
84
- def split_proxy_into_host_and_port(proxy_uri)
85
- return nil unless proxy_uri && !proxy_uri.empty?
86
-
87
- port = proxy_uri.port
88
-
89
- host_spec = ""
90
- host_spec << proxy_uri.scheme || "http"
91
- host_spec << "://"
92
- host_spec << "#{proxy_uri.userinfo}@" if proxy_uri.userinfo
93
- host_spec << proxy_uri.host
94
-
95
- [host_spec, port]
96
- end
97
-
98
- def check_for_key_and_secret
99
- if schema_registry_key and !schema_registry_key.empty?
100
- if !schema_registry_secret or schema_registry_secret.value.empty?
101
- raise LogStash::ConfigurationError, "Setting `schema_registry_secret` is required when `schema_registry_key` is provided."
102
- end
103
- end
104
- end
105
- end
106
- end
107
- end
@@ -1,29 +0,0 @@
1
- module LogStash module PluginMixins module KafkaSupport
2
-
3
- def set_trustore_keystore_config(props)
4
- props.put("ssl.truststore.type", ssl_truststore_type) unless ssl_truststore_type.nil?
5
- props.put("ssl.truststore.location", ssl_truststore_location) unless ssl_truststore_location.nil?
6
- props.put("ssl.truststore.password", ssl_truststore_password.value) unless ssl_truststore_password.nil?
7
-
8
- # Client auth stuff
9
- props.put("ssl.keystore.type", ssl_keystore_type) unless ssl_keystore_type.nil?
10
- props.put("ssl.key.password", ssl_key_password.value) unless ssl_key_password.nil?
11
- props.put("ssl.keystore.location", ssl_keystore_location) unless ssl_keystore_location.nil?
12
- props.put("ssl.keystore.password", ssl_keystore_password.value) unless ssl_keystore_password.nil?
13
- props.put("ssl.endpoint.identification.algorithm", ssl_endpoint_identification_algorithm) unless ssl_endpoint_identification_algorithm.nil?
14
- end
15
-
16
- def set_sasl_config(props)
17
- java.lang.System.setProperty("java.security.auth.login.config", jaas_path) unless jaas_path.nil?
18
- java.lang.System.setProperty("java.security.krb5.conf", kerberos_config) unless kerberos_config.nil?
19
-
20
- props.put("sasl.mechanism", sasl_mechanism)
21
- if sasl_mechanism == "GSSAPI" && sasl_kerberos_service_name.nil?
22
- raise LogStash::ConfigurationError, "sasl_kerberos_service_name must be specified when SASL mechanism is GSSAPI"
23
- end
24
-
25
- props.put("sasl.kerberos.service.name", sasl_kerberos_service_name) unless sasl_kerberos_service_name.nil?
26
- props.put("sasl.jaas.config", sasl_jaas_config) unless sasl_jaas_config.nil?
27
- end
28
-
29
- end end end