logstash-integration-kafka 10.9.0-java → 11.3.2-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +39 -2
- data/DEVELOPER.md +1 -1
- data/docs/index.asciidoc +1 -1
- data/docs/input-kafka.asciidoc +96 -8
- data/docs/output-kafka.asciidoc +20 -5
- data/lib/logstash/inputs/kafka.rb +42 -21
- data/lib/logstash/outputs/kafka.rb +7 -12
- data/lib/logstash/plugin_mixins/kafka/avro_schema_registry.rb +139 -0
- data/lib/logstash/plugin_mixins/kafka/common.rb +55 -0
- data/lib/logstash-integration-kafka_jars.rb +9 -14
- data/logstash-integration-kafka.gemspec +2 -2
- data/spec/integration/inputs/kafka_spec.rb +184 -20
- data/spec/integration/outputs/kafka_spec.rb +21 -1
- data/spec/unit/inputs/kafka_spec.rb +28 -5
- data/spec/unit/outputs/kafka_spec.rb +8 -0
- data/vendor/jar-dependencies/com/github/luben/zstd-jni/1.5.5-4/zstd-jni-1.5.5-4.jar +0 -0
- data/vendor/jar-dependencies/io/confluent/kafka-avro-serializer/7.4.0/kafka-avro-serializer-7.4.0.jar +0 -0
- data/vendor/jar-dependencies/io/confluent/kafka-schema-registry-client/7.4.0/kafka-schema-registry-client-7.4.0.jar +0 -0
- data/vendor/jar-dependencies/io/confluent/kafka-schema-serializer/7.4.0/kafka-schema-serializer-7.4.0.jar +0 -0
- data/vendor/jar-dependencies/org/apache/avro/avro/1.11.3/avro-1.11.3.jar +0 -0
- data/vendor/jar-dependencies/org/apache/kafka/kafka-clients/3.4.1/kafka-clients-3.4.1.jar +0 -0
- data/vendor/jar-dependencies/org/lz4/lz4-java/1.8.0/lz4-java-1.8.0.jar +0 -0
- data/vendor/jar-dependencies/org/slf4j/slf4j-api/{1.7.30/slf4j-api-1.7.30.jar → 1.7.36/slf4j-api-1.7.36.jar} +0 -0
- data/vendor/jar-dependencies/org/xerial/snappy/snappy-java/1.1.10.5/snappy-java-1.1.10.5.jar +0 -0
- metadata +16 -21
- data/lib/logstash/plugin_mixins/common.rb +0 -107
- data/lib/logstash/plugin_mixins/kafka_support.rb +0 -29
- data/vendor/jar-dependencies/com/github/luben/zstd-jni/1.4.4-7/zstd-jni-1.4.4-7.jar +0 -0
- data/vendor/jar-dependencies/io/confluent/common-config/5.5.1/common-config-5.5.1.jar +0 -0
- data/vendor/jar-dependencies/io/confluent/common-utils/5.5.1/common-utils-5.5.1.jar +0 -0
- data/vendor/jar-dependencies/io/confluent/kafka-avro-serializer/5.5.1/kafka-avro-serializer-5.5.1.jar +0 -0
- data/vendor/jar-dependencies/io/confluent/kafka-schema-registry-client/5.5.1/kafka-schema-registry-client-5.5.1.jar +0 -0
- data/vendor/jar-dependencies/io/confluent/kafka-schema-serializer/5.5.1/kafka-schema-serializer-5.5.1.jar +0 -0
- data/vendor/jar-dependencies/javax/ws/rs/javax.ws.rs-api/2.1.1/javax.ws.rs-api-2.1.1.jar +0 -0
- data/vendor/jar-dependencies/org/apache/avro/avro/1.9.2/avro-1.9.2.jar +0 -0
- data/vendor/jar-dependencies/org/apache/kafka/kafka-clients/2.5.1/kafka-clients-2.5.1.jar +0 -0
- data/vendor/jar-dependencies/org/apache/kafka/kafka_2.12/2.5.1/kafka_2.12-2.5.1.jar +0 -0
- data/vendor/jar-dependencies/org/glassfish/jersey/core/jersey-common/2.33/jersey-common-2.33.jar +0 -0
- data/vendor/jar-dependencies/org/lz4/lz4-java/1.7.1/lz4-java-1.7.1.jar +0 -0
- data/vendor/jar-dependencies/org/xerial/snappy/snappy-java/1.1.7.3/snappy-java-1.1.7.3.jar +0 -0
@@ -22,6 +22,14 @@ describe "outputs/kafka" do
|
|
22
22
|
expect(kafka.topic_id).to eql 'test'
|
23
23
|
expect(kafka.key_serializer).to eql 'org.apache.kafka.common.serialization.StringSerializer'
|
24
24
|
end
|
25
|
+
|
26
|
+
it 'should fallback `client_dns_lookup` to `use_all_dns_ips` when the deprecated `default` is specified' do
|
27
|
+
simple_kafka_config["client_dns_lookup"] = 'default'
|
28
|
+
kafka = LogStash::Outputs::Kafka.new(simple_kafka_config)
|
29
|
+
kafka.register
|
30
|
+
|
31
|
+
expect(kafka.client_dns_lookup).to eq('use_all_dns_ips')
|
32
|
+
end
|
25
33
|
end
|
26
34
|
|
27
35
|
context 'when outputting messages' do
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-integration-kafka
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 11.3.2
|
5
5
|
platform: java
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2023-10-16 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -49,7 +49,7 @@ dependencies:
|
|
49
49
|
requirements:
|
50
50
|
- - ">="
|
51
51
|
- !ruby/object:Gem::Version
|
52
|
-
version:
|
52
|
+
version: 8.3.0
|
53
53
|
name: logstash-core
|
54
54
|
prerelease: false
|
55
55
|
type: :runtime
|
@@ -57,7 +57,7 @@ dependencies:
|
|
57
57
|
requirements:
|
58
58
|
- - ">="
|
59
59
|
- !ruby/object:Gem::Version
|
60
|
-
version:
|
60
|
+
version: 8.3.0
|
61
61
|
- !ruby/object:Gem::Dependency
|
62
62
|
requirement: !ruby/object:Gem::Requirement
|
63
63
|
requirements:
|
@@ -245,8 +245,8 @@ files:
|
|
245
245
|
- lib/logstash-integration-kafka_jars.rb
|
246
246
|
- lib/logstash/inputs/kafka.rb
|
247
247
|
- lib/logstash/outputs/kafka.rb
|
248
|
-
- lib/logstash/plugin_mixins/
|
249
|
-
- lib/logstash/plugin_mixins/
|
248
|
+
- lib/logstash/plugin_mixins/kafka/avro_schema_registry.rb
|
249
|
+
- lib/logstash/plugin_mixins/kafka/common.rb
|
250
250
|
- logstash-integration-kafka.gemspec
|
251
251
|
- spec/check_docs_spec.rb
|
252
252
|
- spec/fixtures/jaas.config
|
@@ -257,20 +257,15 @@ files:
|
|
257
257
|
- spec/unit/inputs/avro_schema_fixture_payment.asvc
|
258
258
|
- spec/unit/inputs/kafka_spec.rb
|
259
259
|
- spec/unit/outputs/kafka_spec.rb
|
260
|
-
- vendor/jar-dependencies/com/github/luben/zstd-jni/1.
|
261
|
-
- vendor/jar-dependencies/io/confluent/
|
262
|
-
- vendor/jar-dependencies/io/confluent/
|
263
|
-
- vendor/jar-dependencies/io/confluent/kafka-
|
264
|
-
- vendor/jar-dependencies/
|
265
|
-
- vendor/jar-dependencies/
|
266
|
-
- vendor/jar-dependencies/
|
267
|
-
- vendor/jar-dependencies/org/
|
268
|
-
- vendor/jar-dependencies/org/
|
269
|
-
- vendor/jar-dependencies/org/apache/kafka/kafka_2.12/2.5.1/kafka_2.12-2.5.1.jar
|
270
|
-
- vendor/jar-dependencies/org/glassfish/jersey/core/jersey-common/2.33/jersey-common-2.33.jar
|
271
|
-
- vendor/jar-dependencies/org/lz4/lz4-java/1.7.1/lz4-java-1.7.1.jar
|
272
|
-
- vendor/jar-dependencies/org/slf4j/slf4j-api/1.7.30/slf4j-api-1.7.30.jar
|
273
|
-
- vendor/jar-dependencies/org/xerial/snappy/snappy-java/1.1.7.3/snappy-java-1.1.7.3.jar
|
260
|
+
- vendor/jar-dependencies/com/github/luben/zstd-jni/1.5.5-4/zstd-jni-1.5.5-4.jar
|
261
|
+
- vendor/jar-dependencies/io/confluent/kafka-avro-serializer/7.4.0/kafka-avro-serializer-7.4.0.jar
|
262
|
+
- vendor/jar-dependencies/io/confluent/kafka-schema-registry-client/7.4.0/kafka-schema-registry-client-7.4.0.jar
|
263
|
+
- vendor/jar-dependencies/io/confluent/kafka-schema-serializer/7.4.0/kafka-schema-serializer-7.4.0.jar
|
264
|
+
- vendor/jar-dependencies/org/apache/avro/avro/1.11.3/avro-1.11.3.jar
|
265
|
+
- vendor/jar-dependencies/org/apache/kafka/kafka-clients/3.4.1/kafka-clients-3.4.1.jar
|
266
|
+
- vendor/jar-dependencies/org/lz4/lz4-java/1.8.0/lz4-java-1.8.0.jar
|
267
|
+
- vendor/jar-dependencies/org/slf4j/slf4j-api/1.7.36/slf4j-api-1.7.36.jar
|
268
|
+
- vendor/jar-dependencies/org/xerial/snappy/snappy-java/1.1.10.5/snappy-java-1.1.10.5.jar
|
274
269
|
homepage: http://www.elastic.co/guide/en/logstash/current/index.html
|
275
270
|
licenses:
|
276
271
|
- Apache-2.0
|
@@ -294,7 +289,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
294
289
|
- !ruby/object:Gem::Version
|
295
290
|
version: '0'
|
296
291
|
requirements: []
|
297
|
-
rubygems_version: 3.
|
292
|
+
rubygems_version: 3.2.33
|
298
293
|
signing_key:
|
299
294
|
specification_version: 4
|
300
295
|
summary: Integration with Kafka - input and output plugins
|
@@ -1,107 +0,0 @@
|
|
1
|
-
module LogStash
|
2
|
-
module PluginMixins
|
3
|
-
module KafkaAvroSchemaRegistry
|
4
|
-
|
5
|
-
def self.included(base)
|
6
|
-
base.extend(self)
|
7
|
-
base.setup_schema_registry_config
|
8
|
-
end
|
9
|
-
|
10
|
-
def setup_schema_registry_config
|
11
|
-
# Option to set key to access Schema Registry.
|
12
|
-
config :schema_registry_key, :validate => :string
|
13
|
-
|
14
|
-
# Option to set secret to access Schema Registry.
|
15
|
-
config :schema_registry_secret, :validate => :password
|
16
|
-
|
17
|
-
# Option to set the endpoint of the Schema Registry.
|
18
|
-
# This option permit the usage of Avro Kafka deserializer which retrieve the schema of the Avro message from an
|
19
|
-
# instance of schema registry. If this option has value `value_deserializer_class` nor `topics_pattern` could be valued
|
20
|
-
config :schema_registry_url, :validate => :uri
|
21
|
-
|
22
|
-
# Option to set the proxy of the Schema Registry.
|
23
|
-
# This option permits to define a proxy to be used to reach the schema registry service instance.
|
24
|
-
config :schema_registry_proxy, :validate => :uri
|
25
|
-
|
26
|
-
# Option to skip validating the schema registry during registration. This can be useful when using
|
27
|
-
# certificate based auth
|
28
|
-
config :schema_registry_validation, :validate => ['auto', 'skip'], :default => 'auto'
|
29
|
-
end
|
30
|
-
|
31
|
-
def check_schema_registry_parameters
|
32
|
-
if @schema_registry_url
|
33
|
-
check_for_schema_registry_conflicts
|
34
|
-
@schema_registry_proxy_host, @schema_registry_proxy_port = split_proxy_into_host_and_port(schema_registry_proxy)
|
35
|
-
check_for_key_and_secret
|
36
|
-
check_for_schema_registry_connectivity_and_subjects if schema_registry_validation?
|
37
|
-
end
|
38
|
-
end
|
39
|
-
|
40
|
-
def schema_registry_validation?
|
41
|
-
return false if schema_registry_validation.to_s == 'skip'
|
42
|
-
return false if using_kerberos? # pre-validation doesn't support kerberos
|
43
|
-
|
44
|
-
true
|
45
|
-
end
|
46
|
-
|
47
|
-
def using_kerberos?
|
48
|
-
security_protocol == "SASL_PLAINTEXT" || security_protocol == "SASL_SSL"
|
49
|
-
end
|
50
|
-
|
51
|
-
private
|
52
|
-
def check_for_schema_registry_conflicts
|
53
|
-
if @value_deserializer_class != LogStash::Inputs::Kafka::DEFAULT_DESERIALIZER_CLASS
|
54
|
-
raise LogStash::ConfigurationError, 'Option schema_registry_url prohibit the customization of value_deserializer_class'
|
55
|
-
end
|
56
|
-
if @topics_pattern && !@topics_pattern.empty?
|
57
|
-
raise LogStash::ConfigurationError, 'Option schema_registry_url prohibit the customization of topics_pattern'
|
58
|
-
end
|
59
|
-
end
|
60
|
-
|
61
|
-
private
|
62
|
-
def check_for_schema_registry_connectivity_and_subjects
|
63
|
-
options = {}
|
64
|
-
if schema_registry_proxy && !schema_registry_proxy.empty?
|
65
|
-
options[:proxy] = schema_registry_proxy.to_s
|
66
|
-
end
|
67
|
-
if schema_registry_key and !schema_registry_key.empty?
|
68
|
-
options[:auth] = {:user => schema_registry_key, :password => schema_registry_secret.value}
|
69
|
-
end
|
70
|
-
client = Manticore::Client.new(options)
|
71
|
-
begin
|
72
|
-
response = client.get(@schema_registry_url.uri.to_s + '/subjects').body
|
73
|
-
rescue Manticore::ManticoreException => e
|
74
|
-
raise LogStash::ConfigurationError.new("Schema registry service doesn't respond, error: #{e.message}")
|
75
|
-
end
|
76
|
-
registered_subjects = JSON.parse response
|
77
|
-
expected_subjects = @topics.map { |t| "#{t}-value"}
|
78
|
-
if (expected_subjects & registered_subjects).size != expected_subjects.size
|
79
|
-
undefined_topic_subjects = expected_subjects - registered_subjects
|
80
|
-
raise LogStash::ConfigurationError, "The schema registry does not contain definitions for required topic subjects: #{undefined_topic_subjects}"
|
81
|
-
end
|
82
|
-
end
|
83
|
-
|
84
|
-
def split_proxy_into_host_and_port(proxy_uri)
|
85
|
-
return nil unless proxy_uri && !proxy_uri.empty?
|
86
|
-
|
87
|
-
port = proxy_uri.port
|
88
|
-
|
89
|
-
host_spec = ""
|
90
|
-
host_spec << proxy_uri.scheme || "http"
|
91
|
-
host_spec << "://"
|
92
|
-
host_spec << "#{proxy_uri.userinfo}@" if proxy_uri.userinfo
|
93
|
-
host_spec << proxy_uri.host
|
94
|
-
|
95
|
-
[host_spec, port]
|
96
|
-
end
|
97
|
-
|
98
|
-
def check_for_key_and_secret
|
99
|
-
if schema_registry_key and !schema_registry_key.empty?
|
100
|
-
if !schema_registry_secret or schema_registry_secret.value.empty?
|
101
|
-
raise LogStash::ConfigurationError, "Setting `schema_registry_secret` is required when `schema_registry_key` is provided."
|
102
|
-
end
|
103
|
-
end
|
104
|
-
end
|
105
|
-
end
|
106
|
-
end
|
107
|
-
end
|
@@ -1,29 +0,0 @@
|
|
1
|
-
module LogStash module PluginMixins module KafkaSupport
|
2
|
-
|
3
|
-
def set_trustore_keystore_config(props)
|
4
|
-
props.put("ssl.truststore.type", ssl_truststore_type) unless ssl_truststore_type.nil?
|
5
|
-
props.put("ssl.truststore.location", ssl_truststore_location) unless ssl_truststore_location.nil?
|
6
|
-
props.put("ssl.truststore.password", ssl_truststore_password.value) unless ssl_truststore_password.nil?
|
7
|
-
|
8
|
-
# Client auth stuff
|
9
|
-
props.put("ssl.keystore.type", ssl_keystore_type) unless ssl_keystore_type.nil?
|
10
|
-
props.put("ssl.key.password", ssl_key_password.value) unless ssl_key_password.nil?
|
11
|
-
props.put("ssl.keystore.location", ssl_keystore_location) unless ssl_keystore_location.nil?
|
12
|
-
props.put("ssl.keystore.password", ssl_keystore_password.value) unless ssl_keystore_password.nil?
|
13
|
-
props.put("ssl.endpoint.identification.algorithm", ssl_endpoint_identification_algorithm) unless ssl_endpoint_identification_algorithm.nil?
|
14
|
-
end
|
15
|
-
|
16
|
-
def set_sasl_config(props)
|
17
|
-
java.lang.System.setProperty("java.security.auth.login.config", jaas_path) unless jaas_path.nil?
|
18
|
-
java.lang.System.setProperty("java.security.krb5.conf", kerberos_config) unless kerberos_config.nil?
|
19
|
-
|
20
|
-
props.put("sasl.mechanism", sasl_mechanism)
|
21
|
-
if sasl_mechanism == "GSSAPI" && sasl_kerberos_service_name.nil?
|
22
|
-
raise LogStash::ConfigurationError, "sasl_kerberos_service_name must be specified when SASL mechanism is GSSAPI"
|
23
|
-
end
|
24
|
-
|
25
|
-
props.put("sasl.kerberos.service.name", sasl_kerberos_service_name) unless sasl_kerberos_service_name.nil?
|
26
|
-
props.put("sasl.jaas.config", sasl_jaas_config) unless sasl_jaas_config.nil?
|
27
|
-
end
|
28
|
-
|
29
|
-
end end end
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
data/vendor/jar-dependencies/org/glassfish/jersey/core/jersey-common/2.33/jersey-common-2.33.jar
DELETED
Binary file
|
Binary file
|