fluent-plugin-kafka 0.19.7 → 0.19.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/dependabot.yml +17 -0
- data/.github/workflows/linux.yml +2 -2
- data/.github/workflows/stale-actions.yml +1 -1
- data/ChangeLog +53 -0
- data/README.md +13 -0
- data/fluent-plugin-kafka.gemspec +1 -1
- data/lib/fluent/plugin/in_kafka.rb +19 -12
- data/lib/fluent/plugin/in_kafka_group.rb +4 -4
- data/lib/fluent/plugin/in_rdkafka_group.rb +15 -3
- data/lib/fluent/plugin/kafka_plugin_util.rb +106 -1
- data/lib/fluent/plugin/out_kafka.rb +3 -3
- data/lib/fluent/plugin/out_kafka2.rb +4 -2
- data/lib/fluent/plugin/out_kafka_buffered.rb +3 -3
- data/lib/fluent/plugin/out_rdkafka.rb +6 -4
- data/lib/fluent/plugin/out_rdkafka2.rb +21 -48
- data/lib/fluent/plugin/rdkafka_patch/0_13_0.rb +43 -0
- data/lib/fluent/plugin/rdkafka_patch/0_16_0.rb +3 -2
- data/test/plugin/test_in_kafka.rb +127 -0
- data/test/plugin/test_in_rdkafka_group.rb +198 -0
- data/test/plugin/test_kafka_plugin_util.rb +26 -0
- data/test/plugin/test_out_kafka2.rb +99 -0
- data/test/plugin/test_out_rdkafka.rb +105 -0
- data/test/plugin/test_out_rdkafka2.rb +341 -1
- metadata +5 -2
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
class Rdkafka::NativeKafka
|
|
2
|
+
# return false if producer is forcefully closed, otherwise return true
|
|
3
|
+
def close(timeout=nil, object_id=nil)
|
|
4
|
+
return true if closed?
|
|
5
|
+
|
|
6
|
+
synchronize do
|
|
7
|
+
# Indicate to the outside world that we are closing
|
|
8
|
+
@closing = true
|
|
9
|
+
|
|
10
|
+
thread_status = :unknown
|
|
11
|
+
if @polling_thread
|
|
12
|
+
# Indicate to polling thread that we're closing
|
|
13
|
+
@polling_thread[:closing] = true
|
|
14
|
+
|
|
15
|
+
# Wait for the polling thread to finish up,
|
|
16
|
+
# this can be aborted in practice if this
|
|
17
|
+
# code runs from a finalizer.
|
|
18
|
+
thread_status = @polling_thread.join(timeout)
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
# Destroy the client after locking both mutexes
|
|
22
|
+
@poll_mutex.lock
|
|
23
|
+
|
|
24
|
+
# This check prevents a race condition, where we would enter the close in two threads
|
|
25
|
+
# and after unlocking the primary one that hold the lock but finished, ours would be unlocked
|
|
26
|
+
# and would continue to run, trying to destroy inner twice
|
|
27
|
+
if @inner
|
|
28
|
+
Rdkafka::Bindings.rd_kafka_destroy(@inner)
|
|
29
|
+
@inner = nil
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
!thread_status.nil?
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
class Rdkafka::Producer
|
|
38
|
+
def close(timeout = nil)
|
|
39
|
+
return true if closed?
|
|
40
|
+
ObjectSpace.undefine_finalizer(self)
|
|
41
|
+
@native_kafka.close(timeout)
|
|
42
|
+
end
|
|
43
|
+
end
|
|
@@ -24,7 +24,7 @@ class Rdkafka::NativeKafka
|
|
|
24
24
|
# This check prevents a race condition, where we would enter the close in two threads
|
|
25
25
|
# and after unlocking the primary one that hold the lock but finished, ours would be unlocked
|
|
26
26
|
# and would continue to run, trying to destroy inner twice
|
|
27
|
-
|
|
27
|
+
return unless @inner
|
|
28
28
|
|
|
29
29
|
Rdkafka::Bindings.rd_kafka_destroy(@inner)
|
|
30
30
|
@inner = nil
|
|
@@ -40,7 +40,7 @@ class Rdkafka::Producer
|
|
|
40
40
|
return true if closed?
|
|
41
41
|
ObjectSpace.undefine_finalizer(self)
|
|
42
42
|
|
|
43
|
-
@native_kafka.close(timeout) do
|
|
43
|
+
closed = @native_kafka.close(timeout) do
|
|
44
44
|
# We need to remove the topics references objects before we destroy the producer,
|
|
45
45
|
# otherwise they would leak out
|
|
46
46
|
@topics_refs_map.each_value do |refs|
|
|
@@ -51,5 +51,6 @@ class Rdkafka::Producer
|
|
|
51
51
|
end
|
|
52
52
|
|
|
53
53
|
@topics_refs_map.clear
|
|
54
|
+
closed
|
|
54
55
|
end
|
|
55
56
|
end
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
require 'helper'
|
|
2
2
|
require 'fluent/test/driver/input'
|
|
3
3
|
require 'securerandom'
|
|
4
|
+
require 'json'
|
|
4
5
|
|
|
5
6
|
class KafkaInputTest < Test::Unit::TestCase
|
|
6
7
|
def setup
|
|
@@ -34,6 +35,132 @@ class KafkaInputTest < Test::Unit::TestCase
|
|
|
34
35
|
assert_false d.instance.multi_workers_ready?
|
|
35
36
|
end
|
|
36
37
|
|
|
38
|
+
class TopicWatcherTest < self
|
|
39
|
+
FakeMessage = Struct.new(:value, :key, :offset, :create_time)
|
|
40
|
+
|
|
41
|
+
class FakeRouter
|
|
42
|
+
attr_reader :emitted
|
|
43
|
+
|
|
44
|
+
def initialize
|
|
45
|
+
@emitted = []
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
def emit_stream(tag, es)
|
|
49
|
+
records = []
|
|
50
|
+
es.each { |time, record| records << record }
|
|
51
|
+
@emitted << [tag, records]
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
def create_topic_watcher(messages, router, tag_source: :record, add_prefix: nil, add_suffix: nil)
|
|
56
|
+
kafka = Object.new
|
|
57
|
+
kafka.define_singleton_method(:fetch_messages) { |**args| messages }
|
|
58
|
+
parser = Proc.new { |msg, te| JSON.parse(msg.value) }
|
|
59
|
+
|
|
60
|
+
Fluent::KafkaInput::TopicWatcher.new(
|
|
61
|
+
Fluent::KafkaInput::TopicEntry.new(TOPIC_NAME, 0, 0),
|
|
62
|
+
kafka,
|
|
63
|
+
1,
|
|
64
|
+
parser,
|
|
65
|
+
add_prefix,
|
|
66
|
+
add_suffix,
|
|
67
|
+
nil,
|
|
68
|
+
router,
|
|
69
|
+
nil,
|
|
70
|
+
:now,
|
|
71
|
+
'time',
|
|
72
|
+
tag_source,
|
|
73
|
+
'tag')
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
def tagged_message(tag, message, offset)
|
|
77
|
+
FakeMessage.new({'tag' => tag, 'message' => message}.to_json, nil, offset, Time.now)
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
def test_consume_with_tag_source_record_emits_stream_per_tag
|
|
81
|
+
messages = [
|
|
82
|
+
tagged_message('app.trusted', 'record 1', 0),
|
|
83
|
+
tagged_message('app.trusted', 'record 2', 1),
|
|
84
|
+
tagged_message('attacker.controlled', 'record 3', 2),
|
|
85
|
+
]
|
|
86
|
+
router = FakeRouter.new
|
|
87
|
+
create_topic_watcher(messages, router).consume
|
|
88
|
+
|
|
89
|
+
assert_equal([['app.trusted', ['record 1', 'record 2']],
|
|
90
|
+
['attacker.controlled', ['record 3']]],
|
|
91
|
+
router.emitted.map { |tag, records| [tag, records.map { |r| r['message'] }] })
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
def test_consume_with_tag_source_record_applies_prefix_and_suffix_per_tag
|
|
95
|
+
messages = [
|
|
96
|
+
tagged_message('app.trusted', 'record 1', 0),
|
|
97
|
+
tagged_message('attacker.controlled', 'record 2', 1),
|
|
98
|
+
]
|
|
99
|
+
router = FakeRouter.new
|
|
100
|
+
create_topic_watcher(messages, router, add_prefix: 'prefix', add_suffix: 'suffix').consume
|
|
101
|
+
|
|
102
|
+
assert_equal(['prefix.app.trusted.suffix', 'prefix.attacker.controlled.suffix'],
|
|
103
|
+
router.emitted.map(&:first))
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
def test_consume_with_tag_source_record_skips_unparsable_message
|
|
107
|
+
messages = [
|
|
108
|
+
tagged_message('app.trusted', 'record 1', 0),
|
|
109
|
+
FakeMessage.new('this is not json', nil, 1, Time.now),
|
|
110
|
+
tagged_message('attacker.controlled', 'record 2', 2),
|
|
111
|
+
]
|
|
112
|
+
router = FakeRouter.new
|
|
113
|
+
create_topic_watcher(messages, router).consume
|
|
114
|
+
|
|
115
|
+
assert_equal([['app.trusted', ['record 1']],
|
|
116
|
+
['attacker.controlled', ['record 2']]],
|
|
117
|
+
router.emitted.map { |tag, records| [tag, records.map { |r| r['message'] }] })
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
def test_consume_with_tag_source_record_skips_invalid_tag
|
|
121
|
+
messages = [
|
|
122
|
+
tagged_message('app.trusted', 'record 1', 0),
|
|
123
|
+
tagged_message(7, 'record 2', 1),
|
|
124
|
+
FakeMessage.new({'message' => 'record 3'}.to_json, nil, 2, Time.now),
|
|
125
|
+
tagged_message('', 'record 4', 3),
|
|
126
|
+
tagged_message('attacker.controlled', 'record 5', 4),
|
|
127
|
+
]
|
|
128
|
+
router = FakeRouter.new
|
|
129
|
+
watcher = create_topic_watcher(messages, router)
|
|
130
|
+
watcher.consume
|
|
131
|
+
|
|
132
|
+
assert_equal([['app.trusted', ['record 1']],
|
|
133
|
+
['attacker.controlled', ['record 5']]],
|
|
134
|
+
router.emitted.map { |tag, records| [tag, records.map { |r| r['message'] }] })
|
|
135
|
+
assert_equal(5, watcher.instance_variable_get(:@next_offset))
|
|
136
|
+
end
|
|
137
|
+
|
|
138
|
+
def test_consume_advances_offset_when_every_message_is_skipped
|
|
139
|
+
messages = [
|
|
140
|
+
FakeMessage.new('this is not json', nil, 0, Time.now),
|
|
141
|
+
tagged_message(nil, 'record 1', 1),
|
|
142
|
+
]
|
|
143
|
+
router = FakeRouter.new
|
|
144
|
+
watcher = create_topic_watcher(messages, router)
|
|
145
|
+
watcher.consume
|
|
146
|
+
|
|
147
|
+
assert_equal([], router.emitted)
|
|
148
|
+
assert_equal(2, watcher.instance_variable_get(:@next_offset))
|
|
149
|
+
end
|
|
150
|
+
|
|
151
|
+
def test_consume_with_tag_source_topic_emits_single_stream
|
|
152
|
+
messages = [
|
|
153
|
+
tagged_message('app.trusted', 'record 1', 0),
|
|
154
|
+
tagged_message('attacker.controlled', 'record 2', 1),
|
|
155
|
+
]
|
|
156
|
+
router = FakeRouter.new
|
|
157
|
+
create_topic_watcher(messages, router, tag_source: :topic).consume
|
|
158
|
+
|
|
159
|
+
assert_equal([[TOPIC_NAME, ['record 1', 'record 2']]],
|
|
160
|
+
router.emitted.map { |tag, records| [tag, records.map { |r| r['message'] }] })
|
|
161
|
+
end
|
|
162
|
+
end
|
|
163
|
+
|
|
37
164
|
class ConsumeTest < self
|
|
38
165
|
def setup
|
|
39
166
|
@kafka = Kafka.new(["localhost:9092"], client_id: 'kafka')
|
|
@@ -44,6 +44,204 @@ class RdkafkaGroupInputTest < Test::Unit::TestCase
|
|
|
44
44
|
assert_true d.instance.multi_workers_ready?
|
|
45
45
|
end
|
|
46
46
|
|
|
47
|
+
def test_build_config_without_security_parameters
|
|
48
|
+
d = create_driver
|
|
49
|
+
config = d.instance.build_config
|
|
50
|
+
|
|
51
|
+
assert_equal 'localhost:9092', config[:"bootstrap.servers"]
|
|
52
|
+
assert_equal 'test_group', config[:"group.id"]
|
|
53
|
+
assert_equal 'PLAINTEXT', config[:"security.protocol"]
|
|
54
|
+
assert_nil config[:"ssl.endpoint.identification.algorithm"]
|
|
55
|
+
assert_nil config[:"sasl.mechanisms"]
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
def test_build_config_ssl_ca_certs_from_system
|
|
59
|
+
d = create_driver(CONFIG + %[
|
|
60
|
+
ssl_ca_certs_from_system true
|
|
61
|
+
])
|
|
62
|
+
config = d.instance.build_config
|
|
63
|
+
|
|
64
|
+
assert_equal 'SSL', config[:"security.protocol"]
|
|
65
|
+
assert_equal 'https', config[:"ssl.endpoint.identification.algorithm"]
|
|
66
|
+
assert_equal true, config[:"enable.ssl.certificate.verification"]
|
|
67
|
+
assert_nil config[:"ssl.ca.location"]
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
def test_build_config_ssl_client_cert
|
|
71
|
+
d = create_driver(CONFIG + %[
|
|
72
|
+
ssl_ca_cert /path/to/ca_cert.pem
|
|
73
|
+
ssl_client_cert /path/to/cert.pem
|
|
74
|
+
ssl_client_cert_key /path/to/key.pem
|
|
75
|
+
ssl_client_cert_key_password secret
|
|
76
|
+
])
|
|
77
|
+
config = d.instance.build_config
|
|
78
|
+
|
|
79
|
+
assert_equal 'SSL', config[:"security.protocol"]
|
|
80
|
+
assert_equal '/path/to/ca_cert.pem', config[:"ssl.ca.location"]
|
|
81
|
+
assert_equal '/path/to/cert.pem', config[:"ssl.certificate.location"]
|
|
82
|
+
assert_equal '/path/to/key.pem', config[:"ssl.key.location"]
|
|
83
|
+
assert_equal 'secret', config[:"ssl.key.password"]
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
def test_build_config_ssl_verify_hostname_false
|
|
87
|
+
d = create_driver(CONFIG + %[
|
|
88
|
+
ssl_ca_certs_from_system true
|
|
89
|
+
ssl_verify_hostname false
|
|
90
|
+
])
|
|
91
|
+
config = d.instance.build_config
|
|
92
|
+
|
|
93
|
+
assert_equal 'none', config[:"ssl.endpoint.identification.algorithm"]
|
|
94
|
+
assert_equal true, config[:"enable.ssl.certificate.verification"]
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
def test_build_config_sasl_plain_over_ssl
|
|
98
|
+
d = create_driver(CONFIG + %[
|
|
99
|
+
username testuser
|
|
100
|
+
password testpass
|
|
101
|
+
ssl_ca_certs_from_system true
|
|
102
|
+
])
|
|
103
|
+
config = d.instance.build_config
|
|
104
|
+
|
|
105
|
+
assert_equal 'SASL_SSL', config[:"security.protocol"]
|
|
106
|
+
assert_equal 'PLAIN', config[:"sasl.mechanisms"]
|
|
107
|
+
assert_equal 'testuser', config[:"sasl.username"]
|
|
108
|
+
assert_equal 'testpass', config[:"sasl.password"]
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
def test_configure_sasl_plain_without_ssl_raises
|
|
112
|
+
assert_raise(Fluent::ConfigError) {
|
|
113
|
+
create_driver(CONFIG + %[
|
|
114
|
+
username testuser
|
|
115
|
+
password testpass
|
|
116
|
+
])
|
|
117
|
+
}
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
def test_build_config_sasl_plain_without_ssl_allowed_by_sasl_over_ssl
|
|
121
|
+
d = create_driver(CONFIG + %[
|
|
122
|
+
username testuser
|
|
123
|
+
password testpass
|
|
124
|
+
sasl_over_ssl false
|
|
125
|
+
])
|
|
126
|
+
config = d.instance.build_config
|
|
127
|
+
|
|
128
|
+
assert_equal 'SASL_PLAINTEXT', config[:"security.protocol"]
|
|
129
|
+
assert_equal 'testpass', config[:"sasl.password"]
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
data("sha256" => ["sha256", "SCRAM-SHA-256"],
|
|
133
|
+
"sha512" => ["sha512", "SCRAM-SHA-512"])
|
|
134
|
+
def test_build_config_sasl_scram(data)
|
|
135
|
+
mechanism, expected = data
|
|
136
|
+
d = create_driver(CONFIG + %[
|
|
137
|
+
username testuser
|
|
138
|
+
password testpass
|
|
139
|
+
scram_mechanism #{mechanism}
|
|
140
|
+
ssl_ca_certs_from_system true
|
|
141
|
+
])
|
|
142
|
+
config = d.instance.build_config
|
|
143
|
+
|
|
144
|
+
assert_equal expected, config[:"sasl.mechanisms"]
|
|
145
|
+
assert_equal 'SASL_SSL', config[:"security.protocol"]
|
|
146
|
+
end
|
|
147
|
+
|
|
148
|
+
def test_build_config_sasl_scram_without_credentials_warns
|
|
149
|
+
d = create_driver(CONFIG + %[
|
|
150
|
+
scram_mechanism sha256
|
|
151
|
+
ssl_ca_certs_from_system true
|
|
152
|
+
])
|
|
153
|
+
config = d.instance.build_config
|
|
154
|
+
|
|
155
|
+
assert_equal 'SSL', config[:"security.protocol"]
|
|
156
|
+
assert_nil config[:"sasl.mechanisms"]
|
|
157
|
+
assert_true d.logs.any? { |log| log.include?("scram_mechanism is ignored") }
|
|
158
|
+
end
|
|
159
|
+
|
|
160
|
+
def test_build_config_sasl_gssapi
|
|
161
|
+
d = create_driver(CONFIG + %[
|
|
162
|
+
principal kafka/host@REALM
|
|
163
|
+
keytab /path/to/kafka.keytab
|
|
164
|
+
service_name kafka
|
|
165
|
+
ssl_ca_certs_from_system true
|
|
166
|
+
])
|
|
167
|
+
config = d.instance.build_config
|
|
168
|
+
|
|
169
|
+
assert_equal 'SASL_SSL', config[:"security.protocol"]
|
|
170
|
+
assert_equal 'GSSAPI', config[:"sasl.mechanisms"]
|
|
171
|
+
assert_equal 'kafka/host@REALM', config[:"sasl.kerberos.principal"]
|
|
172
|
+
assert_equal '/path/to/kafka.keytab', config[:"sasl.kerberos.keytab"]
|
|
173
|
+
assert_equal 'kafka', config[:"sasl.kerberos.service.name"]
|
|
174
|
+
end
|
|
175
|
+
|
|
176
|
+
def test_build_config_kafka_configs_take_precedence
|
|
177
|
+
conf = %[
|
|
178
|
+
topics #{TOPIC_NAME}
|
|
179
|
+
ssl_ca_certs_from_system true
|
|
180
|
+
kafka_configs {"bootstrap.servers": "localhost:9092", "group.id": "test_group", "security.protocol": "SASL_SSL", "ssl.endpoint.identification.algorithm": "none"}
|
|
181
|
+
<parse>
|
|
182
|
+
@type none
|
|
183
|
+
</parse>
|
|
184
|
+
]
|
|
185
|
+
d = create_driver(conf)
|
|
186
|
+
config = d.instance.build_config
|
|
187
|
+
|
|
188
|
+
assert_equal 'SASL_SSL', config[:"security.protocol"]
|
|
189
|
+
assert_equal 'none', config[:"ssl.endpoint.identification.algorithm"]
|
|
190
|
+
assert_nil config["security.protocol"]
|
|
191
|
+
end
|
|
192
|
+
|
|
193
|
+
def test_configure_sasl_plaintext_in_kafka_configs_raises
|
|
194
|
+
conf = %[
|
|
195
|
+
topics #{TOPIC_NAME}
|
|
196
|
+
kafka_configs {"bootstrap.servers": "localhost:9092", "group.id": "test_group", "security.protocol": "SASL_PLAINTEXT", "sasl.mechanisms": "PLAIN", "sasl.username": "testuser", "sasl.password": "testpass"}
|
|
197
|
+
<parse>
|
|
198
|
+
@type none
|
|
199
|
+
</parse>
|
|
200
|
+
]
|
|
201
|
+
|
|
202
|
+
assert_raise(Fluent::ConfigError) {
|
|
203
|
+
create_driver(conf)
|
|
204
|
+
}
|
|
205
|
+
assert_nothing_raised {
|
|
206
|
+
create_driver(conf + "sasl_over_ssl false\n")
|
|
207
|
+
}
|
|
208
|
+
end
|
|
209
|
+
|
|
210
|
+
data("uppercase" => "SASL_PLAINTEXT",
|
|
211
|
+
"lowercase" => "sasl_plaintext")
|
|
212
|
+
def test_configure_sasl_plaintext_in_kafka_configs_raises_regardless_of_case(protocol)
|
|
213
|
+
conf = %[
|
|
214
|
+
topics #{TOPIC_NAME}
|
|
215
|
+
username testuser
|
|
216
|
+
password testpass
|
|
217
|
+
kafka_configs {"bootstrap.servers": "localhost:9092", "group.id": "test_group", "security.protocol": "#{protocol}"}
|
|
218
|
+
<parse>
|
|
219
|
+
@type none
|
|
220
|
+
</parse>
|
|
221
|
+
]
|
|
222
|
+
|
|
223
|
+
assert_raise(Fluent::ConfigError) {
|
|
224
|
+
create_driver(conf)
|
|
225
|
+
}
|
|
226
|
+
end
|
|
227
|
+
|
|
228
|
+
def test_setup_consumer_uses_build_config
|
|
229
|
+
d = create_driver(CONFIG + %[
|
|
230
|
+
ssl_ca_certs_from_system true
|
|
231
|
+
])
|
|
232
|
+
consumer = Object.new
|
|
233
|
+
stub(consumer).subscribe
|
|
234
|
+
rdkafka_config = Object.new
|
|
235
|
+
stub(rdkafka_config).consumer { consumer }
|
|
236
|
+
passed = nil
|
|
237
|
+
stub(Rdkafka::Config).new { |config| passed = config; rdkafka_config }
|
|
238
|
+
|
|
239
|
+
d.instance.setup_consumer
|
|
240
|
+
|
|
241
|
+
assert_equal 'localhost:9092', passed[:"bootstrap.servers"]
|
|
242
|
+
assert_equal 'SSL', passed[:"security.protocol"]
|
|
243
|
+
end
|
|
244
|
+
|
|
47
245
|
class ConsumeTest < self
|
|
48
246
|
TOPIC_NAME = "kafka-input-#{SecureRandom.uuid}"
|
|
49
247
|
|
|
@@ -6,6 +6,7 @@ class KafkaPluginUtilTest < Test::Unit::TestCase
|
|
|
6
6
|
def self.config_param(name, type, options)
|
|
7
7
|
end
|
|
8
8
|
include Fluent::KafkaPluginUtil::SSLSettings
|
|
9
|
+
include Fluent::KafkaPluginUtil::PartitionSettings
|
|
9
10
|
|
|
10
11
|
def config_param
|
|
11
12
|
end
|
|
@@ -13,6 +14,31 @@ class KafkaPluginUtilTest < Test::Unit::TestCase
|
|
|
13
14
|
Fluent::Test.setup
|
|
14
15
|
end
|
|
15
16
|
|
|
17
|
+
data("integer" => [3, 3],
|
|
18
|
+
"decimal string" => ["3", 3],
|
|
19
|
+
"zero padded string" => ["010", 10],
|
|
20
|
+
"unassigned partition" => [-1, -1],
|
|
21
|
+
"max int32" => [2**31 - 1, 2**31 - 1])
|
|
22
|
+
def test_coerce_partition(data)
|
|
23
|
+
given, expected = data
|
|
24
|
+
assert_equal(expected, coerce_partition(given))
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
data("non numeric string" => ["not-a-number", ArgumentError],
|
|
28
|
+
"empty string" => ["", ArgumentError],
|
|
29
|
+
"hexadecimal string" => ["0x10", ArgumentError],
|
|
30
|
+
"float string" => ["3.5", ArgumentError],
|
|
31
|
+
"float" => [3.9, TypeError],
|
|
32
|
+
"nil" => [nil, TypeError],
|
|
33
|
+
"negative partition" => [-2, RangeError],
|
|
34
|
+
"too big for int32" => [2**31, RangeError])
|
|
35
|
+
def test_coerce_partition_rejects_invalid_value(data)
|
|
36
|
+
given, expected = data
|
|
37
|
+
assert_raise(expected) do
|
|
38
|
+
coerce_partition(given)
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
|
|
16
42
|
def test_read_ssl_file_when_nil
|
|
17
43
|
stub(File).read(anything) do |path|
|
|
18
44
|
path
|
|
@@ -27,6 +27,32 @@ class Kafka2OutputTest < Test::Unit::TestCase
|
|
|
27
27
|
Fluent::Test::Driver::Output.new(Fluent::Kafka2Output).configure(conf)
|
|
28
28
|
end
|
|
29
29
|
|
|
30
|
+
class DummyProducer
|
|
31
|
+
attr_reader :produced
|
|
32
|
+
|
|
33
|
+
# Coerces like ruby-kafka's Producer#produce, which is where an invalid
|
|
34
|
+
# partition raises today
|
|
35
|
+
def produce(value, key:, partition_key:, partition:, headers:, create_time:, topic:)
|
|
36
|
+
@produced ||= []
|
|
37
|
+
@produced << {value: value && value.to_s, key: key && key.to_s,
|
|
38
|
+
partition: partition && Integer(partition)}
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
def deliver_messages
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
def clear_buffer
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
def create_chunk(records, tag: 'test')
|
|
49
|
+
metadata = Fluent::Plugin::Buffer::Metadata.new(nil, tag, nil)
|
|
50
|
+
chunk = Fluent::Plugin::Buffer::MemoryChunk.new(metadata)
|
|
51
|
+
chunk.extend(Fluent::ChunkMessagePackEventStreamer)
|
|
52
|
+
chunk.append(records.map { |record| [event_time, record].to_msgpack })
|
|
53
|
+
chunk
|
|
54
|
+
end
|
|
55
|
+
|
|
30
56
|
def test_configure
|
|
31
57
|
assert_nothing_raised(Fluent::ConfigError) {
|
|
32
58
|
create_driver(base_config)
|
|
@@ -45,6 +71,79 @@ class Kafka2OutputTest < Test::Unit::TestCase
|
|
|
45
71
|
assert_equal ['localhost:9092'], d.instance.brokers
|
|
46
72
|
end
|
|
47
73
|
|
|
74
|
+
data("cert without key" => {"ssl_client_cert" => "/path/to/cert.pem"},
|
|
75
|
+
"key without cert" => {"ssl_client_cert_key" => "/path/to/key.pem"},
|
|
76
|
+
"chain without cert" => {"ssl_client_cert_chain" => "/path/to/chain.pem"},
|
|
77
|
+
"key password without key" => {"ssl_client_cert_key_password" => "secret"})
|
|
78
|
+
def test_configure_incomplete_ssl_client_cert(params)
|
|
79
|
+
conf = config + config_element('ROOT', '', params, [])
|
|
80
|
+
|
|
81
|
+
assert_raise(Fluent::ConfigError) {
|
|
82
|
+
create_driver(conf)
|
|
83
|
+
}
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
def test_configure_ssl_client_cert_with_key
|
|
87
|
+
conf = config + config_element('ROOT', '', {"ssl_client_cert" => "/path/to/cert.pem",
|
|
88
|
+
"ssl_client_cert_key" => "/path/to/key.pem"}, [])
|
|
89
|
+
|
|
90
|
+
assert_nothing_raised(Fluent::ConfigError) {
|
|
91
|
+
create_driver(conf)
|
|
92
|
+
}
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
data("sha256" => "sha256",
|
|
96
|
+
"sha512" => "sha512")
|
|
97
|
+
def test_configure_scram_mechanism(mechanism)
|
|
98
|
+
conf = config + config_element('ROOT', '', {"username" => "testuser",
|
|
99
|
+
"password" => "testpass",
|
|
100
|
+
"scram_mechanism" => mechanism,
|
|
101
|
+
"ssl_ca_certs_from_system" => "true"}, [])
|
|
102
|
+
d = create_driver(conf)
|
|
103
|
+
|
|
104
|
+
assert_equal mechanism, d.instance.scram_mechanism
|
|
105
|
+
|
|
106
|
+
assert_nothing_raised {
|
|
107
|
+
d.instance.refresh_client
|
|
108
|
+
}
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
def test_configure_unsupported_scram_mechanism
|
|
112
|
+
conf = config + config_element('ROOT', '', {"username" => "testuser",
|
|
113
|
+
"password" => "testpass",
|
|
114
|
+
"scram_mechanism" => "sha1"}, [])
|
|
115
|
+
|
|
116
|
+
assert_raise(Fluent::ConfigError) {
|
|
117
|
+
create_driver(conf)
|
|
118
|
+
}
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
data("non numeric partition" => "not-a-number",
|
|
122
|
+
"hash partition" => {"x" => 1},
|
|
123
|
+
"negative partition" => -2,
|
|
124
|
+
"out of int32 range" => 2**31)
|
|
125
|
+
def test_write_skips_event_with_invalid_partition(partition)
|
|
126
|
+
d = create_driver
|
|
127
|
+
producer = DummyProducer.new
|
|
128
|
+
stub(d.instance).get_producer { producer }
|
|
129
|
+
|
|
130
|
+
assert_nothing_raised {
|
|
131
|
+
d.instance.write(create_chunk([{"a" => "b"}, {"a" => "c", "partition" => partition}, {"a" => "d"}]))
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
assert_equal ['{"a":"b"}', '{"a":"d"}'], producer.produced.collect { |message| message[:value] }
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
def test_write_keeps_event_with_zero_padded_partition
|
|
138
|
+
d = create_driver
|
|
139
|
+
producer = DummyProducer.new
|
|
140
|
+
stub(d.instance).get_producer { producer }
|
|
141
|
+
|
|
142
|
+
d.instance.write(create_chunk([{"a" => "b", "partition" => "010"}]))
|
|
143
|
+
|
|
144
|
+
assert_equal [10], producer.produced.collect { |message| message[:partition] }
|
|
145
|
+
end
|
|
146
|
+
|
|
48
147
|
data("crc32" => "crc32",
|
|
49
148
|
"murmur2" => "murmur2")
|
|
50
149
|
def test_partitioner_hash_function(data)
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
require 'helper'
|
|
2
|
+
require 'fluent/test/helpers'
|
|
3
|
+
require 'fluent/test/driver/output'
|
|
4
|
+
|
|
5
|
+
class RdkafkaOutputTest < Test::Unit::TestCase
|
|
6
|
+
include Fluent::Test::Helpers
|
|
7
|
+
|
|
8
|
+
def have_rdkafka
|
|
9
|
+
begin
|
|
10
|
+
require 'fluent/plugin/out_rdkafka'
|
|
11
|
+
true
|
|
12
|
+
rescue LoadError
|
|
13
|
+
false
|
|
14
|
+
end
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
def setup
|
|
18
|
+
omit_unless(have_rdkafka, "rdkafka isn't installed")
|
|
19
|
+
Fluent::Test.setup
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def base_config(params = {})
|
|
23
|
+
config_element('ROOT', '', {"@type" => "rdkafka",
|
|
24
|
+
"brokers" => "localhost:9092"}.merge(params), [])
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def create_driver(conf = base_config)
|
|
28
|
+
Fluent::Test::Driver::Output.new(Fluent::KafkaOutputBuffered2).configure(conf)
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
def test_configure
|
|
32
|
+
d = create_driver
|
|
33
|
+
|
|
34
|
+
assert_equal 'localhost:9092', d.instance.brokers
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
def test_configure_ssl_ca_cert
|
|
38
|
+
d = create_driver(base_config("ssl_ca_cert" => "/path/to/ca_cert.pem"))
|
|
39
|
+
|
|
40
|
+
config = d.instance.build_config
|
|
41
|
+
|
|
42
|
+
assert_equal 'SSL', config[:"security.protocol"]
|
|
43
|
+
assert_equal '/path/to/ca_cert.pem', config[:"ssl.ca.location"]
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
def test_configure_ssl_ca_certs_from_system
|
|
47
|
+
d = create_driver(base_config("ssl_ca_certs_from_system" => "true"))
|
|
48
|
+
|
|
49
|
+
config = d.instance.build_config
|
|
50
|
+
|
|
51
|
+
assert_equal 'SSL', config[:"security.protocol"]
|
|
52
|
+
assert_nil config[:"ssl.ca.location"]
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
def test_configure_ssl_client_cert_without_ca_cert
|
|
56
|
+
d = create_driver(base_config("ssl_client_cert" => "/path/to/cert.pem",
|
|
57
|
+
"ssl_client_cert_key" => "/path/to/key.pem"))
|
|
58
|
+
|
|
59
|
+
config = d.instance.build_config
|
|
60
|
+
|
|
61
|
+
assert_equal 'SSL', config[:"security.protocol"]
|
|
62
|
+
assert_equal '/path/to/cert.pem', config[:"ssl.certificate.location"]
|
|
63
|
+
assert_equal '/path/to/key.pem', config[:"ssl.key.location"]
|
|
64
|
+
assert_nil config[:"ssl.ca.location"]
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
def test_configure_ssl_verify_hostname_default
|
|
68
|
+
d = create_driver(base_config("ssl_ca_cert" => "/path/to/ca_cert.pem"))
|
|
69
|
+
|
|
70
|
+
config = d.instance.build_config
|
|
71
|
+
|
|
72
|
+
assert_equal 'SSL', config[:"security.protocol"]
|
|
73
|
+
assert_equal 'https', config[:"ssl.endpoint.identification.algorithm"]
|
|
74
|
+
assert_equal true, config[:"enable.ssl.certificate.verification"]
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
def test_configure_ssl_verify_hostname_false
|
|
78
|
+
d = create_driver(base_config("ssl_ca_cert" => "/path/to/ca_cert.pem",
|
|
79
|
+
"ssl_verify_hostname" => "false"))
|
|
80
|
+
|
|
81
|
+
config = d.instance.build_config
|
|
82
|
+
|
|
83
|
+
assert_equal 'none', config[:"ssl.endpoint.identification.algorithm"]
|
|
84
|
+
assert_equal true, config[:"enable.ssl.certificate.verification"]
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
def test_configure_without_ssl_has_no_endpoint_identification
|
|
88
|
+
config = create_driver.instance.build_config
|
|
89
|
+
|
|
90
|
+
assert_equal 'PLAINTEXT', config[:"security.protocol"]
|
|
91
|
+
assert_nil config[:"ssl.endpoint.identification.algorithm"]
|
|
92
|
+
assert_nil config[:"enable.ssl.certificate.verification"]
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
def test_configure_sasl_gssapi_over_ssl
|
|
96
|
+
d = create_driver(base_config("principal" => "testuser@EXAMPLE.COM",
|
|
97
|
+
"ssl_client_cert" => "/path/to/cert.pem",
|
|
98
|
+
"ssl_client_cert_key" => "/path/to/key.pem"))
|
|
99
|
+
|
|
100
|
+
config = d.instance.build_config
|
|
101
|
+
|
|
102
|
+
assert_equal 'SASL_SSL', config[:"security.protocol"]
|
|
103
|
+
assert_equal 'GSSAPI', config[:"sasl.mechanisms"]
|
|
104
|
+
end
|
|
105
|
+
end
|