fluent-plugin-kafka 0.16.3 → 0.17.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/ISSUE_TEMPLATE/bug_report.yaml +71 -0
- data/.github/ISSUE_TEMPLATE/config.yml +5 -0
- data/.github/ISSUE_TEMPLATE/feature_request.yaml +38 -0
- data/.github/workflows/linux.yml +2 -0
- data/.github/workflows/stale-actions.yml +22 -0
- data/ChangeLog +18 -0
- data/Gemfile +2 -0
- data/README.md +38 -5
- data/fluent-plugin-kafka.gemspec +4 -2
- data/lib/fluent/plugin/in_kafka.rb +4 -3
- data/lib/fluent/plugin/in_kafka_group.rb +3 -3
- data/lib/fluent/plugin/out_kafka.rb +14 -7
- data/lib/fluent/plugin/out_kafka2.rb +27 -9
- data/lib/fluent/plugin/out_kafka_buffered.rb +14 -7
- data/lib/fluent/plugin/out_rdkafka.rb +21 -13
- data/lib/fluent/plugin/out_rdkafka2.rb +111 -12
- data/test/helper.rb +5 -0
- data/test/plugin/test_kafka_plugin_util.rb +18 -12
- data/test/plugin/test_out_kafka.rb +10 -0
- data/test/plugin/test_out_kafka2.rb +116 -0
- data/test/plugin/test_out_kafka_buffered.rb +68 -0
- data/test/plugin/test_out_rdkafka2.rb +167 -0
- metadata +42 -4
@@ -0,0 +1,167 @@
|
|
1
|
+
require 'helper'
|
2
|
+
require 'fluent/test/helpers'
|
3
|
+
require 'fluent/test/driver/input'
|
4
|
+
require 'fluent/test/driver/output'
|
5
|
+
require 'securerandom'
|
6
|
+
|
7
|
+
class Rdkafka2OutputTest < Test::Unit::TestCase
|
8
|
+
include Fluent::Test::Helpers
|
9
|
+
|
10
|
+
def have_rdkafka
|
11
|
+
begin
|
12
|
+
require 'fluent/plugin/out_rdkafka2'
|
13
|
+
true
|
14
|
+
rescue LoadError
|
15
|
+
false
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
def setup
|
20
|
+
omit_unless(have_rdkafka, "rdkafka isn't installed")
|
21
|
+
Fluent::Test.setup
|
22
|
+
end
|
23
|
+
|
24
|
+
def base_config
|
25
|
+
config_element('ROOT', '', {"@type" => "rdkafka2"}, [
|
26
|
+
config_element('format', "", {"@type" => "json"})
|
27
|
+
])
|
28
|
+
end
|
29
|
+
|
30
|
+
def config(default_topic: "kitagawakeiko")
|
31
|
+
base_config + config_element('ROOT', '', {"default_topic" => default_topic,
|
32
|
+
"brokers" => "localhost:9092"}, [
|
33
|
+
])
|
34
|
+
end
|
35
|
+
|
36
|
+
def create_driver(conf = config, tag='test')
|
37
|
+
Fluent::Test::Driver::Output.new(Fluent::Rdkafka2Output).configure(conf)
|
38
|
+
end
|
39
|
+
|
40
|
+
def test_configure
|
41
|
+
assert_nothing_raised(Fluent::ConfigError) {
|
42
|
+
create_driver(base_config)
|
43
|
+
}
|
44
|
+
|
45
|
+
assert_nothing_raised(Fluent::ConfigError) {
|
46
|
+
create_driver(config)
|
47
|
+
}
|
48
|
+
|
49
|
+
assert_nothing_raised(Fluent::ConfigError) {
|
50
|
+
create_driver(config + config_element('buffer', "", {"@type" => "memory"}))
|
51
|
+
}
|
52
|
+
|
53
|
+
d = create_driver
|
54
|
+
assert_equal 'kitagawakeiko', d.instance.default_topic
|
55
|
+
assert_equal 'localhost:9092', d.instance.brokers
|
56
|
+
end
|
57
|
+
|
58
|
+
def test_mutli_worker_support
|
59
|
+
d = create_driver
|
60
|
+
assert_equal true, d.instance.multi_workers_ready?
|
61
|
+
end
|
62
|
+
|
63
|
+
class WriteTest < self
|
64
|
+
TOPIC_NAME = "kafka-output-#{SecureRandom.uuid}"
|
65
|
+
|
66
|
+
INPUT_CONFIG = %[
|
67
|
+
@type kafka
|
68
|
+
brokers localhost:9092
|
69
|
+
format json
|
70
|
+
@label @kafka
|
71
|
+
topics #{TOPIC_NAME}
|
72
|
+
]
|
73
|
+
|
74
|
+
def create_target_driver(conf = INPUT_CONFIG)
|
75
|
+
Fluent::Test::Driver::Input.new(Fluent::KafkaInput).configure(conf)
|
76
|
+
end
|
77
|
+
|
78
|
+
def setup
|
79
|
+
@kafka = nil
|
80
|
+
omit_unless(have_rdkafka, "rdkafka isn't installed")
|
81
|
+
@kafka = Kafka.new(["localhost:9092"], client_id: 'kafka')
|
82
|
+
end
|
83
|
+
|
84
|
+
def teardown
|
85
|
+
if @kafka
|
86
|
+
@kafka.delete_topic(TOPIC_NAME)
|
87
|
+
@kafka.close
|
88
|
+
end
|
89
|
+
end
|
90
|
+
|
91
|
+
def test_write
|
92
|
+
target_driver = create_target_driver
|
93
|
+
expected_message = {"a" => 2}
|
94
|
+
target_driver.run(expect_records: 1, timeout: 5) do
|
95
|
+
sleep 2
|
96
|
+
d = create_driver(config(default_topic: TOPIC_NAME))
|
97
|
+
d.run do
|
98
|
+
d.feed("test", event_time, expected_message)
|
99
|
+
end
|
100
|
+
end
|
101
|
+
actual_messages = target_driver.events.collect { |event| event[2] }
|
102
|
+
assert_equal([expected_message], actual_messages)
|
103
|
+
end
|
104
|
+
|
105
|
+
def test_write_with_use_event_time
|
106
|
+
input_config = %[
|
107
|
+
@type kafka
|
108
|
+
brokers localhost:9092
|
109
|
+
format json
|
110
|
+
@label @kafka
|
111
|
+
topics #{TOPIC_NAME}
|
112
|
+
time_source kafka
|
113
|
+
]
|
114
|
+
target_driver = create_target_driver(input_config)
|
115
|
+
expected_message = {"a" => 2}
|
116
|
+
now = event_time
|
117
|
+
target_driver.run(expect_records: 1, timeout: 5) do
|
118
|
+
sleep 2
|
119
|
+
d = create_driver(config(default_topic: TOPIC_NAME) + config_element('ROOT', '', {"use_event_time" => true}))
|
120
|
+
d.run do
|
121
|
+
d.feed("test", now, expected_message)
|
122
|
+
end
|
123
|
+
end
|
124
|
+
actual_time = target_driver.events.collect { |event| event[1] }.last
|
125
|
+
assert_in_delta(actual_time, now, 0.001) # expects millseconds precision
|
126
|
+
actual_messages = target_driver.events.collect { |event| event[2] }
|
127
|
+
assert_equal([expected_message], actual_messages)
|
128
|
+
end
|
129
|
+
|
130
|
+
def test_exclude_fields
|
131
|
+
conf = config(default_topic: TOPIC_NAME) +
|
132
|
+
config_element('ROOT', '', {"exclude_fields" => "$.foo"}, [])
|
133
|
+
target_driver = create_target_driver
|
134
|
+
target_driver.run(expect_records: 1, timeout: 5) do
|
135
|
+
sleep 2
|
136
|
+
d = create_driver(conf)
|
137
|
+
d.run do
|
138
|
+
d.feed('test', event_time, {'a' => 'b', 'foo' => 'bar', 'message' => 'test'})
|
139
|
+
end
|
140
|
+
end
|
141
|
+
actual_messages = target_driver.events.collect { |event| event[2] }
|
142
|
+
assert_equal([{'a' => 'b', 'message' => 'test'}], actual_messages)
|
143
|
+
end
|
144
|
+
|
145
|
+
def test_max_enqueue_bytes_per_second
|
146
|
+
conf = config(default_topic: TOPIC_NAME) +
|
147
|
+
config_element('ROOT', '', {"max_enqueue_bytes_per_second" => 32 * 3}, [])
|
148
|
+
target_driver = create_target_driver
|
149
|
+
expected_messages = []
|
150
|
+
target_driver.run(expect_records: 9, timeout: 10) do
|
151
|
+
sleep 2
|
152
|
+
d = create_driver(conf)
|
153
|
+
start_time = Fluent::Clock.now
|
154
|
+
d.run do
|
155
|
+
9.times do |i|
|
156
|
+
message = {"message" => "32bytes message: #{i}"}
|
157
|
+
d.feed("test", event_time, message)
|
158
|
+
expected_messages << message
|
159
|
+
end
|
160
|
+
end
|
161
|
+
assert_in_delta(2.0, Fluent::Clock.now - start_time, 0.5)
|
162
|
+
end
|
163
|
+
actual_messages = target_driver.events.collect { |event| event[2] }
|
164
|
+
assert_equal(expected_messages, actual_messages)
|
165
|
+
end
|
166
|
+
end
|
167
|
+
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-kafka
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.17.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Hidemasa Togashi
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2021-
|
12
|
+
date: 2021-11-26 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: fluentd
|
@@ -51,7 +51,7 @@ dependencies:
|
|
51
51
|
requirements:
|
52
52
|
- - ">="
|
53
53
|
- !ruby/object:Gem::Version
|
54
|
-
version: 1.
|
54
|
+
version: 1.4.0
|
55
55
|
- - "<"
|
56
56
|
- !ruby/object:Gem::Version
|
57
57
|
version: '2'
|
@@ -61,7 +61,7 @@ dependencies:
|
|
61
61
|
requirements:
|
62
62
|
- - ">="
|
63
63
|
- !ruby/object:Gem::Version
|
64
|
-
version: 1.
|
64
|
+
version: 1.4.0
|
65
65
|
- - "<"
|
66
66
|
- !ruby/object:Gem::Version
|
67
67
|
version: '2'
|
@@ -93,6 +93,20 @@ dependencies:
|
|
93
93
|
- - ">="
|
94
94
|
- !ruby/object:Gem::Version
|
95
95
|
version: 3.0.8
|
96
|
+
- !ruby/object:Gem::Dependency
|
97
|
+
name: test-unit-rr
|
98
|
+
requirement: !ruby/object:Gem::Requirement
|
99
|
+
requirements:
|
100
|
+
- - "~>"
|
101
|
+
- !ruby/object:Gem::Version
|
102
|
+
version: '1.0'
|
103
|
+
type: :development
|
104
|
+
prerelease: false
|
105
|
+
version_requirements: !ruby/object:Gem::Requirement
|
106
|
+
requirements:
|
107
|
+
- - "~>"
|
108
|
+
- !ruby/object:Gem::Version
|
109
|
+
version: '1.0'
|
96
110
|
- !ruby/object:Gem::Dependency
|
97
111
|
name: webrick
|
98
112
|
requirement: !ruby/object:Gem::Requirement
|
@@ -107,6 +121,20 @@ dependencies:
|
|
107
121
|
- - ">="
|
108
122
|
- !ruby/object:Gem::Version
|
109
123
|
version: '0'
|
124
|
+
- !ruby/object:Gem::Dependency
|
125
|
+
name: digest-murmurhash
|
126
|
+
requirement: !ruby/object:Gem::Requirement
|
127
|
+
requirements:
|
128
|
+
- - ">="
|
129
|
+
- !ruby/object:Gem::Version
|
130
|
+
version: '0'
|
131
|
+
type: :development
|
132
|
+
prerelease: false
|
133
|
+
version_requirements: !ruby/object:Gem::Requirement
|
134
|
+
requirements:
|
135
|
+
- - ">="
|
136
|
+
- !ruby/object:Gem::Version
|
137
|
+
version: '0'
|
110
138
|
description: Fluentd plugin for Apache Kafka > 0.8
|
111
139
|
email:
|
112
140
|
- togachiro@gmail.com
|
@@ -115,7 +143,11 @@ executables: []
|
|
115
143
|
extensions: []
|
116
144
|
extra_rdoc_files: []
|
117
145
|
files:
|
146
|
+
- ".github/ISSUE_TEMPLATE/bug_report.yaml"
|
147
|
+
- ".github/ISSUE_TEMPLATE/config.yml"
|
148
|
+
- ".github/ISSUE_TEMPLATE/feature_request.yaml"
|
118
149
|
- ".github/workflows/linux.yml"
|
150
|
+
- ".github/workflows/stale-actions.yml"
|
119
151
|
- ".gitignore"
|
120
152
|
- ChangeLog
|
121
153
|
- Gemfile
|
@@ -139,6 +171,9 @@ files:
|
|
139
171
|
- test/plugin/test_in_kafka_group.rb
|
140
172
|
- test/plugin/test_kafka_plugin_util.rb
|
141
173
|
- test/plugin/test_out_kafka.rb
|
174
|
+
- test/plugin/test_out_kafka2.rb
|
175
|
+
- test/plugin/test_out_kafka_buffered.rb
|
176
|
+
- test/plugin/test_out_rdkafka2.rb
|
142
177
|
homepage: https://github.com/fluent/fluent-plugin-kafka
|
143
178
|
licenses:
|
144
179
|
- Apache-2.0
|
@@ -168,3 +203,6 @@ test_files:
|
|
168
203
|
- test/plugin/test_in_kafka_group.rb
|
169
204
|
- test/plugin/test_kafka_plugin_util.rb
|
170
205
|
- test/plugin/test_out_kafka.rb
|
206
|
+
- test/plugin/test_out_kafka2.rb
|
207
|
+
- test/plugin/test_out_kafka_buffered.rb
|
208
|
+
- test/plugin/test_out_rdkafka2.rb
|