fluent-plugin-kafka 0.16.3 → 0.17.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,167 @@
1
+ require 'helper'
2
+ require 'fluent/test/helpers'
3
+ require 'fluent/test/driver/input'
4
+ require 'fluent/test/driver/output'
5
+ require 'securerandom'
6
+
7
+ class Rdkafka2OutputTest < Test::Unit::TestCase
8
+ include Fluent::Test::Helpers
9
+
10
+ def have_rdkafka
11
+ begin
12
+ require 'fluent/plugin/out_rdkafka2'
13
+ true
14
+ rescue LoadError
15
+ false
16
+ end
17
+ end
18
+
19
+ def setup
20
+ omit_unless(have_rdkafka, "rdkafka isn't installed")
21
+ Fluent::Test.setup
22
+ end
23
+
24
+ def base_config
25
+ config_element('ROOT', '', {"@type" => "rdkafka2"}, [
26
+ config_element('format', "", {"@type" => "json"})
27
+ ])
28
+ end
29
+
30
+ def config(default_topic: "kitagawakeiko")
31
+ base_config + config_element('ROOT', '', {"default_topic" => default_topic,
32
+ "brokers" => "localhost:9092"}, [
33
+ ])
34
+ end
35
+
36
+ def create_driver(conf = config, tag='test')
37
+ Fluent::Test::Driver::Output.new(Fluent::Rdkafka2Output).configure(conf)
38
+ end
39
+
40
+ def test_configure
41
+ assert_nothing_raised(Fluent::ConfigError) {
42
+ create_driver(base_config)
43
+ }
44
+
45
+ assert_nothing_raised(Fluent::ConfigError) {
46
+ create_driver(config)
47
+ }
48
+
49
+ assert_nothing_raised(Fluent::ConfigError) {
50
+ create_driver(config + config_element('buffer', "", {"@type" => "memory"}))
51
+ }
52
+
53
+ d = create_driver
54
+ assert_equal 'kitagawakeiko', d.instance.default_topic
55
+ assert_equal 'localhost:9092', d.instance.brokers
56
+ end
57
+
58
+ def test_mutli_worker_support
59
+ d = create_driver
60
+ assert_equal true, d.instance.multi_workers_ready?
61
+ end
62
+
63
+ class WriteTest < self
64
+ TOPIC_NAME = "kafka-output-#{SecureRandom.uuid}"
65
+
66
+ INPUT_CONFIG = %[
67
+ @type kafka
68
+ brokers localhost:9092
69
+ format json
70
+ @label @kafka
71
+ topics #{TOPIC_NAME}
72
+ ]
73
+
74
+ def create_target_driver(conf = INPUT_CONFIG)
75
+ Fluent::Test::Driver::Input.new(Fluent::KafkaInput).configure(conf)
76
+ end
77
+
78
+ def setup
79
+ @kafka = nil
80
+ omit_unless(have_rdkafka, "rdkafka isn't installed")
81
+ @kafka = Kafka.new(["localhost:9092"], client_id: 'kafka')
82
+ end
83
+
84
+ def teardown
85
+ if @kafka
86
+ @kafka.delete_topic(TOPIC_NAME)
87
+ @kafka.close
88
+ end
89
+ end
90
+
91
+ def test_write
92
+ target_driver = create_target_driver
93
+ expected_message = {"a" => 2}
94
+ target_driver.run(expect_records: 1, timeout: 5) do
95
+ sleep 2
96
+ d = create_driver(config(default_topic: TOPIC_NAME))
97
+ d.run do
98
+ d.feed("test", event_time, expected_message)
99
+ end
100
+ end
101
+ actual_messages = target_driver.events.collect { |event| event[2] }
102
+ assert_equal([expected_message], actual_messages)
103
+ end
104
+
105
+ def test_write_with_use_event_time
106
+ input_config = %[
107
+ @type kafka
108
+ brokers localhost:9092
109
+ format json
110
+ @label @kafka
111
+ topics #{TOPIC_NAME}
112
+ time_source kafka
113
+ ]
114
+ target_driver = create_target_driver(input_config)
115
+ expected_message = {"a" => 2}
116
+ now = event_time
117
+ target_driver.run(expect_records: 1, timeout: 5) do
118
+ sleep 2
119
+ d = create_driver(config(default_topic: TOPIC_NAME) + config_element('ROOT', '', {"use_event_time" => true}))
120
+ d.run do
121
+ d.feed("test", now, expected_message)
122
+ end
123
+ end
124
+ actual_time = target_driver.events.collect { |event| event[1] }.last
125
+ assert_in_delta(actual_time, now, 0.001) # expects millseconds precision
126
+ actual_messages = target_driver.events.collect { |event| event[2] }
127
+ assert_equal([expected_message], actual_messages)
128
+ end
129
+
130
+ def test_exclude_fields
131
+ conf = config(default_topic: TOPIC_NAME) +
132
+ config_element('ROOT', '', {"exclude_fields" => "$.foo"}, [])
133
+ target_driver = create_target_driver
134
+ target_driver.run(expect_records: 1, timeout: 5) do
135
+ sleep 2
136
+ d = create_driver(conf)
137
+ d.run do
138
+ d.feed('test', event_time, {'a' => 'b', 'foo' => 'bar', 'message' => 'test'})
139
+ end
140
+ end
141
+ actual_messages = target_driver.events.collect { |event| event[2] }
142
+ assert_equal([{'a' => 'b', 'message' => 'test'}], actual_messages)
143
+ end
144
+
145
+ def test_max_enqueue_bytes_per_second
146
+ conf = config(default_topic: TOPIC_NAME) +
147
+ config_element('ROOT', '', {"max_enqueue_bytes_per_second" => 32 * 3}, [])
148
+ target_driver = create_target_driver
149
+ expected_messages = []
150
+ target_driver.run(expect_records: 9, timeout: 10) do
151
+ sleep 2
152
+ d = create_driver(conf)
153
+ start_time = Fluent::Clock.now
154
+ d.run do
155
+ 9.times do |i|
156
+ message = {"message" => "32bytes message: #{i}"}
157
+ d.feed("test", event_time, message)
158
+ expected_messages << message
159
+ end
160
+ end
161
+ assert_in_delta(2.0, Fluent::Clock.now - start_time, 0.5)
162
+ end
163
+ actual_messages = target_driver.events.collect { |event| event[2] }
164
+ assert_equal(expected_messages, actual_messages)
165
+ end
166
+ end
167
+ end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-kafka
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.16.3
4
+ version: 0.17.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Hidemasa Togashi
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2021-05-17 00:00:00.000000000 Z
12
+ date: 2021-11-26 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: fluentd
@@ -51,7 +51,7 @@ dependencies:
51
51
  requirements:
52
52
  - - ">="
53
53
  - !ruby/object:Gem::Version
54
- version: 1.2.0
54
+ version: 1.4.0
55
55
  - - "<"
56
56
  - !ruby/object:Gem::Version
57
57
  version: '2'
@@ -61,7 +61,7 @@ dependencies:
61
61
  requirements:
62
62
  - - ">="
63
63
  - !ruby/object:Gem::Version
64
- version: 1.2.0
64
+ version: 1.4.0
65
65
  - - "<"
66
66
  - !ruby/object:Gem::Version
67
67
  version: '2'
@@ -93,6 +93,20 @@ dependencies:
93
93
  - - ">="
94
94
  - !ruby/object:Gem::Version
95
95
  version: 3.0.8
96
+ - !ruby/object:Gem::Dependency
97
+ name: test-unit-rr
98
+ requirement: !ruby/object:Gem::Requirement
99
+ requirements:
100
+ - - "~>"
101
+ - !ruby/object:Gem::Version
102
+ version: '1.0'
103
+ type: :development
104
+ prerelease: false
105
+ version_requirements: !ruby/object:Gem::Requirement
106
+ requirements:
107
+ - - "~>"
108
+ - !ruby/object:Gem::Version
109
+ version: '1.0'
96
110
  - !ruby/object:Gem::Dependency
97
111
  name: webrick
98
112
  requirement: !ruby/object:Gem::Requirement
@@ -107,6 +121,20 @@ dependencies:
107
121
  - - ">="
108
122
  - !ruby/object:Gem::Version
109
123
  version: '0'
124
+ - !ruby/object:Gem::Dependency
125
+ name: digest-murmurhash
126
+ requirement: !ruby/object:Gem::Requirement
127
+ requirements:
128
+ - - ">="
129
+ - !ruby/object:Gem::Version
130
+ version: '0'
131
+ type: :development
132
+ prerelease: false
133
+ version_requirements: !ruby/object:Gem::Requirement
134
+ requirements:
135
+ - - ">="
136
+ - !ruby/object:Gem::Version
137
+ version: '0'
110
138
  description: Fluentd plugin for Apache Kafka > 0.8
111
139
  email:
112
140
  - togachiro@gmail.com
@@ -115,7 +143,11 @@ executables: []
115
143
  extensions: []
116
144
  extra_rdoc_files: []
117
145
  files:
146
+ - ".github/ISSUE_TEMPLATE/bug_report.yaml"
147
+ - ".github/ISSUE_TEMPLATE/config.yml"
148
+ - ".github/ISSUE_TEMPLATE/feature_request.yaml"
118
149
  - ".github/workflows/linux.yml"
150
+ - ".github/workflows/stale-actions.yml"
119
151
  - ".gitignore"
120
152
  - ChangeLog
121
153
  - Gemfile
@@ -139,6 +171,9 @@ files:
139
171
  - test/plugin/test_in_kafka_group.rb
140
172
  - test/plugin/test_kafka_plugin_util.rb
141
173
  - test/plugin/test_out_kafka.rb
174
+ - test/plugin/test_out_kafka2.rb
175
+ - test/plugin/test_out_kafka_buffered.rb
176
+ - test/plugin/test_out_rdkafka2.rb
142
177
  homepage: https://github.com/fluent/fluent-plugin-kafka
143
178
  licenses:
144
179
  - Apache-2.0
@@ -168,3 +203,6 @@ test_files:
168
203
  - test/plugin/test_in_kafka_group.rb
169
204
  - test/plugin/test_kafka_plugin_util.rb
170
205
  - test/plugin/test_out_kafka.rb
206
+ - test/plugin/test_out_kafka2.rb
207
+ - test/plugin/test_out_kafka_buffered.rb
208
+ - test/plugin/test_out_rdkafka2.rb