fluent-plugin-kafka 0.16.2 → 0.16.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/linux.yml +10 -0
- data/ChangeLog +3 -0
- data/ci/prepare-kafka-server.sh +33 -0
- data/fluent-plugin-kafka.gemspec +1 -1
- data/lib/fluent/plugin/in_kafka_group.rb +1 -1
- data/test/helper.rb +2 -0
- data/test/plugin/test_in_kafka.rb +66 -0
- data/test/plugin/test_in_kafka_group.rb +67 -0
- metadata +6 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 1cc628766df718a6fff17debcf2cffdcc041419cb96c1c9b6ce1fee532807b58
|
4
|
+
data.tar.gz: 543ccbd91345f45ee75c3e5e84ef5b3414420a0e2557c76a40afc8e308d70735
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 7cb27a4fe28ccc0f31e36449046faf8d4d3389be7820f7cfce8f612aad84b3239234663a71b50523cb08a13e145f78fdc86e93cf9cec28d35d82765b96c9dddc
|
7
|
+
data.tar.gz: 98f52a5d84fb348178f9df6ede6592412ddccd43891c95e9d6937622bff8b02bb4cbb3231c7ca7d709a3ae6cdbfb90d3f00074eefe4cc0703379315098424809
|
data/.github/workflows/linux.yml
CHANGED
@@ -17,10 +17,20 @@ jobs:
|
|
17
17
|
- uses: ruby/setup-ruby@v1
|
18
18
|
with:
|
19
19
|
ruby-version: ${{ matrix.ruby }}
|
20
|
+
- name: Install confluent-kafka
|
21
|
+
run: |
|
22
|
+
sudo apt install -V -y gnupg2 wget
|
23
|
+
wget https://packages.confluent.io/deb/6.0/archive.key
|
24
|
+
sudo gpg2 --homedir /tmp --no-default-keyring --keyring gnupg-ring:/usr/share/keyrings/confluent-archive-keyring.gpg --import archive.key
|
25
|
+
sudo chmod 644 /usr/share/keyrings/confluent-archive-keyring.gpg
|
26
|
+
sudo sh -c 'echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/confluent-archive-keyring.gpg] https://packages.confluent.io/deb/6.0 stable main" > /etc/apt/sources.list.d/confluent.list'
|
27
|
+
sudo apt update
|
28
|
+
sudo apt install -y confluent-community-2.13 openjdk-11-jre netcat-openbsd
|
20
29
|
- name: unit testing
|
21
30
|
env:
|
22
31
|
CI: true
|
23
32
|
run: |
|
33
|
+
sudo ./ci/prepare-kafka-server.sh
|
24
34
|
gem install bundler rake
|
25
35
|
bundle install --jobs 4 --retry 3
|
26
36
|
bundle exec rake test
|
data/ChangeLog
CHANGED
@@ -0,0 +1,33 @@
|
|
1
|
+
#!/bin/sh
|
2
|
+
|
3
|
+
export KAFKA_OPTS=-Dzookeeper.4lw.commands.whitelist=ruok
|
4
|
+
/usr/bin/zookeeper-server-start /etc/kafka/zookeeper.properties &
|
5
|
+
N_POLLING=30
|
6
|
+
n=1
|
7
|
+
while true ; do
|
8
|
+
sleep 1
|
9
|
+
status=$(echo ruok | nc localhost 2181)
|
10
|
+
if [ "$status" = "imok" ]; then
|
11
|
+
break
|
12
|
+
fi
|
13
|
+
n=$((n + 1))
|
14
|
+
if [ $n -ge $N_POLLING ]; then
|
15
|
+
echo "failed to get response from zookeeper-server"
|
16
|
+
exit 1
|
17
|
+
fi
|
18
|
+
done
|
19
|
+
/usr/bin/kafka-server-start /etc/kafka/server.properties &
|
20
|
+
n=1
|
21
|
+
while true ; do
|
22
|
+
sleep 1
|
23
|
+
status=$(/usr/bin/zookeeper-shell localhost:2181 ls /brokers/ids | sed -n 6p)
|
24
|
+
if [ "$status" = "[0]" ]; then
|
25
|
+
break
|
26
|
+
fi
|
27
|
+
n=$((n + 1))
|
28
|
+
if [ $n -ge $N_POLLING ]; then
|
29
|
+
echo "failed to get response from kafka-server"
|
30
|
+
exit 1
|
31
|
+
fi
|
32
|
+
done
|
33
|
+
/usr/bin/kafka-topics --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic test
|
data/fluent-plugin-kafka.gemspec
CHANGED
@@ -13,7 +13,7 @@ Gem::Specification.new do |gem|
|
|
13
13
|
gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
|
14
14
|
gem.name = "fluent-plugin-kafka"
|
15
15
|
gem.require_paths = ["lib"]
|
16
|
-
gem.version = '0.16.
|
16
|
+
gem.version = '0.16.3'
|
17
17
|
gem.required_ruby_version = ">= 2.1.0"
|
18
18
|
|
19
19
|
gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]
|
@@ -345,7 +345,7 @@ class Fluent::KafkaGroupInput < Fluent::Input
|
|
345
345
|
def run
|
346
346
|
while @consumer
|
347
347
|
begin
|
348
|
-
@consumer.each_batch(
|
348
|
+
@consumer.each_batch(**@fetch_opts) { |batch|
|
349
349
|
if @tag_source == :record
|
350
350
|
process_batch_with_record_tag(batch)
|
351
351
|
else
|
data/test/helper.rb
CHANGED
@@ -0,0 +1,66 @@
|
|
1
|
+
require 'helper'
|
2
|
+
require 'fluent/test/driver/input'
|
3
|
+
require 'securerandom'
|
4
|
+
|
5
|
+
class KafkaInputTest < Test::Unit::TestCase
|
6
|
+
def setup
|
7
|
+
Fluent::Test.setup
|
8
|
+
end
|
9
|
+
|
10
|
+
TOPIC_NAME = "kafka-input-#{SecureRandom.uuid}"
|
11
|
+
|
12
|
+
CONFIG = %[
|
13
|
+
@type kafka
|
14
|
+
brokers localhost:9092
|
15
|
+
format text
|
16
|
+
@label @kafka
|
17
|
+
topics #{TOPIC_NAME}
|
18
|
+
]
|
19
|
+
|
20
|
+
def create_driver(conf = CONFIG)
|
21
|
+
Fluent::Test::Driver::Input.new(Fluent::KafkaInput).configure(conf)
|
22
|
+
end
|
23
|
+
|
24
|
+
|
25
|
+
def test_configure
|
26
|
+
d = create_driver
|
27
|
+
assert_equal TOPIC_NAME, d.instance.topics
|
28
|
+
assert_equal 'text', d.instance.format
|
29
|
+
assert_equal 'localhost:9092', d.instance.brokers
|
30
|
+
end
|
31
|
+
|
32
|
+
def test_multi_worker_support
|
33
|
+
d = create_driver
|
34
|
+
assert_false d.instance.multi_workers_ready?
|
35
|
+
end
|
36
|
+
|
37
|
+
class ConsumeTest < self
|
38
|
+
def setup
|
39
|
+
@kafka = Kafka.new(["localhost:9092"], client_id: 'kafka')
|
40
|
+
@producer = @kafka.producer
|
41
|
+
end
|
42
|
+
|
43
|
+
def teardown
|
44
|
+
@kafka.delete_topic(TOPIC_NAME)
|
45
|
+
@kafka.close
|
46
|
+
end
|
47
|
+
|
48
|
+
def test_consume
|
49
|
+
conf = %[
|
50
|
+
@type kafka
|
51
|
+
brokers localhost:9092
|
52
|
+
format text
|
53
|
+
@label @kafka
|
54
|
+
topics #{TOPIC_NAME}
|
55
|
+
]
|
56
|
+
d = create_driver
|
57
|
+
|
58
|
+
d.run(expect_records: 1, timeout: 10) do
|
59
|
+
@producer.produce("Hello, fluent-plugin-kafka!", topic: TOPIC_NAME)
|
60
|
+
@producer.deliver_messages
|
61
|
+
end
|
62
|
+
expected = {'message' => 'Hello, fluent-plugin-kafka!'}
|
63
|
+
assert_equal expected, d.events[0][2]
|
64
|
+
end
|
65
|
+
end
|
66
|
+
end
|
@@ -0,0 +1,67 @@
|
|
1
|
+
require 'helper'
|
2
|
+
require 'fluent/test/driver/input'
|
3
|
+
require 'securerandom'
|
4
|
+
|
5
|
+
class KafkaGroupInputTest < Test::Unit::TestCase
|
6
|
+
def setup
|
7
|
+
Fluent::Test.setup
|
8
|
+
end
|
9
|
+
|
10
|
+
TOPIC_NAME = "kafka-input-#{SecureRandom.uuid}"
|
11
|
+
|
12
|
+
CONFIG = %[
|
13
|
+
@type kafka
|
14
|
+
brokers localhost:9092
|
15
|
+
consumer_group fluentd
|
16
|
+
format text
|
17
|
+
@label @kafka
|
18
|
+
topics #{TOPIC_NAME}
|
19
|
+
]
|
20
|
+
|
21
|
+
def create_driver(conf = CONFIG)
|
22
|
+
Fluent::Test::Driver::Input.new(Fluent::KafkaGroupInput).configure(conf)
|
23
|
+
end
|
24
|
+
|
25
|
+
|
26
|
+
def test_configure
|
27
|
+
d = create_driver
|
28
|
+
assert_equal [TOPIC_NAME], d.instance.topics
|
29
|
+
assert_equal 'text', d.instance.format
|
30
|
+
assert_equal 'localhost:9092', d.instance.brokers
|
31
|
+
end
|
32
|
+
|
33
|
+
def test_multi_worker_support
|
34
|
+
d = create_driver
|
35
|
+
assert_true d.instance.multi_workers_ready?
|
36
|
+
end
|
37
|
+
|
38
|
+
class ConsumeTest < self
|
39
|
+
def setup
|
40
|
+
@kafka = Kafka.new(["localhost:9092"], client_id: 'kafka')
|
41
|
+
@producer = @kafka.producer
|
42
|
+
end
|
43
|
+
|
44
|
+
def teardown
|
45
|
+
@kafka.delete_topic(TOPIC_NAME)
|
46
|
+
@kafka.close
|
47
|
+
end
|
48
|
+
|
49
|
+
def test_consume
|
50
|
+
conf = %[
|
51
|
+
@type kafka
|
52
|
+
brokers localhost:9092
|
53
|
+
format text
|
54
|
+
@label @kafka
|
55
|
+
topics #{TOPIC_NAME}
|
56
|
+
]
|
57
|
+
d = create_driver
|
58
|
+
|
59
|
+
d.run(expect_records: 1, timeout: 10) do
|
60
|
+
@producer.produce("Hello, fluent-plugin-kafka!", topic: TOPIC_NAME)
|
61
|
+
@producer.deliver_messages
|
62
|
+
end
|
63
|
+
expected = {'message' => 'Hello, fluent-plugin-kafka!'}
|
64
|
+
assert_equal expected, d.events[0][2]
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-kafka
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.16.
|
4
|
+
version: 0.16.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Hidemasa Togashi
|
@@ -122,6 +122,7 @@ files:
|
|
122
122
|
- LICENSE
|
123
123
|
- README.md
|
124
124
|
- Rakefile
|
125
|
+
- ci/prepare-kafka-server.sh
|
125
126
|
- fluent-plugin-kafka.gemspec
|
126
127
|
- lib/fluent/plugin/in_kafka.rb
|
127
128
|
- lib/fluent/plugin/in_kafka_group.rb
|
@@ -134,6 +135,8 @@ files:
|
|
134
135
|
- lib/fluent/plugin/out_rdkafka.rb
|
135
136
|
- lib/fluent/plugin/out_rdkafka2.rb
|
136
137
|
- test/helper.rb
|
138
|
+
- test/plugin/test_in_kafka.rb
|
139
|
+
- test/plugin/test_in_kafka_group.rb
|
137
140
|
- test/plugin/test_kafka_plugin_util.rb
|
138
141
|
- test/plugin/test_out_kafka.rb
|
139
142
|
homepage: https://github.com/fluent/fluent-plugin-kafka
|
@@ -161,5 +164,7 @@ specification_version: 4
|
|
161
164
|
summary: Fluentd plugin for Apache Kafka > 0.8
|
162
165
|
test_files:
|
163
166
|
- test/helper.rb
|
167
|
+
- test/plugin/test_in_kafka.rb
|
168
|
+
- test/plugin/test_in_kafka_group.rb
|
164
169
|
- test/plugin/test_kafka_plugin_util.rb
|
165
170
|
- test/plugin/test_out_kafka.rb
|