fluent-plugin-kafka 0.16.2 → 0.16.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 1f388f79dd4b9bf95418b71cd78a9e2f712fc01aa791dab8e0f5e22cd899c2b6
4
- data.tar.gz: 00cee257a09cbfe3773d8e077b651668e9115d94f813b2fe575917789248a966
3
+ metadata.gz: 1cc628766df718a6fff17debcf2cffdcc041419cb96c1c9b6ce1fee532807b58
4
+ data.tar.gz: 543ccbd91345f45ee75c3e5e84ef5b3414420a0e2557c76a40afc8e308d70735
5
5
  SHA512:
6
- metadata.gz: 5e23cb803cef76ac8446007b1a60e091868c99c62b839d040093232ae90456306e1eb6bef9630f4c8230e9662ac3a0039eca1728640bccd906f4ed099f5fed4a
7
- data.tar.gz: bc2c9f55199a5227d4ce6d5ea56c91fc31ca03afc0d45b23966d81f3d3a9f1eeea356b2af161d851b5e800fe4492ca6d7df7c9d07c81942e1602768d64a21156
6
+ metadata.gz: 7cb27a4fe28ccc0f31e36449046faf8d4d3389be7820f7cfce8f612aad84b3239234663a71b50523cb08a13e145f78fdc86e93cf9cec28d35d82765b96c9dddc
7
+ data.tar.gz: 98f52a5d84fb348178f9df6ede6592412ddccd43891c95e9d6937622bff8b02bb4cbb3231c7ca7d709a3ae6cdbfb90d3f00074eefe4cc0703379315098424809
@@ -17,10 +17,20 @@ jobs:
17
17
  - uses: ruby/setup-ruby@v1
18
18
  with:
19
19
  ruby-version: ${{ matrix.ruby }}
20
+ - name: Install confluent-kafka
21
+ run: |
22
+ sudo apt install -V -y gnupg2 wget
23
+ wget https://packages.confluent.io/deb/6.0/archive.key
24
+ sudo gpg2 --homedir /tmp --no-default-keyring --keyring gnupg-ring:/usr/share/keyrings/confluent-archive-keyring.gpg --import archive.key
25
+ sudo chmod 644 /usr/share/keyrings/confluent-archive-keyring.gpg
26
+ sudo sh -c 'echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/confluent-archive-keyring.gpg] https://packages.confluent.io/deb/6.0 stable main" > /etc/apt/sources.list.d/confluent.list'
27
+ sudo apt update
28
+ sudo apt install -y confluent-community-2.13 openjdk-11-jre netcat-openbsd
20
29
  - name: unit testing
21
30
  env:
22
31
  CI: true
23
32
  run: |
33
+ sudo ./ci/prepare-kafka-server.sh
24
34
  gem install bundler rake
25
35
  bundle install --jobs 4 --retry 3
26
36
  bundle exec rake test
data/ChangeLog CHANGED
@@ -1,3 +1,6 @@
1
+ Release 0.16.3 - 2021/05/17
2
+ * in_kafka_group: Fix one more Ruby 3.0 keyword arguments issue
3
+
1
4
  Release 0.16.2 - 2021/05/17
2
5
  * in_kafka, in_kafka_group: Support Ruby 3.0 keyword arguments interop
3
6
 
@@ -0,0 +1,33 @@
1
+ #!/bin/sh
2
+
3
+ export KAFKA_OPTS=-Dzookeeper.4lw.commands.whitelist=ruok
4
+ /usr/bin/zookeeper-server-start /etc/kafka/zookeeper.properties &
5
+ N_POLLING=30
6
+ n=1
7
+ while true ; do
8
+ sleep 1
9
+ status=$(echo ruok | nc localhost 2181)
10
+ if [ "$status" = "imok" ]; then
11
+ break
12
+ fi
13
+ n=$((n + 1))
14
+ if [ $n -ge $N_POLLING ]; then
15
+ echo "failed to get response from zookeeper-server"
16
+ exit 1
17
+ fi
18
+ done
19
+ /usr/bin/kafka-server-start /etc/kafka/server.properties &
20
+ n=1
21
+ while true ; do
22
+ sleep 1
23
+ status=$(/usr/bin/zookeeper-shell localhost:2181 ls /brokers/ids | sed -n 6p)
24
+ if [ "$status" = "[0]" ]; then
25
+ break
26
+ fi
27
+ n=$((n + 1))
28
+ if [ $n -ge $N_POLLING ]; then
29
+ echo "failed to get response from kafka-server"
30
+ exit 1
31
+ fi
32
+ done
33
+ /usr/bin/kafka-topics --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic test
@@ -13,7 +13,7 @@ Gem::Specification.new do |gem|
13
13
  gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
14
14
  gem.name = "fluent-plugin-kafka"
15
15
  gem.require_paths = ["lib"]
16
- gem.version = '0.16.2'
16
+ gem.version = '0.16.3'
17
17
  gem.required_ruby_version = ">= 2.1.0"
18
18
 
19
19
  gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]
@@ -345,7 +345,7 @@ class Fluent::KafkaGroupInput < Fluent::Input
345
345
  def run
346
346
  while @consumer
347
347
  begin
348
- @consumer.each_batch(@fetch_opts) { |batch|
348
+ @consumer.each_batch(**@fetch_opts) { |batch|
349
349
  if @tag_source == :record
350
350
  process_batch_with_record_tag(batch)
351
351
  else
data/test/helper.rb CHANGED
@@ -22,6 +22,8 @@ unless ENV.has_key?('VERBOSE')
22
22
  end
23
23
 
24
24
  require 'fluent/plugin/out_kafka'
25
+ require 'fluent/plugin/in_kafka'
26
+ require 'fluent/plugin/in_kafka_group'
25
27
 
26
28
  class Test::Unit::TestCase
27
29
  end
@@ -0,0 +1,66 @@
1
+ require 'helper'
2
+ require 'fluent/test/driver/input'
3
+ require 'securerandom'
4
+
5
+ class KafkaInputTest < Test::Unit::TestCase
6
+ def setup
7
+ Fluent::Test.setup
8
+ end
9
+
10
+ TOPIC_NAME = "kafka-input-#{SecureRandom.uuid}"
11
+
12
+ CONFIG = %[
13
+ @type kafka
14
+ brokers localhost:9092
15
+ format text
16
+ @label @kafka
17
+ topics #{TOPIC_NAME}
18
+ ]
19
+
20
+ def create_driver(conf = CONFIG)
21
+ Fluent::Test::Driver::Input.new(Fluent::KafkaInput).configure(conf)
22
+ end
23
+
24
+
25
+ def test_configure
26
+ d = create_driver
27
+ assert_equal TOPIC_NAME, d.instance.topics
28
+ assert_equal 'text', d.instance.format
29
+ assert_equal 'localhost:9092', d.instance.brokers
30
+ end
31
+
32
+ def test_multi_worker_support
33
+ d = create_driver
34
+ assert_false d.instance.multi_workers_ready?
35
+ end
36
+
37
+ class ConsumeTest < self
38
+ def setup
39
+ @kafka = Kafka.new(["localhost:9092"], client_id: 'kafka')
40
+ @producer = @kafka.producer
41
+ end
42
+
43
+ def teardown
44
+ @kafka.delete_topic(TOPIC_NAME)
45
+ @kafka.close
46
+ end
47
+
48
+ def test_consume
49
+ conf = %[
50
+ @type kafka
51
+ brokers localhost:9092
52
+ format text
53
+ @label @kafka
54
+ topics #{TOPIC_NAME}
55
+ ]
56
+ d = create_driver
57
+
58
+ d.run(expect_records: 1, timeout: 10) do
59
+ @producer.produce("Hello, fluent-plugin-kafka!", topic: TOPIC_NAME)
60
+ @producer.deliver_messages
61
+ end
62
+ expected = {'message' => 'Hello, fluent-plugin-kafka!'}
63
+ assert_equal expected, d.events[0][2]
64
+ end
65
+ end
66
+ end
@@ -0,0 +1,67 @@
1
+ require 'helper'
2
+ require 'fluent/test/driver/input'
3
+ require 'securerandom'
4
+
5
+ class KafkaGroupInputTest < Test::Unit::TestCase
6
+ def setup
7
+ Fluent::Test.setup
8
+ end
9
+
10
+ TOPIC_NAME = "kafka-input-#{SecureRandom.uuid}"
11
+
12
+ CONFIG = %[
13
+ @type kafka
14
+ brokers localhost:9092
15
+ consumer_group fluentd
16
+ format text
17
+ @label @kafka
18
+ topics #{TOPIC_NAME}
19
+ ]
20
+
21
+ def create_driver(conf = CONFIG)
22
+ Fluent::Test::Driver::Input.new(Fluent::KafkaGroupInput).configure(conf)
23
+ end
24
+
25
+
26
+ def test_configure
27
+ d = create_driver
28
+ assert_equal [TOPIC_NAME], d.instance.topics
29
+ assert_equal 'text', d.instance.format
30
+ assert_equal 'localhost:9092', d.instance.brokers
31
+ end
32
+
33
+ def test_multi_worker_support
34
+ d = create_driver
35
+ assert_true d.instance.multi_workers_ready?
36
+ end
37
+
38
+ class ConsumeTest < self
39
+ def setup
40
+ @kafka = Kafka.new(["localhost:9092"], client_id: 'kafka')
41
+ @producer = @kafka.producer
42
+ end
43
+
44
+ def teardown
45
+ @kafka.delete_topic(TOPIC_NAME)
46
+ @kafka.close
47
+ end
48
+
49
+ def test_consume
50
+ conf = %[
51
+ @type kafka
52
+ brokers localhost:9092
53
+ format text
54
+ @label @kafka
55
+ topics #{TOPIC_NAME}
56
+ ]
57
+ d = create_driver
58
+
59
+ d.run(expect_records: 1, timeout: 10) do
60
+ @producer.produce("Hello, fluent-plugin-kafka!", topic: TOPIC_NAME)
61
+ @producer.deliver_messages
62
+ end
63
+ expected = {'message' => 'Hello, fluent-plugin-kafka!'}
64
+ assert_equal expected, d.events[0][2]
65
+ end
66
+ end
67
+ end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-kafka
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.16.2
4
+ version: 0.16.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Hidemasa Togashi
@@ -122,6 +122,7 @@ files:
122
122
  - LICENSE
123
123
  - README.md
124
124
  - Rakefile
125
+ - ci/prepare-kafka-server.sh
125
126
  - fluent-plugin-kafka.gemspec
126
127
  - lib/fluent/plugin/in_kafka.rb
127
128
  - lib/fluent/plugin/in_kafka_group.rb
@@ -134,6 +135,8 @@ files:
134
135
  - lib/fluent/plugin/out_rdkafka.rb
135
136
  - lib/fluent/plugin/out_rdkafka2.rb
136
137
  - test/helper.rb
138
+ - test/plugin/test_in_kafka.rb
139
+ - test/plugin/test_in_kafka_group.rb
137
140
  - test/plugin/test_kafka_plugin_util.rb
138
141
  - test/plugin/test_out_kafka.rb
139
142
  homepage: https://github.com/fluent/fluent-plugin-kafka
@@ -161,5 +164,7 @@ specification_version: 4
161
164
  summary: Fluentd plugin for Apache Kafka > 0.8
162
165
  test_files:
163
166
  - test/helper.rb
167
+ - test/plugin/test_in_kafka.rb
168
+ - test/plugin/test_in_kafka_group.rb
164
169
  - test/plugin/test_kafka_plugin_util.rb
165
170
  - test/plugin/test_out_kafka.rb