fluent-plugin-kafka 0.16.2 → 0.16.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1f388f79dd4b9bf95418b71cd78a9e2f712fc01aa791dab8e0f5e22cd899c2b6
-  data.tar.gz: 00cee257a09cbfe3773d8e077b651668e9115d94f813b2fe575917789248a966
+  metadata.gz: 1cc628766df718a6fff17debcf2cffdcc041419cb96c1c9b6ce1fee532807b58
+  data.tar.gz: 543ccbd91345f45ee75c3e5e84ef5b3414420a0e2557c76a40afc8e308d70735
 SHA512:
-  metadata.gz: 5e23cb803cef76ac8446007b1a60e091868c99c62b839d040093232ae90456306e1eb6bef9630f4c8230e9662ac3a0039eca1728640bccd906f4ed099f5fed4a
-  data.tar.gz: bc2c9f55199a5227d4ce6d5ea56c91fc31ca03afc0d45b23966d81f3d3a9f1eeea356b2af161d851b5e800fe4492ca6d7df7c9d07c81942e1602768d64a21156
+  metadata.gz: 7cb27a4fe28ccc0f31e36449046faf8d4d3389be7820f7cfce8f612aad84b3239234663a71b50523cb08a13e145f78fdc86e93cf9cec28d35d82765b96c9dddc
+  data.tar.gz: 98f52a5d84fb348178f9df6ede6592412ddccd43891c95e9d6937622bff8b02bb4cbb3231c7ca7d709a3ae6cdbfb90d3f00074eefe4cc0703379315098424809

data/.github/workflows/linux.yml

@@ -17,10 +17,20 @@ jobs:
     - uses: ruby/setup-ruby@v1
       with:
         ruby-version: ${{ matrix.ruby }}
+    - name: Install confluent-kafka
+      run: |
+        sudo apt install -V -y gnupg2 wget
+        wget https://packages.confluent.io/deb/6.0/archive.key
+        sudo gpg2 --homedir /tmp --no-default-keyring --keyring gnupg-ring:/usr/share/keyrings/confluent-archive-keyring.gpg --import archive.key
+        sudo chmod 644 /usr/share/keyrings/confluent-archive-keyring.gpg
+        sudo sh -c 'echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/confluent-archive-keyring.gpg] https://packages.confluent.io/deb/6.0 stable main" > /etc/apt/sources.list.d/confluent.list'
+        sudo apt update
+        sudo apt install -y confluent-community-2.13 openjdk-11-jre netcat-openbsd
     - name: unit testing
       env:
         CI: true
       run: |
+        sudo ./ci/prepare-kafka-server.sh
         gem install bundler rake
         bundle install --jobs 4 --retry 3
         bundle exec rake test

data/ChangeLog

@@ -1,3 +1,6 @@
+Release 0.16.3 - 2021/05/17
+	* in_kafka_group: Fix one more Ruby 3.0 keyword arguments issue
+
 Release 0.16.2 - 2021/05/17
 	* in_kafka, in_kafka_group: Support Ruby 3.0 keyword arguments interop
 

data/ci/prepare-kafka-server.sh

@@ -0,0 +1,33 @@
+#!/bin/sh
+
+export KAFKA_OPTS=-Dzookeeper.4lw.commands.whitelist=ruok
+/usr/bin/zookeeper-server-start /etc/kafka/zookeeper.properties  &
+N_POLLING=30
+n=1
+while true ; do
+    sleep 1
+    status=$(echo ruok | nc localhost 2181)
+    if [ "$status" = "imok" ]; then
+	break
+    fi
+    n=$((n + 1))
+    if [ $n -ge $N_POLLING ]; then
+	echo "failed to get response from zookeeper-server"
+	exit 1
+    fi
+done
+/usr/bin/kafka-server-start /etc/kafka/server.properties &
+n=1
+while true ; do
+    sleep 1
+    status=$(/usr/bin/zookeeper-shell localhost:2181 ls /brokers/ids | sed -n 6p)
+    if [ "$status" = "[0]" ]; then
+	break
+    fi
+    n=$((n + 1))
+    if [ $n -ge $N_POLLING ]; then
+	echo "failed to get response from kafka-server"
+	exit 1
+    fi
+done
+/usr/bin/kafka-topics --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic test

data/fluent-plugin-kafka.gemspec

@@ -13,7 +13,7 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "fluent-plugin-kafka"
   gem.require_paths = ["lib"]
-  gem.version       = '0.16.2'
+  gem.version       = '0.16.3'
   gem.required_ruby_version = ">= 2.1.0"
 
   gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]

data/lib/fluent/plugin/in_kafka_group.rb

@@ -345,7 +345,7 @@ class Fluent::KafkaGroupInput < Fluent::Input
   def run
     while @consumer
       begin
-        @consumer.each_batch(@fetch_opts) { |batch|
+        @consumer.each_batch(**@fetch_opts) { |batch|
           if @tag_source == :record
             process_batch_with_record_tag(batch)
           else

data/test/helper.rb

@@ -22,6 +22,8 @@ unless ENV.has_key?('VERBOSE')
 end
 
 require 'fluent/plugin/out_kafka'
+require 'fluent/plugin/in_kafka'
+require 'fluent/plugin/in_kafka_group'
 
 class Test::Unit::TestCase
 end

data/test/plugin/test_in_kafka.rb

@@ -0,0 +1,66 @@
+require 'helper'
+require 'fluent/test/driver/input'
+require 'securerandom'
+
+class KafkaInputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  TOPIC_NAME = "kafka-input-#{SecureRandom.uuid}"
+
+  CONFIG = %[
+    @type kafka
+    brokers localhost:9092
+    format text
+    @label @kafka
+    topics #{TOPIC_NAME}
+  ]
+
+  def create_driver(conf = CONFIG)
+    Fluent::Test::Driver::Input.new(Fluent::KafkaInput).configure(conf)
+  end
+
+
+  def test_configure
+    d = create_driver
+    assert_equal TOPIC_NAME, d.instance.topics
+    assert_equal 'text', d.instance.format
+    assert_equal 'localhost:9092', d.instance.brokers
+  end
+
+  def test_multi_worker_support
+    d = create_driver
+    assert_false d.instance.multi_workers_ready?
+  end
+
+  class ConsumeTest < self
+    def setup
+      @kafka = Kafka.new(["localhost:9092"], client_id: 'kafka')
+      @producer = @kafka.producer
+    end
+
+    def teardown
+      @kafka.delete_topic(TOPIC_NAME)
+      @kafka.close
+    end
+
+    def test_consume
+      conf = %[
+        @type kafka
+        brokers localhost:9092
+        format text
+        @label @kafka
+        topics #{TOPIC_NAME}
+      ]
+      d = create_driver
+
+      d.run(expect_records: 1, timeout: 10) do
+        @producer.produce("Hello, fluent-plugin-kafka!", topic: TOPIC_NAME)
+        @producer.deliver_messages
+      end
+      expected = {'message'  => 'Hello, fluent-plugin-kafka!'}
+      assert_equal expected, d.events[0][2]
+    end
+  end
+end

data/test/plugin/test_in_kafka_group.rb

@@ -0,0 +1,67 @@
+require 'helper'
+require 'fluent/test/driver/input'
+require 'securerandom'
+
+class KafkaGroupInputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  TOPIC_NAME = "kafka-input-#{SecureRandom.uuid}"
+
+  CONFIG = %[
+    @type kafka
+    brokers localhost:9092
+    consumer_group fluentd
+    format text
+    @label @kafka
+    topics #{TOPIC_NAME}
+  ]
+
+  def create_driver(conf = CONFIG)
+    Fluent::Test::Driver::Input.new(Fluent::KafkaGroupInput).configure(conf)
+  end
+
+
+  def test_configure
+    d = create_driver
+    assert_equal [TOPIC_NAME], d.instance.topics
+    assert_equal 'text', d.instance.format
+    assert_equal 'localhost:9092', d.instance.brokers
+  end
+
+  def test_multi_worker_support
+    d = create_driver
+    assert_true d.instance.multi_workers_ready?
+  end
+
+  class ConsumeTest < self
+    def setup
+      @kafka = Kafka.new(["localhost:9092"], client_id: 'kafka')
+      @producer = @kafka.producer
+    end
+
+    def teardown
+      @kafka.delete_topic(TOPIC_NAME)
+      @kafka.close
+    end
+
+    def test_consume
+      conf = %[
+        @type kafka
+        brokers localhost:9092
+        format text
+        @label @kafka
+        topics #{TOPIC_NAME}
+      ]
+      d = create_driver
+
+      d.run(expect_records: 1, timeout: 10) do
+        @producer.produce("Hello, fluent-plugin-kafka!", topic: TOPIC_NAME)
+        @producer.deliver_messages
+      end
+      expected = {'message'  => 'Hello, fluent-plugin-kafka!'}
+      assert_equal expected, d.events[0][2]
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kafka
 version: !ruby/object:Gem::Version
-  version: 0.16.2
+  version: 0.16.3
 platform: ruby
 authors:
 - Hidemasa Togashi
@@ -122,6 +122,7 @@ files:
 - LICENSE
 - README.md
 - Rakefile
+- ci/prepare-kafka-server.sh
 - fluent-plugin-kafka.gemspec
 - lib/fluent/plugin/in_kafka.rb
 - lib/fluent/plugin/in_kafka_group.rb
@@ -134,6 +135,8 @@ files:
 - lib/fluent/plugin/out_rdkafka.rb
 - lib/fluent/plugin/out_rdkafka2.rb
 - test/helper.rb
+- test/plugin/test_in_kafka.rb
+- test/plugin/test_in_kafka_group.rb
 - test/plugin/test_kafka_plugin_util.rb
 - test/plugin/test_out_kafka.rb
 homepage: https://github.com/fluent/fluent-plugin-kafka
@@ -161,5 +164,7 @@ specification_version: 4
 summary: Fluentd plugin for Apache Kafka > 0.8
 test_files:
 - test/helper.rb
+- test/plugin/test_in_kafka.rb
+- test/plugin/test_in_kafka_group.rb
 - test/plugin/test_kafka_plugin_util.rb
 - test/plugin/test_out_kafka.rb