fluent-plugin-kafka 0.6.2 → 0.6.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a038696697cc60a6c53b5204dd2146cc5961bd8b
-  data.tar.gz: cafd403c4c422ba5d8bfc0214cff9566838c498b
+  metadata.gz: 2b9941b6f4988f1fb14b5a0a591a81574411f915
+  data.tar.gz: '08c543d8db1c038e908584460e39a6a9df89e86c'
 SHA512:
-  metadata.gz: c469841e4e3f21c907bdbcd44c27eb3e2f6de18086cf622eb4c67b603395aced8b37aab6cb4afd108a0199e620ec26f3cdbaf7a9ea298fb43e8b86e69a06fb0e
-  data.tar.gz: ab2db78c09756aa938971495db83288b93647da1cb28de15050aee7dbbced2a87313c5ad261a3a3464fc7f6731af3a1396abc275828bed538ca82f50397175ba
+  metadata.gz: 253f7354704fecc1fa1d21bc84056727169fc9dcbcbde55afe595ae9c882213dae910c82bd2b7e34918169f9275f6a8ab031646405409a115912a92f93092b94
+  data.tar.gz: 87cdd5a750bc91c036901d64f76fda7ae874a7fcc16f9f573311bfc5a061492e6d5a3612c6c3378bf7ab34f7a99b8f6b029d75405a2ef240cf3bb40c709498f0

data/ChangeLog

@@ -1,3 +1,7 @@
+Release 0.6.3 - 2017/11/14
+
+	* in_kafka_group: re-create consumer when error happens during event fetch
+
 Release 0.6.2 - 2017/11/1
 
 	* Fix ltsv parsing issue which generates symbol keys

data/fluent-plugin-kafka.gemspec

@@ -12,7 +12,7 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "fluent-plugin-kafka"
   gem.require_paths = ["lib"]
-  gem.version       = '0.6.2'
+  gem.version       = '0.6.3'
   gem.required_ruby_version = ">= 2.1.0"
 
   gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]

data/lib/fluent/plugin/in_kafka_group.rb

@@ -26,6 +26,7 @@ class Fluent::KafkaGroupInput < Fluent::Input
   config_param :time_format, :string, :default => nil,
                :desc => "Time format to be used to parse 'time' filed."
 
+  config_param :retry_wait_seconds, :integer, :default => 30
   # Kafka consumer options
   config_param :max_bytes, :integer, :default => 1048576,
                :desc => "Maximum number of bytes to fetch."
@@ -122,7 +123,7 @@ class Fluent::KafkaGroupInput < Fluent::Input
 
   def start
     super
-
+   
     @kafka = Kafka.new(seed_brokers: @brokers,
                        ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
                        ssl_client_cert: read_ssl_file(@ssl_client_cert),
@@ -152,7 +153,21 @@ class Fluent::KafkaGroupInput < Fluent::Input
     }
     consumer
   end
-
+  
+  def reconnect_consumer
+    log.warn "Stopping Consumer"
+    consumer = @consumer
+    @consumer = nil
+    consumer.stop
+    log.warn "Could not connect to broker. Next retry will be in #{@retry_wait_seconds} seconds"
+    sleep @retry_wait_seconds
+    @consumer = setup_consumer
+    log.warn "Re-starting consumer #{Time.now.to_s}"
+  rescue =>e
+    log.error "unexpected error during re-starting consumer object access", :error => e.to_s
+    log.error_backtrace
+  end
+  
   def run
     while @consumer
       begin
@@ -189,6 +204,7 @@ class Fluent::KafkaGroupInput < Fluent::Input
       rescue => e
         log.error "unexpected error during consuming events from kafka. Re-fetch events.", :error => e.to_s
         log.error_backtrace
+        reconnect_consumer
       end
     end
   rescue => e

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kafka
 version: !ruby/object:Gem::Version
-  version: 0.6.2
+  version: 0.6.3
 platform: ruby
 authors:
 - Hidemasa Togashi
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-11-02 00:00:00.000000000 Z
+date: 2017-11-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd