fluent-plugin-kafka 0.3.1 → 0.3.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f58dd81646c8ff29f5f9e55bb200fda728ef83f1
-  data.tar.gz: 2b47f106dfebc12704d5b751290bfff7a42a44ad
+  metadata.gz: 512f9e6a1ab34f3206a79904ff3134da1ee6de54
+  data.tar.gz: 27bcf99b9a4eac5ed2cf15694a249db4f3db6ffb
 SHA512:
-  metadata.gz: 6ecc97a6901090f2f9cd54df11e0af55bc9df901ae6c5186867bd14f26796e762b8d1b3e3140d050827408094adb3d0f99a849bd4171d466e4bbe05b806b2cb8
-  data.tar.gz: 41800c1ddd32cd17cccb0af7cdaff2da11e3f7354f050c4b554ce35aeafb1ddff486f41dcee41806dc9ee002cb0c068025183d9e96996e6c45dba4337c10740b
+  metadata.gz: 0d926b09af1ed8611d85a40eb353aeda21b39f862c85a4131ada9e9fcfa9d057157fb107a56808af905c89c669c6a452b03eef508e665ae3957f7599c25c19af
+  data.tar.gz: 60d029566a17521e19dd93e0e6d2cefd35990a8b14d89202c42ba6cf6512608f78dced06e23d0e1386e95d206173ad78a77c65dba081f1c3e5adc150f74c07d4

data/README.md

@@ -138,6 +138,11 @@ Install snappy module before you use snappy compression.
 
     $ gem install snappy
 
+snappy gem uses native extension, so you need to install several packages before.
+On Ubuntu, need development packages and snappy library.
+
+    $ sudo apt-get install build-essential autoconf automake libtool libsnappy-dev
+
 #### Load balancing
 
 Messages will be assigned a partition at random as default by ruby-kafka, but messages with the same partition key will always be assigned to the same partition by setting `default_partition_key` in config file.

data/fluent-plugin-kafka.gemspec

@@ -12,7 +12,7 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "fluent-plugin-kafka"
   gem.require_paths = ["lib"]
-  gem.version       = '0.3.1'
+  gem.version       = '0.3.2'
   gem.required_ruby_version = ">= 2.1.0"
 
   gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]

data/lib/fluent/plugin/in_kafka_group.rb

@@ -68,6 +68,15 @@ class Fluent::KafkaGroupInput < Fluent::Input
     end
 
     @parser_proc = setup_parser
+
+    @consumer_opts = {:group_id => @consumer_group}
+    @consumer_opts[:session_timeout] = @session_timeout if @session_timeout
+    @consumer_opts[:offset_commit_interval] = @offset_commit_interval if @offset_commit_interval
+    @consumer_opts[:offset_commit_threshold] = @offset_commit_threshold if @offset_commit_threshold
+
+    @fetch_opts = {}
+    @fetch_opts[:max_wait_time] = @max_wait_time if @max_wait_time
+    @fetch_opts[:min_bytes] = @min_bytes if @min_bytes
   end
 
   def setup_parser
@@ -89,55 +98,64 @@ class Fluent::KafkaGroupInput < Fluent::Input
   def start
     super
 
-    consumer_opts = {:group_id => @consumer_group}
-    consumer_opts[:session_timeout] = @session_timeout if @session_timeout
-    consumer_opts[:offset_commit_interval] = @offset_commit_interval if @offset_commit_interval
-    consumer_opts[:offset_commit_threshold] = @offset_commit_threshold if @offset_commit_threshold
-
-    @fetch_opts = {}
-    @fetch_opts[:max_wait_time] = @max_wait_time if @max_wait_time
-    @fetch_opts[:min_bytes] = @min_bytes if @min_bytes
-
     @kafka = Kafka.new(seed_brokers: @brokers,
                        ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
                        ssl_client_cert: read_ssl_file(@ssl_client_cert),
                        ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key))
-    @consumer = @kafka.consumer(consumer_opts)
-    @topics.each { |topic|
-      @consumer.subscribe(topic, start_from_beginning: @start_from_beginning)
-    }
+    @consumer = setup_consumer
     @thread = Thread.new(&method(:run))
   end
 
   def shutdown
-    @consumer.stop
+    # This nil assignment should be guarded by mutex in multithread programming manner.
+    # But the situation is very low contention, so we don't use mutex for now.
+    # If the problem happens, we will add a guard for consumer.
+    consumer = @consumer
+    @consumer = nil
+    consumer.stop
+
     @thread.join
     @kafka.close
     super
   end
 
+  def setup_consumer
+    consumer = @kafka.consumer(@consumer_opts)
+    @topics.each { |topic|
+      consumer.subscribe(topic, start_from_beginning: @start_from_beginning)
+    }
+    consumer
+  end
+
   def run
-    @consumer.each_batch(@fetch_opts) { |batch|
-      es = Fluent::MultiEventStream.new
-      tag = batch.topic
-      tag = @add_prefix + "." + tag if @add_prefix
-      tag = tag + "." + @add_suffix if @add_suffix
-
-      batch.messages.each { |msg|
-        begin
-          es.add(Fluent::Engine.now, @parser_proc.call(msg))
-        rescue => e
-          $log.warn "parser error in #{batch.topic}/#{batch.partition}", :error => e.to_s, :value => msg.value, :offset => msg.offset
-          $log.debug_backtrace
-        end
-      }
-
-      unless es.empty?
-        router.emit_stream(tag, es)
+    while @consumer
+      begin
+        @consumer.each_batch(@fetch_opts) { |batch|
+          es = Fluent::MultiEventStream.new
+          tag = batch.topic
+          tag = @add_prefix + "." + tag if @add_prefix
+          tag = tag + "." + @add_suffix if @add_suffix
+
+          batch.messages.each { |msg|
+            begin
+              es.add(Fluent::Engine.now, @parser_proc.call(msg))
+            rescue => e
+              log.warn "parser error in #{batch.topic}/#{batch.partition}", :error => e.to_s, :value => msg.value, :offset => msg.offset
+              log.debug_backtrace
+            end
+          }
+
+          unless es.empty?
+            router.emit_stream(tag, es)
+          end
+        }
+      rescue => e
+        log.error "unexpected error during consuming events from kafka. Re-fetch events.", :error => e.to_s
+        log.error_backtrace
       end
-    }
+    end
   rescue => e
-    $log.error "unexpected error", :error => e.to_s
-    $log.error_backtrace
+    log.error "unexpected error during consumer object access", :error => e.to_s
+    log.error_backtrace
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kafka
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.3.2
 platform: ruby
 authors:
 - Hidemasa Togashi
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-08-27 00:00:00.000000000 Z
+date: 2016-10-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd