fluent-plugin-kafka 0.0.9 → 0.0.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9cad342ed5984ebc424872a660379b6e471a135d
-  data.tar.gz: 73ea1a4a10f3584f719c45582d54f52b67b52c72
+  metadata.gz: 887e010ab7b1ac813f4fa0c412d8348ffc192831
+  data.tar.gz: 52f631a447899497ac9765294f697783755ae5ab
 SHA512:
-  metadata.gz: 82ff85d1bc2353109a05092f8cda28237bb4b24cec7af7cce539c55c8d4125e3a9a96e540e43ad33bf2dfe63c1120585b4c3ba7430d70812b973894a92cfc79b
-  data.tar.gz: ff574d372eee407d78e199809551bd1ac397c611b018b665416b0c8812cfe3f442b48e75aa320e66933b72a166172d9c09cc0fca913e5141c202d18a0cdd794a
+  metadata.gz: dcd0a646d4cbb926fa3503a00e75f0ac6188e4a7a5bd8a3c3332481c2989dd913a3d4f072d839c35236e0a9ae2728076db2426d2664ec0aee8778a86246673f6
+  data.tar.gz: bf1f03fb1134a9e16153156332da8013450c8f5a5cbc45a343ff9a6bcb4c682a8c5858339267a588a0246edce614f51f9b8ff5babdde5f26fb3a0c3b36d75bb8

data/fluent-plugin-kafka.gemspec

@@ -12,7 +12,7 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "fluent-plugin-kafka"
   gem.require_paths = ["lib"]
-  gem.version = '0.0.9'
+  gem.version = '0.0.10'
   gem.add_dependency 'fluentd'
   gem.add_dependency 'poseidon'
   gem.add_dependency 'ltsv'

data/lib/fluent/plugin/out_kafka.rb

@@ -23,23 +23,39 @@ class Fluent::KafkaOutput < Fluent::Output
   attr_accessor :output_data_type
   attr_accessor :field_separator
 
-  def configure(conf)
-    super
+  @seed_brokers = []
+
+  def refresh_producer()
     if @zookeeper
-      require 'zookeeper'
-      require 'yajl'
       @seed_brokers = []
       z = Zookeeper.new(@zookeeper)
       z.get_children(:path => '/brokers/ids')[:children].each do |id|
         broker = Yajl.load(z.get(:path => "/brokers/ids/#{id}")[:data])
         @seed_brokers.push("#{broker['host']}:#{broker['port']}")
       end
-      log.info "brokers has been set via Zookeeper: #{@seed_brokers}"
+      log.info "brokers has been refreshed via Zookeeper: #{@seed_brokers}"
+    end
+    begin
+      if @seed_brokers.length > 0
+        @producer = Poseidon::Producer.new(@seed_brokers, @client_id, :max_send_retries => @max_send_retries, :required_acks => @required_acks, :ack_timeout_ms => @ack_timeout_ms)
+        log.info "initialized producer #{@client_id}"
+      else
+        log.warn "No brokers found on Zookeeper"
+      end
+    rescue Exception => e
+      log.error e
+    end
+  end
+
+  def configure(conf)
+    super
+    if @zookeeper
+      require 'zookeeper'
+      require 'yajl'
     else
       @seed_brokers = @brokers.match(",").nil? ? [@brokers] : @brokers.split(",")
       log.info "brokers has been set directly: #{@seed_brokers}"
     end
-    @producers = {} # keyed by topic:partition
     case @output_data_type
     when 'json'
       require 'yajl'
@@ -72,6 +88,7 @@ class Fluent::KafkaOutput < Fluent::Output
 
   def start
     super
+    refresh_producer()
   end
 
   def shutdown
@@ -100,15 +117,20 @@ class Fluent::KafkaOutput < Fluent::Output
   end
 
   def emit(tag, es, chain)
-    chain.next
-    es.each do |time,record|
-      record['time'] = time if @output_include_time
-      record['tag'] = tag if @output_include_tag
-      topic = record['topic'] || self.default_topic || tag
-      partition = record['partition'] || self.default_partition
-      message = Poseidon::MessageToSend.new(topic, parse_record(record))
-      @producers[topic] ||= Poseidon::Producer.new(@seed_brokers, self.client_id, :max_send_retries => @max_send_retries, :required_acks => @required_acks, :ack_timeout_ms => @ack_timeout_ms)
-      @producers[topic].send_messages([message])
+    begin
+      chain.next
+      es.each do |time,record|
+        record['time'] = time if @output_include_time
+        record['tag'] = tag if @output_include_tag
+        topic = record['topic'] || self.default_topic || tag
+        partition = record['partition'] || self.default_partition
+        message = Poseidon::MessageToSend.new(topic, parse_record(record))
+        @producer.send_messages([message])
+      end
+    rescue Exception => e
+      log.warn("Send exception occurred: #{e}")
+      refresh_producer()
+      raise e
     end
   end
 

data/lib/fluent/plugin/out_kafka_buffered.rb

@@ -29,18 +29,35 @@ class Fluent::KafkaOutputBuffered < Fluent::BufferedOutput
     define_method("log") { $log }
   end
 
-  def configure(conf)
-    super
+  @seed_brokers = []
+
+  def refresh_producer()
     if @zookeeper
-      require 'zookeeper'
-      require 'yajl'
       @seed_brokers = []
       z = Zookeeper.new(@zookeeper)
       z.get_children(:path => '/brokers/ids')[:children].each do |id|
         broker = Yajl.load(z.get(:path => "/brokers/ids/#{id}")[:data])
         @seed_brokers.push("#{broker['host']}:#{broker['port']}")
       end
-      log.info "brokers has been set via Zookeeper: #{@seed_brokers}"
+      log.info "brokers has been refreshed via Zookeeper: #{@seed_brokers}"
+    end
+    begin
+      if @seed_brokers.length > 0
+        @producer = Poseidon::Producer.new(@seed_brokers, @client_id, :max_send_retries => @max_send_retries, :required_acks => @required_acks, :ack_timeout_ms => @ack_timeout_ms)
+        log.info "initialized producer #{@client_id}"
+      else
+        log.warn "No brokers found on Zookeeper"
+      end
+    rescue Exception => e
+      log.error e
+    end
+  end
+
+  def configure(conf)
+    super
+    if @zookeeper
+      require 'zookeeper'
+      require 'yajl'
     else
       @seed_brokers = @brokers.match(",").nil? ? [@brokers] : @brokers.split(",")
       log.info "brokers has been set directly: #{@seed_brokers}"
@@ -77,8 +94,7 @@ class Fluent::KafkaOutputBuffered < Fluent::BufferedOutput
 
   def start
     super
-    @producer = Poseidon::Producer.new(@seed_brokers, @client_id, :max_send_retries => @max_send_retries, :required_acks => @required_acks, :ack_timeout_ms => @ack_timeout_ms)
-    log.info "initialized producer #{@client_id}"
+    refresh_producer()
   end
 
   def shutdown
@@ -115,31 +131,37 @@ class Fluent::KafkaOutputBuffered < Fluent::BufferedOutput
     bytes_by_topic = {}
     messages = []
     messages_bytes = 0
-    chunk.msgpack_each { |tag, time, record|
-      record['time'] = time if @output_include_time
-      record['tag'] = tag if @output_include_tag
-      topic = record['topic'] || @default_topic || tag
-
-      records_by_topic[topic] ||= 0
-      bytes_by_topic[topic] ||= 0
-
-      record_buf = parse_record(record)
-      record_buf_bytes = record_buf.bytesize
-      if messages.length > 0 and messages_bytes + record_buf_bytes > @kafka_agg_max_bytes
+    begin
+      chunk.msgpack_each { |tag, time, record|
+        record['time'] = time if @output_include_time
+        record['tag'] = tag if @output_include_tag
+        topic = record['topic'] || @default_topic || tag
+
+        records_by_topic[topic] ||= 0
+        bytes_by_topic[topic] ||= 0
+
+        record_buf = parse_record(record)
+        record_buf_bytes = record_buf.bytesize
+        if messages.length > 0 and messages_bytes + record_buf_bytes > @kafka_agg_max_bytes
+          @producer.send_messages(messages)
+          messages = []
+          messages_bytes = 0
+        end
+        messages << Poseidon::MessageToSend.new(topic, record_buf)
+        messages_bytes += record_buf_bytes
+
+        records_by_topic[topic] += 1
+        bytes_by_topic[topic] += record_buf_bytes
+      }
+      if messages.length > 0
         @producer.send_messages(messages)
-        messages = []
-        messages_bytes = 0
       end
-      messages << Poseidon::MessageToSend.new(topic, record_buf)
-      messages_bytes += record_buf_bytes
-
-      records_by_topic[topic] += 1
-      bytes_by_topic[topic] += record_buf_bytes
-    }
-    if messages.length > 0
-      @producer.send_messages(messages)
+      log.debug "(records|bytes) (#{records_by_topic}|#{bytes_by_topic})"
     end
-    log.debug "(records|bytes) (#{records_by_topic}|#{bytes_by_topic})"
+  rescue Exception => e
+    log.warn "Send exception occurred: #{e}"
+    refresh_producer()
+    # Raise exception to retry sendind messages
+    raise e
   end
-
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kafka
 version: !ruby/object:Gem::Version
-  version: 0.0.9
+  version: 0.0.10
 platform: ruby
 authors:
 - Hidemasa Togashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-01-18 00:00:00.000000000 Z
+date: 2015-03-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd