fluent-plugin-kafka 0.0.8 → 0.0.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b1253b7f59126580e11cc6eb1d6f40318f9367bd
-  data.tar.gz: dc75b9c99e96455add399e4645dacf5f26358810
+  metadata.gz: 9cad342ed5984ebc424872a660379b6e471a135d
+  data.tar.gz: 73ea1a4a10f3584f719c45582d54f52b67b52c72
 SHA512:
-  metadata.gz: de391f877d6db11e0ccbd1a35646043274ef5aeab91015ff09981571fc356d24965289c93f599887d12958549114ef6441a4bce4b856728e8e00ca7dc4f1c011
-  data.tar.gz: 8c95e61c6617890c7be10dae81f963b4cdb5257829b8e5618cf4c5dfcda618456429a08d8347bc088520174c0bb63ed674528f41d887e3aa3793334ec9f978f4
+  metadata.gz: 82ff85d1bc2353109a05092f8cda28237bb4b24cec7af7cce539c55c8d4125e3a9a96e540e43ad33bf2dfe63c1120585b4c3ba7430d70812b973894a92cfc79b
+  data.tar.gz: ff574d372eee407d78e199809551bd1ac397c611b018b665416b0c8812cfe3f442b48e75aa320e66933b72a166172d9c09cc0fca913e5141c202d18a0cdd794a

data/README.md

@@ -35,7 +35,11 @@ Or install it yourself as:
 
     <match *.**>
       type                kafka
-      brokers             <broker1_host>:<broker1_ip>,<broker2_host>:<broker2_ip>,..
+
+      # Brokers: you can choose either brokers or zookeeper.
+      brokers             <broker1_host>:<broker1_port>,<broker2_host>:<broker2_port>,.. # Set brokers directly
+      zookeeper           <zookeeper_host>:<zookeeper_port> # Set brokers via Zookeeper
+
       default_topic       <output topic>
       output_data_type    (json|ltsv|msgpack|attr:<record name>)
       output_include_tag  (true|false) :default => false
@@ -57,7 +61,11 @@ See also [Poseidon::Producer](http://www.rubydoc.info/github/bpot/poseidon/Posei
 
     <match *.**>
       type                kafka_buffered
-      brokers             <broker1_host>:<broker1_ip>,<broker2_host>:<broker2_ip>,..
+
+      # Brokers: you can choose either brokers or zookeeper.
+      brokers             <broker1_host>:<broker1_port>,<broker2_host>:<broker2_port>,.. # Set brokers directly
+      zookeeper           <zookeeper_host>:<zookeeper_port> # Set brokers via Zookeeper
+
       default_topic       <output topic>
       flush_interval      <flush interval (sec) :default => 60>
       buffer_type         (file|memory)

data/fluent-plugin-kafka.gemspec

@@ -12,10 +12,11 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "fluent-plugin-kafka"
   gem.require_paths = ["lib"]
-  gem.version = '0.0.8'
+  gem.version = '0.0.9'
   gem.add_dependency 'fluentd'
   gem.add_dependency 'poseidon'
   gem.add_dependency 'ltsv'
   gem.add_dependency 'yajl-ruby'
   gem.add_dependency 'msgpack'
+  gem.add_dependency 'zookeeper'
 end

data/lib/fluent/plugin/out_kafka.rb

@@ -7,6 +7,7 @@ class Fluent::KafkaOutput < Fluent::Output
   end
 
   config_param :brokers, :string, :default => 'localhost:9092'
+  config_param :zookeeper, :string, :default => nil
   config_param :default_topic, :string, :default => nil
   config_param :default_partition, :integer, :default => 0
   config_param :client_id, :string, :default => 'kafka'
@@ -24,7 +25,20 @@ class Fluent::KafkaOutput < Fluent::Output
 
   def configure(conf)
     super
-    @seed_brokers = @brokers.match(",").nil? ? [@brokers] : @brokers.split(",")
+    if @zookeeper
+      require 'zookeeper'
+      require 'yajl'
+      @seed_brokers = []
+      z = Zookeeper.new(@zookeeper)
+      z.get_children(:path => '/brokers/ids')[:children].each do |id|
+        broker = Yajl.load(z.get(:path => "/brokers/ids/#{id}")[:data])
+        @seed_brokers.push("#{broker['host']}:#{broker['port']}")
+      end
+      log.info "brokers has been set via Zookeeper: #{@seed_brokers}"
+    else
+      @seed_brokers = @brokers.match(",").nil? ? [@brokers] : @brokers.split(",")
+      log.info "brokers has been set directly: #{@seed_brokers}"
+    end
     @producers = {} # keyed by topic:partition
     case @output_data_type
     when 'json'

data/lib/fluent/plugin/out_kafka_buffered.rb

@@ -8,12 +8,14 @@ class Fluent::KafkaOutputBuffered < Fluent::BufferedOutput
   end
 
   config_param :brokers, :string, :default => 'localhost:9092'
+  config_param :zookeeper, :string, :default => nil
   config_param :default_topic, :string, :default => nil
   config_param :default_partition, :integer, :default => 0
   config_param :client_id, :string, :default => 'kafka'
   config_param :output_data_type, :string, :default => 'json'
   config_param :output_include_tag, :bool, :default => false
   config_param :output_include_time, :bool, :default => false
+  config_param :kafka_agg_max_bytes, :size, :default => 4*1024  #4k
 
   # poseidon producer options
   config_param :max_send_retries, :integer, :default => 3
@@ -23,10 +25,27 @@ class Fluent::KafkaOutputBuffered < Fluent::BufferedOutput
   attr_accessor :output_data_type
   attr_accessor :field_separator
 
+  unless method_defined?(:log)
+    define_method("log") { $log }
+  end
+
   def configure(conf)
     super
-    @seed_brokers = @brokers.match(",").nil? ? [@brokers] : @brokers.split(",")
-    @producers = {} # keyed by topic:partition
+    if @zookeeper
+      require 'zookeeper'
+      require 'yajl'
+      @seed_brokers = []
+      z = Zookeeper.new(@zookeeper)
+      z.get_children(:path => '/brokers/ids')[:children].each do |id|
+        broker = Yajl.load(z.get(:path => "/brokers/ids/#{id}")[:data])
+        @seed_brokers.push("#{broker['host']}:#{broker['port']}")
+      end
+      log.info "brokers has been set via Zookeeper: #{@seed_brokers}"
+    else
+      @seed_brokers = @brokers.match(",").nil? ? [@brokers] : @brokers.split(",")
+      log.info "brokers has been set directly: #{@seed_brokers}"
+    end
+
     case @output_data_type
     when 'json'
       require 'yajl'
@@ -58,6 +77,8 @@ class Fluent::KafkaOutputBuffered < Fluent::BufferedOutput
 
   def start
     super
+    @producer = Poseidon::Producer.new(@seed_brokers, @client_id, :max_send_retries => @max_send_retries, :required_acks => @required_acks, :ack_timeout_ms => @ack_timeout_ms)
+    log.info "initialized producer #{@client_id}"
   end
 
   def shutdown
@@ -91,26 +112,34 @@ class Fluent::KafkaOutputBuffered < Fluent::BufferedOutput
 
   def write(chunk)
     records_by_topic = {}
+    bytes_by_topic = {}
+    messages = []
+    messages_bytes = 0
     chunk.msgpack_each { |tag, time, record|
       record['time'] = time if @output_include_time
       record['tag'] = tag if @output_include_tag
-      topic = record['topic'] || self.default_topic || tag
-      partition = record['partition'] || self.default_partition
-      message = Poseidon::MessageToSend.new(topic, parse_record(record))
-      records_by_topic[topic] ||= []
-      records_by_topic[topic][partition] ||= []
-      records_by_topic[topic][partition] << message
-    }
-    publish(records_by_topic)
-  end
+      topic = record['topic'] || @default_topic || tag
 
-  def publish(records_by_topic)
-    records_by_topic.each { |topic, partitions|
-      partitions.each_with_index { |messages, partition|
-        next if not messages
-        @producers[topic] ||= Poseidon::Producer.new(@seed_brokers, self.client_id, :max_send_retries => @max_send_retries, :required_acks => @required_acks, :ack_timeout_ms => @ack_timeout_ms)
-        @producers[topic].send_messages(messages)
-      }
+      records_by_topic[topic] ||= 0
+      bytes_by_topic[topic] ||= 0
+
+      record_buf = parse_record(record)
+      record_buf_bytes = record_buf.bytesize
+      if messages.length > 0 and messages_bytes + record_buf_bytes > @kafka_agg_max_bytes
+        @producer.send_messages(messages)
+        messages = []
+        messages_bytes = 0
+      end
+      messages << Poseidon::MessageToSend.new(topic, record_buf)
+      messages_bytes += record_buf_bytes
+
+      records_by_topic[topic] += 1
+      bytes_by_topic[topic] += record_buf_bytes
     }
+    if messages.length > 0
+      @producer.send_messages(messages)
+    end
+    log.debug "(records|bytes) (#{records_by_topic}|#{bytes_by_topic})"
   end
+
 end

metadata

@@ -1,83 +1,97 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kafka
 version: !ruby/object:Gem::Version
-  version: 0.0.8
+  version: 0.0.9
 platform: ruby
 authors:
 - Hidemasa Togashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-10-15 00:00:00.000000000 Z
+date: 2015-01-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: poseidon
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: ltsv
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: yajl-ruby
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: msgpack
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: zookeeper
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 description: Fluentd plugin for Apache Kafka > 0.8
@@ -106,17 +120,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.14
+rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
 summary: Fluentd plugin for Apache Kafka > 0.8