fluent-plugin-kafka 0.1.5 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 08281169934f59e651e0a601af17ab914d6a72af
-  data.tar.gz: 2098afa88314d62e83729aad0b97be02461142a6
+  metadata.gz: dfe217eaf0f6bd1b83a27e0ab49bbc959b8aa311
+  data.tar.gz: d45ade2033759a9c41f2ec98354f792db1302e65
 SHA512:
-  metadata.gz: b081eb740e9732e4dc919adc9166d3303df020af95bffbc01194e0eac1c477544a4c6c52b94affb71082ff1d8539e83d37e27cd4d0b696983065ba0f0a5433b4
-  data.tar.gz: f95855d295f0527f68384d0b636ddae382b767a725fe521f4b6dd9cd3687a16d1ae3c3e0a90429c3b75cd2f088a1168c32e7989bb4b8cb35baee6a868444045f
+  metadata.gz: 04e02ee2614432dc8e0a07e99ef2cb25cf86e764b4e1c37e2b2b719e0e4592d890539b74d5b4ca2def0b3dace18770c28b7eb08bb797fead911b7e6188cf0083
+  data.tar.gz: 0192de37ab4a8408a8fe22effa2a71d7bf0864f553fcc9ea139d2a9146f5b8b26f3e7702d5579bff2b2e4f12b4a66f4b12bec66f7c0c9e45459d44c2e5571eaf

data/fluent-plugin-kafka.gemspec

@@ -12,7 +12,7 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "fluent-plugin-kafka"
   gem.require_paths = ["lib"]
-  gem.version = '0.1.5'
+  gem.version = '0.2.0'
   gem.add_development_dependency 'test-unit'
   gem.add_dependency 'fluentd'
   gem.add_dependency 'poseidon_cluster'
@@ -20,4 +20,5 @@ Gem::Specification.new do |gem|
   gem.add_dependency 'yajl-ruby'
   gem.add_dependency 'msgpack'
   gem.add_dependency 'zookeeper'
+  gem.add_dependency 'ruby-kafka', '~> 0.3.2'
 end

data/lib/fluent/plugin/out_kafka_buffered.rb

@@ -1,10 +1,17 @@
 # encode: utf-8
+require 'thread'
+
 class Fluent::KafkaOutputBuffered < Fluent::BufferedOutput
   Fluent::Plugin.register_output('kafka_buffered', self)
 
   def initialize
     super
-    require 'poseidon'
+
+    require 'kafka'
+
+    @kafka = nil
+    @producers = {}
+    @producers_mutex = Mutex.new
   end
 
   config_param :brokers, :string, :default => 'localhost:9092',
@@ -32,17 +39,25 @@ DESC
   config_param :output_include_time, :bool, :default => false
   config_param :kafka_agg_max_bytes, :size, :default => 4*1024  #4k
 
+  # https://github.com/zendesk/ruby-kafka#encryption-and-authentication-using-ssl
+  config_param :ssl_ca_cert, :string, :default => nil,
+               :desc => "a PEM encoded CA cert to use with and SSL connection."
+  config_param :ssl_client_cert, :string, :default => nil,
+               :desc => "a PEM encoded client cert to use with and SSL connection. Must be used in combination with ssl_client_cert_key."
+  config_param :ssl_client_cert_key, :string, :default => nil,
+               :desc => "a PEM encoded client cert key to use with and SSL connection. Must be used in combination with ssl_client_cert."
+
   # poseidon producer options
-  config_param :max_send_retries, :integer, :default => 3,
+  config_param :max_send_retries, :integer, :default => 1,
                :desc => "Number of times to retry sending of messages to a leader."
   config_param :required_acks, :integer, :default => 0,
                :desc => "The number of acks required per request."
-  config_param :ack_timeout_ms, :integer, :default => 1500,
+  config_param :ack_timeout, :time, :default => nil,
                :desc => "How long the producer waits for acks."
-  config_param :compression_codec, :string, :default => 'none',
+  config_param :compression_codec, :string, :default => nil,
                :desc => <<-DESC
 The codec the producer uses to compress messages.
-Supported codecs: (none|gzip|snappy)
+Supported codecs: (gzip|snappy)
 DESC
 
   config_param :time_format, :string, :default => nil
@@ -54,9 +69,7 @@ DESC
     define_method("log") { $log }
   end
 
-  @seed_brokers = []
-
-  def refresh_producer()
+  def refresh_client(raise_error = true)
     if @zookeeper
       @seed_brokers = []
       z = Zookeeper.new(@zookeeper)
@@ -69,27 +82,39 @@ DESC
     end
     begin
       if @seed_brokers.length > 0
-        @producer = Poseidon::Producer.new(@seed_brokers, @client_id, :max_send_retries => @max_send_retries, :required_acks => @required_acks, :ack_timeout_ms => @ack_timeout_ms, :compression_codec => @compression_codec.to_sym)
-        log.info "initialized producer #{@client_id}"
+        @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
+                           ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key))
+        log.info "initialized kafka producer: #{@client_id}"
       else
         log.warn "No brokers found on Zookeeper"
       end
     rescue Exception => e
-      log.error e
+      if raise_error # During startup, error should be reported to engine and stop its phase for safety.
+        raise e
+      else
+        log.error e
+      end
     end
   end
 
+  def read_ssl_file(path)
+    return nil if path.nil?
+    File.read(path)
+  end
+
   def configure(conf)
     super
+
     if @zookeeper
       require 'zookeeper'
-      require 'yajl'
     else
       @seed_brokers = @brokers.match(",").nil? ? [@brokers] : @brokers.split(",")
       log.info "brokers has been set directly: #{@seed_brokers}"
     end
-    if @compression_codec == 'snappy'
-      require 'snappy'
+
+    if conf['ack_timeout_ms']
+      log.warn "'ack_timeout_ms' parameter is deprecated. Use second unit 'ack_timeout' instead"
+      @ack_timeout = conf['ack_timeout_ms'].to_i / 1000
     end
 
     @f_separator = case @field_separator
@@ -100,21 +125,48 @@ DESC
                    end
 
     @formatter_proc = setup_formatter(conf)
+
+    @producer_opts = {max_retries: @max_send_retries, required_acks: @required_acks,
+                      max_buffer_size: @buffer.buffer_chunk_limit / 10, max_buffer_bytesize: @buffer.buffer_chunk_limit * 2}
+    @producer_opts[:ack_timeout] = @ack_timeout if @ack_timeout
+    @producer_opts[:compression_codec] = @compression_codec.to_sym if @compression_codec
   end
 
   def start
     super
-    refresh_producer()
+    refresh_client
   end
 
   def shutdown
     super
+    shutdown_producers
+    @kafka = nil
   end
 
   def format(tag, time, record)
     [tag, time, record].to_msgpack
   end
 
+  def shutdown_producers
+    @producers_mutex.synchronize {
+      @producers.each { |key, producer|
+        producer.shutdown
+      }
+      @producers = {}
+    }
+  end
+
+  def get_producer
+    @producers_mutex.synchronize {
+      producer = @producers[Thread.current.object_id]
+      unless producer
+        producer = @kafka.producer(@producer_opts)
+        @producers[Thread.current.object_id] = producer
+      end
+      producer
+    }
+  end
+
   def setup_formatter(conf)
     if @output_data_type == 'json'
       require 'yajl'
@@ -144,9 +196,11 @@ DESC
   end
 
   def write(chunk)
+    producer = get_producer
+
     records_by_topic = {}
     bytes_by_topic = {}
-    messages = []
+    messages = 0
     messages_bytes = 0
     begin
       chunk.msgpack_each { |tag, time, record|
@@ -167,29 +221,31 @@ DESC
 
         record_buf = @formatter_proc.call(tag, time, record)
         record_buf_bytes = record_buf.bytesize
-        if messages.length > 0 and messages_bytes + record_buf_bytes > @kafka_agg_max_bytes
-          log.on_trace { log.trace("#{messages.length} messages send.") }
-          @producer.send_messages(messages)
-          messages = []
+        if (messages > 0) and (messages_bytes + record_buf_bytes > @kafka_agg_max_bytes)
+          log.on_trace { log.trace("#{messages} messages send.") }
+          producer.deliver_messages
+          messages = 0
           messages_bytes = 0
         end
         log.on_trace { log.trace("message will send to #{topic} with key: #{partition_key} and value: #{record_buf}.") }
-        messages << Poseidon::MessageToSend.new(topic, record_buf, partition_key)
+        messages += 1
+        producer.produce(record_buf, topic: topic, partition_key: partition_key)
         messages_bytes += record_buf_bytes
 
         records_by_topic[topic] += 1
         bytes_by_topic[topic] += record_buf_bytes
       }
-      if messages.length > 0
-        log.trace("#{messages.length} messages send.")
-        @producer.send_messages(messages)
+      if messages > 0
+        log.trace("#{messages} messages send.")
+        producer.deliver_messages
       end
       log.debug "(records|bytes) (#{records_by_topic}|#{bytes_by_topic})"
     end
   rescue Exception => e
     log.warn "Send exception occurred: #{e}"
-    @producer.close if @producer
-    refresh_producer()
+    # For safety, refresh client and its producers
+    shutdown_producers
+    refresh_client(false)
     # Raise exception to retry sendind messages
     raise e
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kafka
 version: !ruby/object:Gem::Version
-  version: 0.1.5
+  version: 0.2.0
 platform: ruby
 authors:
 - Hidemasa Togashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-06-02 00:00:00.000000000 Z
+date: 2016-06-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: test-unit
@@ -108,6 +108,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: ruby-kafka
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.3.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.3.2
 description: Fluentd plugin for Apache Kafka > 0.8
 email:
 - togachiro@gmail.com