fluent-plugin-kafka 0.4.2 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3022fed18061233c956a0b13292cbebb9da8a79f
-  data.tar.gz: 5e8f4ecf691f72b620b21358a60f8d8994b9e359
+  metadata.gz: a09a4933e7d0f7a30094cd98900a03a80dac3c9a
+  data.tar.gz: 9421658f52091e37e39e32ae10e7e93132d9394f
 SHA512:
-  metadata.gz: fb824c849ef4feb963a1c6675717aadcaea78969ed30285b1d871f9e2a8f62d026489ca660a849a79c4ac8040673ec816b870037647fdde1dc19b7cf772c4f4f
-  data.tar.gz: e1c6ff1982adcac8ce67f0c7f87f94c6b1b497b92f994a4c5c50ac8d55151c01aa2f798dcf7c873ea5158069b835ea92255bc177da972df83b07c641a67c1aaf
+  metadata.gz: e3a72bb6fecbe2dd0204e8bc84234d3d11c44017699eb7ce88ee4b154d04966f939487d85ef90b4811510c490fb733a922ba1f70012dc6fbf9490afebf843ed7
+  data.tar.gz: 5b633c21eadd8797a5a6672191a391468a5b8c8e2b4c7968f74b67d7e3edc2978f3db0df2c157b835a7dcd75e899f4b26baf1d62c02ac7cf47f55943dd72dba8

data/ChangeLog

@@ -1,4 +1,8 @@
-Release 0.4.2 - 2016012/10
+Release 0.5.0 - 2017/01/17
+
+	* output: Add out_kafka2 plugin with v0.14 API
+
+Release 0.4.2 - 2016/12/10
 
 	* input: Add use_record_time and time_format parameters
 	* Update ruby-kafka dependency to 0.3.16.beta2

data/fluent-plugin-kafka.gemspec

@@ -12,7 +12,7 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "fluent-plugin-kafka"
   gem.require_paths = ["lib"]
-  gem.version       = '0.4.2'
+  gem.version       = '0.5.0'
   gem.required_ruby_version = ">= 2.1.0"
 
   gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]

data/lib/fluent/plugin/kafka_producer_ext.rb

@@ -1,5 +1,13 @@
+require "set"
+require "kafka/partitioner"
+require "kafka/message_buffer"
+require "kafka/produce_operation"
+require "kafka/pending_message_queue"
+require "kafka/pending_message"
+require "kafka/compressor"
 require 'kafka/producer'
 
+# for out_kafka_buffered
 module Kafka
   class Producer
     def produce2(value, key: nil, topic:, partition: nil, partition_key: nil)
@@ -22,3 +30,196 @@ module Kafka
     end
   end
 end
+
+# for out_kafka2
+module Kafka
+  class Client
+    def topic_producer(topic, compression_codec: nil, compression_threshold: 1, ack_timeout: 5, required_acks: :all, max_retries: 2, retry_backoff: 1, max_buffer_size: 1000, max_buffer_bytesize: 10_000_000)
+      compressor = Compressor.new(
+        codec_name: compression_codec,
+        threshold: compression_threshold,
+        instrumenter: @instrumenter,
+      )
+
+      TopicProducer.new(topic,
+        cluster: initialize_cluster,
+        logger: @logger,
+        instrumenter: @instrumenter,
+        compressor: compressor,
+        ack_timeout: ack_timeout,
+        required_acks: required_acks,
+        max_retries: max_retries,
+        retry_backoff: retry_backoff,
+        max_buffer_size: max_buffer_size,
+        max_buffer_bytesize: max_buffer_bytesize,
+      )
+    end
+  end
+
+  class TopicProducer
+    def initialize(topic, cluster:, logger:, instrumenter:, compressor:, ack_timeout:, required_acks:, max_retries:, retry_backoff:, max_buffer_size:, max_buffer_bytesize:)
+      @cluster = cluster
+      @logger = logger
+      @instrumenter = instrumenter
+      @required_acks = required_acks == :all ? -1 : required_acks
+      @ack_timeout = ack_timeout
+      @max_retries = max_retries
+      @retry_backoff = retry_backoff
+      @max_buffer_size = max_buffer_size
+      @max_buffer_bytesize = max_buffer_bytesize
+      @compressor = compressor
+
+      @topic = topic
+      @cluster.add_target_topics(Set.new([topic]))
+
+      # A buffer organized by topic/partition.
+      @buffer = MessageBuffer.new
+
+      # Messages added by `#produce` but not yet assigned a partition.
+      @pending_message_queue = PendingMessageQueue.new
+    end
+
+    def produce(value, key, partition, partition_key)
+      create_time = Time.now
+
+      message = PendingMessage.new(
+        value,
+        key,
+        @topic,
+        partition,
+        partition_key,
+        create_time,
+        key.to_s.bytesize + value.to_s.bytesize
+      )
+
+      @pending_message_queue.write(message)
+
+      nil
+    end
+
+    def deliver_messages
+      # There's no need to do anything if the buffer is empty.
+      return if buffer_size == 0
+
+      deliver_messages_with_retries
+    end
+
+    # Returns the number of messages currently held in the buffer.
+    #
+    # @return [Integer] buffer size.
+    def buffer_size
+      @pending_message_queue.size + @buffer.size
+    end
+
+    def buffer_bytesize
+      @pending_message_queue.bytesize + @buffer.bytesize
+    end
+
+    # Deletes all buffered messages.
+    #
+    # @return [nil]
+    def clear_buffer
+      @buffer.clear
+      @pending_message_queue.clear
+    end
+
+    # Closes all connections to the brokers.
+    #
+    # @return [nil]
+    def shutdown
+      @cluster.disconnect
+    end
+
+    private
+
+    def deliver_messages_with_retries
+      attempt = 0
+
+      #@cluster.add_target_topics(@target_topics)
+
+      operation = ProduceOperation.new(
+        cluster: @cluster,
+        buffer: @buffer,
+        required_acks: @required_acks,
+        ack_timeout: @ack_timeout,
+        compressor: @compressor,
+        logger: @logger,
+        instrumenter: @instrumenter,
+      )
+
+      loop do
+        attempt += 1
+
+        @cluster.refresh_metadata_if_necessary!
+
+        assign_partitions!
+        operation.execute
+
+        if @required_acks.zero?
+          # No response is returned by the brokers, so we can't know which messages
+          # have been successfully written. Our only option is to assume that they all
+          # have.
+          @buffer.clear
+        end
+
+        if buffer_size.zero?
+          break
+        elsif attempt <= @max_retries
+          @logger.warn "Failed to send all messages; attempting retry #{attempt} of #{@max_retries} after #{@retry_backoff}s"
+
+          sleep @retry_backoff
+        else
+          @logger.error "Failed to send all messages; keeping remaining messages in buffer"
+          break
+        end
+      end
+
+      unless @pending_message_queue.empty?
+        # Mark the cluster as stale in order to force a cluster metadata refresh.
+        @cluster.mark_as_stale!
+        raise DeliveryFailed, "Failed to assign partitions to #{@pending_message_queue.size} messages"
+      end
+
+      unless @buffer.empty?
+        partitions = @buffer.map {|topic, partition, _| "#{topic}/#{partition}" }.join(", ")
+
+        raise DeliveryFailed, "Failed to send messages to #{partitions}"
+      end
+    end
+
+    def assign_partitions!
+      failed_messages = []
+      partition_count = @cluster.partitions_for(@topic).count
+
+      @pending_message_queue.each do |message|
+        partition = message.partition
+
+        begin
+          if partition.nil?
+            partition = Partitioner.partition_for_key(partition_count, message)
+          end
+
+          @buffer.write(
+            value: message.value,
+            key: message.key,
+            topic: message.topic,
+            partition: partition,
+            create_time: message.create_time,
+          )
+        rescue Kafka::Error => e
+          failed_messages << message
+        end
+      end
+
+      if failed_messages.any?
+        failed_messages.group_by(&:topic).each do |topic, messages|
+          @logger.error "Failed to assign partitions to #{messages.count} messages in #{topic}"
+        end
+
+        @cluster.mark_as_stale!
+      end
+
+      @pending_message_queue.replace(failed_messages)
+    end
+  end
+end

data/lib/fluent/plugin/out_kafka2.rb

@@ -0,0 +1,187 @@
+require 'fluent/plugin/output'
+require 'fluent/plugin/kafka_plugin_util'
+
+require 'kafka'
+require 'fluent/plugin/kafka_producer_ext'
+
+module Fluent::Plugin
+  class Fluent::Kafka2Output < Output
+    Fluent::Plugin.register_output('kafka2', self)
+
+    helpers :inject, :formatter
+
+    config_param :brokers, :array, :value_type => :string, :default => ['localhost:9092'],
+                 :desc => <<-DESC
+Set brokers directly:
+<broker1_host>:<broker1_port>,<broker2_host>:<broker2_port>,..
+DESC
+    config_param :default_topic, :string, :default => nil,
+                 :desc => "Default output topic when record doesn't have topic field"
+    config_param :default_message_key, :string, :default => nil
+    config_param :default_partition_key, :string, :default => nil
+    config_param :default_partition, :integer, :default => nil
+    config_param :client_id, :string, :default => 'fluentd'
+    config_param :exclude_partition_key, :bool, :default => false,
+                 :desc => 'Set true to remove partition key from data'
+    config_param :exclude_partition, :bool, :default => false,
+                 :desc => 'Set true to remove partition from data'
+    config_param :exclude_message_key, :bool, :default => false,
+                 :desc => 'Set true to remove partition key from data'
+    config_param :exclude_topic_key, :bool, :default => false,
+                 :desc => 'Set true to remove topic name key from data'
+
+    config_param :get_kafka_client_log, :bool, :default => false
+
+    # ruby-kafka producer options
+    config_param :max_send_retries, :integer, :default => 2,
+                 :desc => "Number of times to retry sending of messages to a leader."
+    config_param :required_acks, :integer, :default => -1,
+                 :desc => "The number of acks required per request."
+    config_param :ack_timeout, :time, :default => nil,
+                 :desc => "How long the producer waits for acks."
+    config_param :compression_codec, :string, :default => nil,
+                 :desc => <<-DESC
+The codec the producer uses to compress messages.
+Supported codecs: (gzip|snappy)
+DESC
+
+    config_section :buffer do
+      config_set_default :chunk_keys, ["topic"]
+    end
+    config_section :format do
+      config_set_default :@type, 'json'
+    end
+
+    include Fluent::KafkaPluginUtil::SSLSettings
+
+    def initialize
+      super
+
+      @kafka = nil
+    end
+
+    def refresh_client(raise_error = true)
+      begin
+        logger = @get_kafka_client_log ? log : nil
+        @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: logger, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
+                           ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key))
+        log.info "initialized kafka producer: #{@client_id}"
+      rescue Exception => e
+        if raise_error # During startup, error should be reported to engine and stop its phase for safety.
+          raise e
+        else
+          log.error e
+        end
+      end
+    end
+
+    def configure(conf)
+      super
+
+      if @brokers.size > 0
+        log.info "brokers has been set: #{@brokers}"
+      else
+        raise Fluent::Config, 'No brokers specified. Need one broker at least.'
+      end
+
+      formatter_conf = conf.elements('format').first
+      unless formatter_conf
+        raise Fluent::ConfigError, "<format> section is required."
+      end
+      unless formatter_conf["@type"]
+        raise Fluent::ConfigError, "format/@type is required."
+      end
+      @formatter_proc = setup_formatter(formatter_conf)
+
+      @producer_opts = {max_retries: @max_send_retries, required_acks: @required_acks}
+      @producer_opts[:ack_timeout] = @ack_timeout if @ack_timeout
+      @producer_opts[:compression_codec] = @compression_codec.to_sym if @compression_codec
+    end
+
+    def multi_workers_ready?
+      true
+    end
+
+    def start
+      super
+      refresh_client
+    end
+
+    def close
+      super
+      @kafka.close if @kafka
+    end
+
+    def terminate
+      super
+      @kafka = nil
+    end
+
+    def setup_formatter(conf)
+      type = conf['@type']
+      case type
+      when 'json'
+        begin
+          require 'oj'
+          Oj.default_options = Fluent::DEFAULT_OJ_OPTIONS
+          Proc.new { |tag, time, record| Oj.dump(record) }
+        rescue LoadError
+          require 'yajl'
+          Proc.new { |tag, time, record| Yajl::Encoder.encode(record) }
+        end
+      when 'ltsv'
+        require 'ltsv'
+        Proc.new { |tag, time, record| LTSV.dump(record) }
+      else
+        @formatter = formatter_create(usage: 'kafka-plugin', conf: conf)
+        @formatter.method(:format)
+      end
+    end
+
+    # TODO: optimize write performance
+    def write(chunk)
+      tag = chunk.metadata.tag
+      topic = chunk.metadata.variables[:topic] || @default_topic || tag
+      producer = @kafka.topic_producer(topic, @producer_opts)
+
+      messages = 0
+      record_buf = nil
+
+      begin
+        chunk.msgpack_each { |time, record|
+          begin
+            record = inject_values_to_record(tag, time, record)
+            record.delete('topic'.freeze) if @exclude_topic_key
+            partition_key = (@exclude_partition_key ? record.delete('partition_key'.freeze) : record['partition_key'.freeze]) || @default_partition_key
+            partition = (@exclude_partition ? record.delete('partition'.freeze) : record['partition'.freeze]) || @default_partition
+            message_key = (@exclude_message_key ? record.delete('message_key'.freeze) : record['message_key'.freeze]) || @default_message_key
+
+            record_buf = @formatter_proc.call(tag, time, record)
+          rescue StandardError => e
+            log.warn "unexpected error during format record. Skip broken event:", :error => e.to_s, :error_class => e.class.to_s, :time => time, :record => record
+            next
+          end
+
+          log.on_trace { "message will send to #{topic} with partition_key: #{partition_key}, partition: #{partition}, message_key: #{message_key} and value: #{record_buf}." }
+          messages += 1
+
+          producer.produce(record_buf, message_key, partition, partition_key)
+        }
+
+        if messages > 0
+          log.trace { "#{messages} messages send." }
+          producer.deliver_messages
+        end
+      end
+    rescue Exception => e
+      log.warn "Send exception occurred: #{e}"
+      log.warn "Exception Backtrace : #{e.backtrace.join("\n")}"
+      # For safety, refresh client and its producers
+      refresh_client(false)
+      # Raise exception to retry sendind messages
+      raise e
+    ensure
+      producer.shutdown if producer
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kafka
 version: !ruby/object:Gem::Version
-  version: 0.4.2
+  version: 0.5.0
 platform: ruby
 authors:
 - Hidemasa Togashi
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-01-10 00:00:00.000000000 Z
+date: 2017-01-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -108,6 +108,7 @@ files:
 - lib/fluent/plugin/kafka_plugin_util.rb
 - lib/fluent/plugin/kafka_producer_ext.rb
 - lib/fluent/plugin/out_kafka.rb
+- lib/fluent/plugin/out_kafka2.rb
 - lib/fluent/plugin/out_kafka_buffered.rb
 - test/helper.rb
 - test/plugin/test_out_kafka.rb
@@ -130,7 +131,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.1
+rubygems_version: 2.6.8
 signing_key: 
 specification_version: 4
 summary: Fluentd plugin for Apache Kafka > 0.8