sk-fluent-plugin-kafka 0.8.0

data/lib/fluent/plugin/kafka_plugin_util.rb

@@ -0,0 +1,52 @@
+module Fluent
+  module KafkaPluginUtil
+    module SSLSettings
+      def self.included(klass)
+        klass.instance_eval {
+          # https://github.com/zendesk/ruby-kafka#encryption-and-authentication-using-ssl
+          config_param :ssl_ca_cert, :array, :value_type => :string, :default => nil,
+                       :desc => "a PEM encoded CA cert to use with and SSL connection."
+          config_param :ssl_client_cert, :string, :default => nil,
+                       :desc => "a PEM encoded client cert to use with and SSL connection. Must be used in combination with ssl_client_cert_key."
+          config_param :ssl_client_cert_key, :string, :default => nil,
+                       :desc => "a PEM encoded client cert key to use with and SSL connection. Must be used in combination with ssl_client_cert."
+          config_param :ssl_ca_certs_from_system, :bool, :default => false,
+                       :desc => "this configures the store to look up CA certificates from the system default certificate store on an as needed basis. The location of the store can usually be determined by: OpenSSL::X509::DEFAULT_CERT_FILE."
+        }
+      end
+
+      def read_ssl_file(path)
+        return nil if path.nil?
+
+        if path.is_a?(Array)
+          path.map { |fp| File.read(fp) }
+        else
+          File.read(path)
+        end
+      end
+
+      def pickup_ssl_endpoint(node)
+        ssl_endpoint = node['endpoints'].find {|e| e.start_with?('SSL')}
+        raise 'no SSL endpoint found on Zookeeper' unless ssl_endpoint
+        return [URI.parse(ssl_endpoint).host, URI.parse(ssl_endpoint).port].join(':')
+      end
+    end
+
+    module SaslSettings
+      def self.included(klass)
+        klass.instance_eval {
+          config_param :principal, :string, :default => nil,
+                       :desc => "a Kerberos principal to use with SASL authentication (GSSAPI)."
+          config_param :keytab, :string, :default => nil,
+                       :desc => "a filepath to Kerberos keytab. Must be used with principal."
+          config_param :username, :string, :default => nil,
+                       :desc => "a username when using PLAIN/SCRAM SASL authentication"
+          config_param :password, :string, :default => nil, secret: true,
+                       :desc => "a password when using PLAIN/SCRAM SASL authentication"
+          config_param :scram_mechanism, :string, :default => nil,
+                       :desc => "if set, use SCRAM authentication with specified mechanism. When unset, default to PLAIN authentication"
+        }
+      end
+    end
+  end
+end

data/lib/fluent/plugin/kafka_producer_ext.rb

@@ -0,0 +1,308 @@
+require "set"
+require "kafka/partitioner"
+require "kafka/message_buffer"
+require "kafka/produce_operation"
+require "kafka/pending_message_queue"
+require "kafka/pending_message"
+require "kafka/compressor"
+require 'kafka/producer'
+
+# for out_kafka_buffered
+module Kafka
+  EMPTY_HEADER = {}
+
+  class Producer
+    def produce_for_buffered(value, key: nil, topic:, partition: nil, partition_key: nil)
+      create_time = Time.now
+
+      message = PendingMessage.new(
+        value: value,
+        key: key,
+        headers: EMPTY_HEADER,
+        topic: topic,
+        partition: partition,
+        partition_key: partition_key,
+        create_time: create_time
+      )
+
+      # If the producer is in transactional mode, all the message production
+      # must be used when the producer is currently in transaction
+      if @transaction_manager.transactional? && !@transaction_manager.in_transaction?
+        raise 'You must trigger begin_transaction before producing messages'
+      end
+
+      @target_topics.add(topic)
+      @pending_message_queue.write(message)
+
+      nil
+    end
+  end
+end
+
+# for out_kafka2
+module Kafka
+  class Client
+    def topic_producer(topic, compression_codec: nil, compression_threshold: 1, ack_timeout: 5, required_acks: :all, max_retries: 2, retry_backoff: 1, max_buffer_size: 1000, max_buffer_bytesize: 10_000_000, idempotent: false, transactional: false, transactional_id: nil, transactional_timeout: 60)
+      cluster = initialize_cluster
+      compressor = Compressor.new(
+        codec_name: compression_codec,
+        threshold: compression_threshold,
+        instrumenter: @instrumenter,
+      )
+
+      transaction_manager = TransactionManager.new(
+        cluster: cluster,
+        logger: @logger,
+        idempotent: idempotent,
+        transactional: transactional,
+        transactional_id: transactional_id,
+        transactional_timeout: transactional_timeout,
+      )
+
+      TopicProducer.new(topic,
+        cluster: cluster,
+        transaction_manager: transaction_manager,
+        logger: @logger,
+        instrumenter: @instrumenter,
+        compressor: compressor,
+        ack_timeout: ack_timeout,
+        required_acks: required_acks,
+        max_retries: max_retries,
+        retry_backoff: retry_backoff,
+        max_buffer_size: max_buffer_size,
+        max_buffer_bytesize: max_buffer_bytesize,
+      )
+    end
+  end
+
+  class TopicProducer
+    def initialize(topic, cluster:, transaction_manager:, logger:, instrumenter:, compressor:, ack_timeout:, required_acks:, max_retries:, retry_backoff:, max_buffer_size:, max_buffer_bytesize:)
+      @cluster = cluster
+      @transaction_manager = transaction_manager
+      @logger = logger
+      @instrumenter = instrumenter
+      @required_acks = required_acks == :all ? -1 : required_acks
+      @ack_timeout = ack_timeout
+      @max_retries = max_retries
+      @retry_backoff = retry_backoff
+      @max_buffer_size = max_buffer_size
+      @max_buffer_bytesize = max_buffer_bytesize
+      @compressor = compressor
+
+      @topic = topic
+      @cluster.add_target_topics(Set.new([topic]))
+
+      # A buffer organized by topic/partition.
+      @buffer = MessageBuffer.new
+
+      # Messages added by `#produce` but not yet assigned a partition.
+      @pending_message_queue = PendingMessageQueue.new
+    end
+
+    def produce(value, key: nil, partition: nil, partition_key: nil)
+      create_time = Time.now
+
+      message = PendingMessage.new(
+        value: value,
+        key: key,
+        headers: EMPTY_HEADER,
+        topic: @topic,
+        partition: partition,
+        partition_key: partition_key,
+        create_time: create_time
+      )
+
+      # If the producer is in transactional mode, all the message production
+      # must be used when the producer is currently in transaction
+      if @transaction_manager.transactional? && !@transaction_manager.in_transaction?
+        raise 'You must trigger begin_transaction before producing messages'
+      end
+
+      @pending_message_queue.write(message)
+
+      nil
+    end
+
+    def deliver_messages
+      # There's no need to do anything if the buffer is empty.
+      return if buffer_size == 0
+
+      deliver_messages_with_retries
+    end
+
+    # Returns the number of messages currently held in the buffer.
+    #
+    # @return [Integer] buffer size.
+    def buffer_size
+      @pending_message_queue.size + @buffer.size
+    end
+
+    def buffer_bytesize
+      @pending_message_queue.bytesize + @buffer.bytesize
+    end
+
+    # Deletes all buffered messages.
+    #
+    # @return [nil]
+    def clear_buffer
+      @buffer.clear
+      @pending_message_queue.clear
+    end
+
+    # Closes all connections to the brokers.
+    #
+    # @return [nil]
+    def shutdown
+      @transaction_manager.close
+      @cluster.disconnect
+    end
+
+    def init_transactions
+      @transaction_manager.init_transactions
+    end
+
+    def begin_transaction
+      @transaction_manager.begin_transaction
+    end
+
+    def commit_transaction
+      @transaction_manager.commit_transaction
+    end
+
+    def abort_transaction
+      @transaction_manager.abort_transaction
+    end
+
+    def transaction
+      raise 'This method requires a block' unless block_given?
+      begin_transaction
+      yield
+      commit_transaction
+    rescue Kafka::Producer::AbortTransaction
+      abort_transaction
+    rescue
+      abort_transaction
+      raise
+    end
+
+    def deliver_messages_with_retries
+      attempt = 0
+
+      #@cluster.add_target_topics(@target_topics)
+
+      operation = ProduceOperation.new(
+        cluster: @cluster,
+        transaction_manager: @transaction_manager,
+        buffer: @buffer,
+        required_acks: @required_acks,
+        ack_timeout: @ack_timeout,
+        compressor: @compressor,
+        logger: @logger,
+        instrumenter: @instrumenter,
+      )
+
+      loop do
+        attempt += 1
+
+        begin
+          @cluster.refresh_metadata_if_necessary!
+        rescue ConnectionError => e
+          raise DeliveryFailed.new(e, buffer_messages)
+        end
+
+        assign_partitions!
+        operation.execute
+
+        if @required_acks.zero?
+          # No response is returned by the brokers, so we can't know which messages
+          # have been successfully written. Our only option is to assume that they all
+          # have.
+          @buffer.clear
+        end
+
+        if buffer_size.zero?
+          break
+        elsif attempt <= @max_retries
+          @logger.warn "Failed to send all messages; attempting retry #{attempt} of #{@max_retries} after #{@retry_backoff}s"
+
+          sleep @retry_backoff
+        else
+          @logger.error "Failed to send all messages; keeping remaining messages in buffer"
+          break
+        end
+      end
+
+      unless @pending_message_queue.empty?
+        # Mark the cluster as stale in order to force a cluster metadata refresh.
+        @cluster.mark_as_stale!
+        raise DeliveryFailed, "Failed to assign partitions to #{@pending_message_queue.size} messages"
+      end
+
+      unless @buffer.empty?
+        partitions = @buffer.map {|topic, partition, _| "#{topic}/#{partition}" }.join(", ")
+
+        raise DeliveryFailed, "Failed to send messages to #{partitions}"
+      end
+    end
+
+    def assign_partitions!
+      failed_messages = []
+      partition_count = @cluster.partitions_for(@topic).count
+
+      @pending_message_queue.each do |message|
+        partition = message.partition
+
+        begin
+          if partition.nil?
+            partition = Partitioner.partition_for_key(partition_count, message)
+          end
+
+          @buffer.write(
+            value: message.value,
+            key: message.key,
+            headers: message.headers,
+            topic: message.topic,
+            partition: partition,
+            create_time: message.create_time,
+          )
+        rescue Kafka::Error => e
+          failed_messages << message
+        end
+      end
+
+      if failed_messages.any?
+        failed_messages.group_by(&:topic).each do |topic, messages|
+          @logger.error "Failed to assign partitions to #{messages.count} messages in #{topic}"
+        end
+
+        @cluster.mark_as_stale!
+      end
+
+      @pending_message_queue.replace(failed_messages)
+    end
+
+    def buffer_messages
+      messages = []
+
+      @pending_message_queue.each do |message|
+        messages << message
+      end
+
+      @buffer.each do |topic, partition, messages_for_partition|
+        messages_for_partition.each do |message|
+          messages << PendingMessage.new(
+            value: message.value,
+            key: message.key,
+            headers: message.headers,
+            topic: topic,
+            partition: partition,
+            partition_key: nil,
+            create_time: message.create_time
+          )
+        end
+      end
+
+      messages
+    end
+  end
+end

data/lib/fluent/plugin/out_kafka.rb

@@ -0,0 +1,254 @@
+require 'fluent/output'
+require 'fluent/plugin/kafka_plugin_util'
+
+class Fluent::KafkaOutput < Fluent::Output
+  Fluent::Plugin.register_output('kafka', self)
+
+  config_param :brokers, :string, :default => 'localhost:9092',
+               :desc => <<-DESC
+Set brokers directly
+<broker1_host>:<broker1_port>,<broker2_host>:<broker2_port>,..
+Note that you can choose to use either brokers or zookeeper.
+DESC
+  config_param :zookeeper, :string, :default => nil,
+               :desc => "Set brokers via Zookeeper: <zookeeper_host>:<zookeeper_port>"
+  config_param :zookeeper_path, :string, :default => '/brokers/ids',
+               :desc => "Path in path for Broker id. Default to /brokers/ids"
+  config_param :default_topic, :string, :default => nil,
+               :desc => "Output topic."
+  config_param :default_message_key, :string, :default => nil
+  config_param :default_partition_key, :string, :default => nil
+  config_param :default_partition, :integer, :default => nil
+  config_param :client_id, :string, :default => 'kafka'
+  config_param :sasl_over_ssl, :bool, :default => true,
+               :desc => <<-DESC
+Set to false to prevent SSL strict mode when using SASL authentication
+DESC
+  config_param :output_data_type, :string, :default => 'json',
+               :desc => "Supported format: (json|ltsv|msgpack|attr:<record name>|<formatter name>)"
+  config_param :output_include_tag, :bool, :default => false
+  config_param :output_include_time, :bool, :default => false
+  config_param :exclude_partition_key, :bool, :default => false,
+                       :desc => <<-DESC
+Set true to remove partition key from data
+DESC
+  config_param :exclude_partition, :bool, :default => false,
+               :desc => <<-DESC
+Set true to remove partition from data
+DESC
+
+  config_param :exclude_message_key, :bool, :default => false,
+                         :desc => <<-DESC
+Set true to remove message key from data
+DESC
+  config_param :exclude_topic_key, :bool, :default => false,
+                         :desc => <<-DESC
+Set true to remove topic name key from data
+DESC
+
+  # ruby-kafka producer options
+  config_param :max_send_retries, :integer, :default => 2,
+               :desc => "Number of times to retry sending of messages to a leader."
+  config_param :required_acks, :integer, :default => -1,
+               :desc => "The number of acks required per request."
+  config_param :ack_timeout, :integer, :default => nil,
+               :desc => "How long the producer waits for acks."
+  config_param :compression_codec, :string, :default => nil,
+               :desc => "The codec the producer uses to compress messages."
+
+  config_param :time_format, :string, :default => nil
+
+  config_param :max_buffer_size, :integer, :default => nil,
+               :desc => "Number of messages to be buffered by the kafka producer."
+
+  config_param :max_buffer_bytesize, :integer, :default => nil,
+               :desc => "Maximum size in bytes to be buffered."
+
+  config_param :active_support_notification_regex, :string, :default => nil,
+               :desc => <<-DESC
+Add a regular expression to capture ActiveSupport notifications from the Kafka client
+requires activesupport gem - records will be generated under fluent_kafka_stats.**
+DESC
+
+  include Fluent::KafkaPluginUtil::SSLSettings
+  include Fluent::KafkaPluginUtil::SaslSettings
+
+  attr_accessor :output_data_type
+  attr_accessor :field_separator
+
+  unless method_defined?(:log)
+    define_method("log") { $log }
+  end
+
+  def initialize
+    super
+
+    require 'kafka'
+
+    @kafka = nil
+  end
+
+  def refresh_client
+    if @zookeeper
+      @seed_brokers = []
+      z = Zookeeper.new(@zookeeper)
+      z.get_children(:path => @zookeeper_path)[:children].each do |id|
+        broker = Yajl.load(z.get(:path => @zookeeper_path + "/#{id}")[:data])
+        if @ssl_client_cert
+          @seed_brokers.push(pickup_ssl_endpoint(broker))
+        else
+          @seed_brokers.push("#{broker['host']}:#{broker['port']}")
+        end
+      end
+      z.close
+      log.info "brokers has been refreshed via Zookeeper: #{@seed_brokers}"
+    end
+    begin
+      if @seed_brokers.length > 0
+        if @scram_mechanism != nil && @username != nil && @password != nil
+          @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
+                             ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key), ssl_ca_certs_from_system: @ssl_ca_certs_from_system,
+                             sasl_scram_username: @username, sasl_scram_password: @password, sasl_scram_mechanism: @scram_mechanism, sasl_over_ssl: @sasl_over_ssl)
+        elsif @username != nil && @password != nil
+          @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
+                             ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key), ssl_ca_certs_from_system: @ssl_ca_certs_from_system,
+                             sasl_plain_username: @username, sasl_plain_password: @password)
+        else
+          @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
+                             ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key), ssl_ca_certs_from_system: @ssl_ca_certs_from_system,
+                             sasl_gssapi_principal: @principal, sasl_gssapi_keytab: @keytab)
+        end
+        log.info "initialized kafka producer: #{@client_id}"
+      else
+        log.warn "No brokers found on Zookeeper"
+      end
+    rescue Exception => e
+      log.error e
+    end
+  end
+
+  def configure(conf)
+    super
+
+    if @zookeeper
+      require 'zookeeper'
+    else
+      @seed_brokers = @brokers.split(",")
+      log.info "brokers has been set directly: #{@seed_brokers}"
+    end
+
+    if conf['ack_timeout_ms']
+      log.warn "'ack_timeout_ms' parameter is deprecated. Use second unit 'ack_timeout' instead"
+      @ack_timeout = conf['ack_timeout_ms'].to_i / 1000
+    end
+
+    @f_separator = case @field_separator
+                   when /SPACE/i then ' '
+                   when /COMMA/i then ','
+                   when /SOH/i then "\x01"
+                   else "\t"
+                   end
+
+    @formatter_proc = setup_formatter(conf)
+
+    @producer_opts = {max_retries: @max_send_retries, required_acks: @required_acks}
+    @producer_opts[:ack_timeout] = @ack_timeout if @ack_timeout
+    @producer_opts[:compression_codec] = @compression_codec.to_sym if @compression_codec
+    @producer_opts[:max_buffer_size] = @max_buffer_size if @max_buffer_size
+    @producer_opts[:max_buffer_bytesize] = @max_buffer_bytesize if @max_buffer_bytesize
+    if @active_support_notification_regex
+      require 'active_support/notifications'
+      require 'active_support/core_ext/hash/keys'
+      ActiveSupport::Notifications.subscribe(Regexp.new(@active_support_notification_regex)) do |*args|
+        event = ActiveSupport::Notifications::Event.new(*args)
+        message = event.payload.respond_to?(:stringify_keys) ? event.payload.stringify_keys : event.payload
+        @router.emit("fluent_kafka_stats.#{event.name}", Time.now.to_i, message)
+      end
+    end
+  end
+
+  def multi_workers_ready?
+    true
+  end
+
+  def start
+    super
+    refresh_client
+  end
+
+  def shutdown
+    super
+    @kafka = nil
+  end
+
+  def setup_formatter(conf)
+    if @output_data_type == 'json'
+      require 'yajl'
+      Proc.new { |tag, time, record| Yajl::Encoder.encode(record) }
+    elsif @output_data_type == 'ltsv'
+      require 'ltsv'
+      Proc.new { |tag, time, record| LTSV.dump(record) }
+    elsif @output_data_type == 'msgpack'
+      require 'msgpack'
+      Proc.new { |tag, time, record| record.to_msgpack }
+    elsif @output_data_type =~ /^attr:(.*)$/
+      @custom_attributes = $1.split(',').map(&:strip).reject(&:empty?)
+      @custom_attributes.unshift('time') if @output_include_time
+      @custom_attributes.unshift('tag') if @output_include_tag
+      Proc.new { |tag, time, record|
+        @custom_attributes.map { |attr|
+          record[attr].nil? ? '' : record[attr].to_s
+        }.join(@f_separator)
+      }
+    else
+      @formatter = Fluent::Plugin.new_formatter(@output_data_type)
+      @formatter.configure(conf)
+      @formatter.method(:format)
+    end
+  end
+
+  def emit(tag, es, chain)
+    begin
+      chain.next
+
+      # out_kafka is mainly for testing so don't need the performance unlike out_kafka_buffered.
+      producer = @kafka.producer(@producer_opts)
+
+      es.each do |time, record|
+        if @output_include_time
+          if @time_format
+            record['time'] = Time.at(time).strftime(@time_format)
+          else
+            record['time'] = time
+          end
+        end
+        record['tag'] = tag if @output_include_tag
+        topic = (@exclude_topic_key ? record.delete('topic') : record['topic']) || @default_topic || tag
+        partition_key = (@exclude_partition_key ? record.delete('partition_key') : record['partition_key']) || @default_partition_key
+        partition = (@exclude_partition ? record.delete('partition'.freeze) : record['partition'.freeze]) || @default_partition
+        message_key = (@exclude_message_key ? record.delete('message_key') : record['message_key']) || @default_message_key
+
+        value = @formatter_proc.call(tag, time, record)
+
+        log.trace { "message will send to #{topic} with partition_key: #{partition_key}, partition: #{partition}, message_key: #{message_key} and value: #{value}." }
+	begin
+          producer.produce(value, topic: topic, key: message_key, partition: partition, partition_key: partition_key)
+	rescue Kafka::BufferOverflow => e
+	  log.warn "BufferOverflow occurred: #{e}"
+	  log.info "Trying to deliver the messages to prevent the buffer from overflowing again."
+	  producer.deliver_messages
+	  log.info "Recovered from BufferOverflow successfully`"
+	end
+      end
+
+      producer.deliver_messages
+      producer.shutdown
+    rescue Exception => e
+      log.warn "Send exception occurred: #{e}"
+      producer.shutdown if producer
+      refresh_client
+      raise e
+    end
+  end
+
+end