roched-fluent-plugin-kafka 0.6.5

data/lib/fluent/plugin/in_kafka.rb

@@ -0,0 +1,310 @@
+require 'fluent/input'
+require 'fluent/time'
+require 'fluent/plugin/kafka_plugin_util'
+
+class Fluent::KafkaInput < Fluent::Input
+  Fluent::Plugin.register_input('kafka', self)
+
+  config_param :format, :string, :default => 'json',
+               :desc => "Supported format: (json|text|ltsv|msgpack)"
+  config_param :message_key, :string, :default => 'message',
+               :desc => "For 'text' format only."
+  config_param :host, :string, :default => nil,
+               :desc => "Broker host"
+  config_param :port, :integer, :default => nil,
+               :desc => "Broker port"
+  config_param :brokers, :string, :default => 'localhost:9092',
+               :desc => "List of broker-host:port, separate with comma, must set."
+  config_param :interval, :integer, :default => 1, # seconds
+               :desc => "Interval (Unit: seconds)"
+  config_param :topics, :string, :default => nil,
+               :desc => "Listening topics(separate with comma',')"
+  config_param :client_id, :string, :default => 'kafka'
+  config_param :partition, :integer, :default => 0,
+               :desc => "Listening partition"
+  config_param :offset, :integer, :default => -1,
+               :desc => "Listening start offset"
+  config_param :add_prefix, :string, :default => nil,
+               :desc => "Tag prefix"
+  config_param :add_suffix, :string, :default => nil,
+               :desc => "tag suffix"
+  config_param :add_offset_in_record, :bool, :default => false
+
+  config_param :offset_zookeeper, :string, :default => nil
+  config_param :offset_zk_root_node, :string, :default => '/fluent-plugin-kafka'
+  config_param :use_record_time, :bool, :default => false,
+               :desc => "Replace message timestamp with contents of 'time' field."
+  config_param :time_format, :string, :default => nil,
+               :desc => "Time format to be used to parse 'time' filed."
+
+  # Kafka#fetch_messages options
+  config_param :max_bytes, :integer, :default => nil,
+               :desc => "Maximum number of bytes to fetch."
+  config_param :max_wait_time, :integer, :default => nil,
+               :desc => "How long to block until the server sends us data."
+  config_param :min_bytes, :integer, :default => nil,
+               :desc => "Smallest amount of data the server should send us."
+
+  include Fluent::KafkaPluginUtil::SSLSettings
+  include Fluent::KafkaPluginUtil::SaslSettings
+
+  unless method_defined?(:router)
+    define_method("router") { Fluent::Engine }
+  end
+
+  def initialize
+    super
+    require 'kafka'
+
+    @time_parser = nil
+  end
+
+  def configure(conf)
+    super
+
+    @topic_list = []
+    if @topics
+      @topic_list = @topics.split(',').map { |topic|
+        TopicEntry.new(topic.strip, @partition, @offset)
+      }
+    else
+      conf.elements.select { |element| element.name == 'topic' }.each do |element|
+        unless element.has_key?('topic')
+          raise Fluent::ConfigError, "kafka: 'topic' is a require parameter in 'topic element'."
+        end
+        partition = element.has_key?('partition') ? element['partition'].to_i : 0
+        offset = element.has_key?('offset') ? element['offset'].to_i : -1
+        @topic_list.push(TopicEntry.new(element['topic'], partition, offset))
+      end
+    end
+
+    if @topic_list.empty?
+      raise Fluent::ConfigError, "kafka: 'topics' or 'topic element' is a require parameter"
+    end
+
+    # For backward compatibility
+    @brokers = case
+               when @host && @port
+                 ["#{@host}:#{@port}"]
+               when @host
+                 ["#{@host}:9092"]
+               when @port
+                 ["localhost:#{@port}"]
+               else
+                 @brokers
+               end
+
+    if conf['max_wait_ms']
+      log.warn "'max_wait_ms' parameter is deprecated. Use second unit 'max_wait_time' instead"
+      @max_wait_time = conf['max_wait_ms'].to_i / 1000
+    end
+
+    @max_wait_time = @interval if @max_wait_time.nil?
+
+    require 'zookeeper' if @offset_zookeeper
+
+    @parser_proc = setup_parser
+
+    if @use_record_time and @time_format
+      @time_parser = Fluent::TextParser::TimeParser.new(@time_format)
+    end
+  end
+
+  def setup_parser
+    case @format
+    when 'json'
+      require 'yajl'
+      Proc.new { |msg, te|
+        r = Yajl::Parser.parse(msg.value)
+        add_offset_in_hash(r, te, msg.offset) if @add_offset_in_record
+        r
+      }
+    when 'ltsv'
+      require 'ltsv'
+      Proc.new { |msg, te|
+        r = LTSV.parse(msg.value, {:symbolize_keys => false}).first
+        add_offset_in_hash(r, te, msg.offset) if @add_offset_in_record
+        r
+      }
+    when 'msgpack'
+      require 'msgpack'
+      Proc.new { |msg, te|
+        r = MessagePack.unpack(msg.value)
+        add_offset_in_hash(r, te, msg.offset) if @add_offset_in_record
+        r
+      }
+    when 'text'
+      Proc.new { |msg, te|
+        r = {@message_key => msg.value}
+        add_offset_in_hash(r, te, msg.offset) if @add_offset_in_record
+        r
+      }
+    end
+  end
+
+  def add_offset_in_hash(hash, te, offset)
+    hash['kafka_topic'.freeze] = te.topic
+    hash['kafka_partition'.freeze] = te.partition
+    hash['kafka_offset'.freeze] = offset
+  end
+
+  def start
+    super
+
+    @loop = Coolio::Loop.new
+    opt = {}
+    opt[:max_bytes] = @max_bytes if @max_bytes
+    opt[:max_wait_time] = @max_wait_time if @max_wait_time
+    opt[:min_bytes] = @min_bytes if @min_bytes
+
+    @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id,
+                       ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
+                       ssl_client_cert: read_ssl_file(@ssl_client_cert),
+                       ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key),
+                       sasl_gssapi_principal: @principal, sasl_gssapi_keytab: @keytab)
+    @zookeeper = Zookeeper.new(@offset_zookeeper) if @offset_zookeeper
+
+    @topic_watchers = @topic_list.map {|topic_entry|
+      offset_manager = OffsetManager.new(topic_entry, @zookeeper, @offset_zk_root_node) if @offset_zookeeper
+      TopicWatcher.new(
+        topic_entry,
+        @kafka,
+        interval,
+        @parser_proc,
+        @add_prefix,
+        @add_suffix,
+        offset_manager,
+        router,
+        opt)
+    }
+    @topic_watchers.each {|tw|
+      tw.attach(@loop)
+    }
+    @thread = Thread.new(&method(:run))
+  end
+
+  def shutdown
+    @loop.stop
+    @zookeeper.close! if @zookeeper
+    @thread.join
+    @kafka.close
+    super
+  end
+
+  def run
+    @loop.run
+  rescue => e
+    $log.error "unexpected error", :error => e.to_s
+    $log.error_backtrace
+  end
+
+  class TopicWatcher < Coolio::TimerWatcher
+    def initialize(topic_entry, kafka, interval, parser, add_prefix, add_suffix, offset_manager, router, options={})
+      @topic_entry = topic_entry
+      @kafka = kafka
+      @callback = method(:consume)
+      @parser = parser
+      @add_prefix = add_prefix
+      @add_suffix = add_suffix
+      @options = options
+      @offset_manager = offset_manager
+      @router = router
+
+      @next_offset = @topic_entry.offset
+      if @topic_entry.offset == -1 && offset_manager
+        @next_offset = offset_manager.next_offset
+      end
+      @fetch_args = {
+        topic: @topic_entry.topic,
+        partition: @topic_entry.partition,
+      }.merge(@options)
+
+      super(interval, true)
+    end
+
+    def on_timer
+      @callback.call
+    rescue => e
+      # TODO log?
+      $log.error e.to_s
+      $log.error_backtrace
+    end
+
+    def consume
+      offset = @next_offset
+      @fetch_args[:offset] = offset
+      messages = @kafka.fetch_messages(@fetch_args)
+
+      return if messages.size.zero?
+
+      es = Fluent::MultiEventStream.new
+      tag = @topic_entry.topic
+      tag = @add_prefix + "." + tag if @add_prefix
+      tag = tag + "." + @add_suffix if @add_suffix
+
+      messages.each { |msg|
+        begin
+          record = @parser.call(msg, @topic_entry)
+          if @use_record_time
+            if @time_format
+              record_time = @time_parser.parse(record['time'])
+            else
+              record_time = record['time']
+            end
+          else
+            record_time = Fluent::Engine.now
+          end
+          es.add(record_time, record)
+        rescue => e
+          $log.warn "parser error in #{@topic_entry.topic}/#{@topic_entry.partition}", :error => e.to_s, :value => msg.value, :offset => msg.offset
+          $log.debug_backtrace
+        end
+      }
+      offset = messages.last.offset + 1
+
+      unless es.empty?
+        @router.emit_stream(tag, es)
+
+        if @offset_manager
+          @offset_manager.save_offset(offset)
+        end
+        @next_offset = offset
+      end
+    end
+  end
+
+  class TopicEntry
+    def initialize(topic, partition, offset)
+      @topic = topic
+      @partition = partition
+      @offset = offset
+    end
+    attr_reader :topic, :partition, :offset
+  end
+
+  class OffsetManager
+    def initialize(topic_entry, zookeeper, zk_root_node)
+      @zookeeper = zookeeper
+      @zk_path = "#{zk_root_node}/#{topic_entry.topic}/#{topic_entry.partition}/next_offset"
+      create_node(@zk_path, topic_entry.topic, topic_entry.partition)
+    end
+
+    def create_node(zk_path, topic, partition)
+      path = ""
+      zk_path.split(/(\/[^\/]+)/).reject(&:empty?).each { |dir|
+        path = path + dir
+        @zookeeper.create(:path => "#{path}")
+      }
+      $log.trace "use zk offset node : #{path}"
+    end
+
+    def next_offset
+      @zookeeper.get(:path => @zk_path)[:data].to_i
+    end
+
+    def save_offset(offset)
+      @zookeeper.set(:path => @zk_path, :data => offset.to_s)
+      $log.trace "update zk offset node : #{offset.to_s}"
+    end
+  end
+end

data/lib/fluent/plugin/in_kafka_group.rb

@@ -0,0 +1,236 @@
+require 'fluent/input'
+require 'fluent/time'
+require 'fluent/plugin/kafka_plugin_util'
+
+class Fluent::KafkaGroupInput < Fluent::Input
+  Fluent::Plugin.register_input('kafka_group', self)
+
+  config_param :brokers, :string, :default => 'localhost:9092',
+               :desc => "List of broker-host:port, separate with comma, must set."
+  config_param :consumer_group, :string,
+               :desc => "Consumer group name, must set."
+  config_param :topics, :string,
+               :desc => "Listening topics(separate with comma',')."
+  config_param :format, :string, :default => 'json',
+               :desc => "Supported format: (json|text|ltsv|msgpack)"
+  config_param :message_key, :string, :default => 'message',
+               :desc => "For 'text' format only."
+  config_param :add_prefix, :string, :default => nil,
+               :desc => "Tag prefix (Optional)"
+  config_param :add_suffix, :string, :default => nil,
+               :desc => "Tag suffix (Optional)"
+  config_param :retry_emit_limit, :integer, :default => nil,
+               :desc => "How long to stop event consuming when BufferQueueLimitError happens. Wait retry_emit_limit x 1s. The default is waiting until BufferQueueLimitError is resolved"
+  config_param :use_record_time, :bool, :default => false,
+               :desc => "Replace message timestamp with contents of 'time' field."
+  config_param :time_format, :string, :default => nil,
+               :desc => "Time format to be used to parse 'time' filed."
+
+  config_param :retry_wait_seconds, :integer, :default => 30
+  # Kafka consumer options
+  config_param :max_bytes, :integer, :default => 1048576,
+               :desc => "Maximum number of bytes to fetch."
+  config_param :max_wait_time, :integer, :default => nil,
+               :desc => "How long to block until the server sends us data."
+  config_param :min_bytes, :integer, :default => nil,
+               :desc => "Smallest amount of data the server should send us."
+  config_param :session_timeout, :integer, :default => nil,
+               :desc => "The number of seconds after which, if a client hasn't contacted the Kafka cluster"
+  config_param :offset_commit_interval, :integer, :default => nil,
+               :desc => "The interval between offset commits, in seconds"
+  config_param :offset_commit_threshold, :integer, :default => nil,
+               :desc => "The number of messages that can be processed before their offsets are committed"
+  config_param :start_from_beginning, :bool, :default => true,
+               :desc => "Whether to start from the beginning of the topic or just subscribe to new messages being produced"
+
+  include Fluent::KafkaPluginUtil::SSLSettings
+  include Fluent::KafkaPluginUtil::SaslSettings
+
+  class ForShutdown < StandardError
+  end
+
+  BufferError = if defined?(Fluent::Plugin::Buffer::BufferOverflowError)
+                  Fluent::Plugin::Buffer::BufferOverflowError
+                else
+                  Fluent::BufferQueueLimitError
+                end
+
+  unless method_defined?(:router)
+    define_method("router") { Fluent::Engine }
+  end
+
+  def initialize
+    super
+    require 'kafka'
+
+    @time_parser = nil
+  end
+
+  def _config_to_array(config)
+    config_array = config.split(',').map {|k| k.strip }
+    if config_array.empty?
+      raise Fluent::ConfigError, "kafka_group: '#{config}' is a required parameter"
+    end
+    config_array
+  end
+
+  private :_config_to_array
+
+  def configure(conf)
+    super
+
+    $log.info "Will watch for topics #{@topics} at brokers " \
+              "#{@brokers} and '#{@consumer_group}' group"
+
+    @topics = _config_to_array(@topics)
+
+    if conf['max_wait_ms']
+      log.warn "'max_wait_ms' parameter is deprecated. Use second unit 'max_wait_time' instead"
+      @max_wait_time = conf['max_wait_ms'].to_i / 1000
+    end
+
+    @parser_proc = setup_parser
+
+    @consumer_opts = {:group_id => @consumer_group}
+    @consumer_opts[:session_timeout] = @session_timeout if @session_timeout
+    @consumer_opts[:offset_commit_interval] = @offset_commit_interval if @offset_commit_interval
+    @consumer_opts[:offset_commit_threshold] = @offset_commit_threshold if @offset_commit_threshold
+
+    @fetch_opts = {}
+    @fetch_opts[:max_wait_time] = @max_wait_time if @max_wait_time
+    @fetch_opts[:min_bytes] = @min_bytes if @min_bytes
+
+    if @use_record_time and @time_format
+      @time_parser = Fluent::TextParser::TimeParser.new(@time_format)
+    end
+  end
+
+  def setup_parser
+    case @format
+    when 'json'
+      require 'yajl'
+      Proc.new { |msg| Yajl::Parser.parse(msg.value) }
+    when 'ltsv'
+      require 'ltsv'
+      Proc.new { |msg| LTSV.parse(msg.value, {:symbolize_keys => false}).first }
+    when 'msgpack'
+      require 'msgpack'
+      Proc.new { |msg| MessagePack.unpack(msg.value) }
+    when 'text'
+      Proc.new { |msg| {@message_key => msg.value} }
+    end
+  end
+
+  def start
+    super
+   
+    @kafka = Kafka.new(seed_brokers: @brokers,
+                       ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
+                       ssl_client_cert: read_ssl_file(@ssl_client_cert),
+                       ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key),
+                       sasl_gssapi_principal: @principal, sasl_gssapi_keytab: @keytab)
+    @consumer = setup_consumer
+    @thread = Thread.new(&method(:run))
+  end
+
+  def shutdown
+    # This nil assignment should be guarded by mutex in multithread programming manner.
+    # But the situation is very low contention, so we don't use mutex for now.
+    # If the problem happens, we will add a guard for consumer.
+    consumer = @consumer
+    @consumer = nil
+    consumer.stop
+
+    @thread.join
+    @kafka.close
+    super
+  end
+
+  def setup_consumer
+    consumer = @kafka.consumer(@consumer_opts)
+    @topics.each { |topic|
+      consumer.subscribe(topic, start_from_beginning: @start_from_beginning, max_bytes_per_partition: @max_bytes)
+    }
+    consumer
+  end
+  
+  def reconnect_consumer
+    log.warn "Stopping Consumer"
+    consumer = @consumer
+    @consumer = nil
+    consumer.stop
+    log.warn "Could not connect to broker. Next retry will be in #{@retry_wait_seconds} seconds"
+    sleep @retry_wait_seconds
+    @consumer = setup_consumer
+    log.warn "Re-starting consumer #{Time.now.to_s}"
+  rescue =>e
+    log.error "unexpected error during re-starting consumer object access", :error => e.to_s
+    log.error_backtrace
+  end
+  
+  def run
+    while @consumer
+      begin
+        @consumer.each_batch(@fetch_opts) { |batch|
+          es = Fluent::MultiEventStream.new
+          tag = batch.topic
+          tag = @add_prefix + "." + tag if @add_prefix
+          tag = tag + "." + @add_suffix if @add_suffix
+
+          batch.messages.each { |msg|
+            begin
+              record = @parser_proc.call(msg)
+              if @use_record_time
+                if @time_format
+                  record_time = @time_parser.parse(record['time'])
+                else
+                  record_time = record['time']
+                end
+              else
+                record_time = Fluent::Engine.now
+              end
+              es.add(record_time, record)
+            rescue => e
+              log.warn "parser error in #{batch.topic}/#{batch.partition}", :error => e.to_s, :value => msg.value, :offset => msg.offset
+              log.debug_backtrace
+            end
+          }
+
+          unless es.empty?
+            emit_events(tag, es)
+          end
+        }
+      rescue ForShutdown
+      rescue => e
+        log.error "unexpected error during consuming events from kafka. Re-fetch events.", :error => e.to_s
+        log.error_backtrace
+        reconnect_consumer
+      end
+    end
+  rescue => e
+    log.error "unexpected error during consumer object access", :error => e.to_s
+    log.error_backtrace
+  end
+
+  def emit_events(tag, es)
+    retries = 0
+    begin
+      router.emit_stream(tag, es)
+    rescue BufferError
+      raise ForShutdown if @consumer.nil?
+
+      if @retry_emit_limit.nil?
+        sleep 1
+        retry
+      end
+
+      if retries < @retry_emit_limit
+        retries += 1
+        sleep 1
+        retry
+      else
+        raise RuntimeError, "Exceeds retry_emit_limit"
+      end
+    end
+  end
+end

data/lib/fluent/plugin/kafka_plugin_util.rb

@@ -0,0 +1,44 @@
+module Fluent
+  module KafkaPluginUtil
+    module SSLSettings
+      def self.included(klass)
+        klass.instance_eval {
+          # https://github.com/zendesk/ruby-kafka#encryption-and-authentication-using-ssl
+          config_param :ssl_ca_cert, :array, :value_type => :string, :default => nil,
+                       :desc => "a PEM encoded CA cert to use with and SSL connection."
+          config_param :ssl_client_cert, :string, :default => nil,
+                       :desc => "a PEM encoded client cert to use with and SSL connection. Must be used in combination with ssl_client_cert_key."
+          config_param :ssl_client_cert_key, :string, :default => nil,
+                       :desc => "a PEM encoded client cert key to use with and SSL connection. Must be used in combination with ssl_client_cert."
+        }
+      end
+
+      def read_ssl_file(path)
+        return nil if path.nil?
+
+        if path.is_a?(Array)
+          path.map { |fp| File.read(fp) }
+        else
+          File.read(path)
+        end
+      end
+    end
+
+    module SaslSettings
+      def self.included(klass)
+        klass.instance_eval {
+          config_param :principal, :string, :default => nil,
+                       :desc => "a Kerberos principal to use with SASL authentication (GSSAPI)."
+          config_param :keytab, :string, :default => nil,
+                       :desc => "a filepath to Kerberos keytab. Must be used with principal."
+          config_param :username, :string, :default => nil,
+                       :desc => "a username when using PLAIN/SCRAM SASL authentication"
+          config_param :password, :string, :default => nil,
+                       :desc => "a password when using PLAIN/SCRAM SASL authentication"
+          config_param :scram_mechanism, :string, :default => nil,
+                       :desc => "if set, use SCRAM authentication with specified mechanism. When unset, default to PLAIN authentication"
+        }
+      end
+    end
+  end
+end