fluentd 0.14.1 → 0.14.2

data/lib/fluent/plugin/in_syslog.rb

@@ -17,13 +17,15 @@
 require 'cool.io'
 require 'yajl'
 
-require 'fluent/input'
+require 'fluent/plugin/input'
 require 'fluent/config/error'
-require 'fluent/parser'
+require 'fluent/plugin/parser'
 
-module Fluent
+module Fluent::Plugin
   class SyslogInput < Input
-    Plugin.register_input('syslog', self)
+    Fluent::Plugin.register_input('syslog', self)
+
+    helpers :parser, :event_loop
 
     SYSLOG_REGEXP = /^\<([0-9]+)\>(.*)/
 
@@ -92,6 +94,7 @@ module Fluent
     desc 'Specify key of source host when include_source_host is true.'
     config_param :source_host_key, :string, default: 'source_host'.freeze
     config_param :blocking_timeout, :time, default: 0.5
+    config_param :message_length_limit, :size, default: 2048
 
     def configure(conf)
       super
@@ -99,14 +102,13 @@ module Fluent
       @use_default = false
 
       if conf.has_key?('format')
-        @parser = Plugin.new_parser(conf['format'])
-        @parser.configure(conf)
+        @parser = parser_create(usage: 'syslog_input', type: conf['format'], conf: conf)
       else
         conf['with_priority'] = true
-        @parser = TextParser::SyslogParser.new
-        @parser.configure(conf)
+        @parser = parser_create(usage: 'syslog_input', type: 'syslog', conf: conf)
         @use_default = true
       end
+      @_event_loop_run_timeout = @blocking_timeout
     end
 
     def start
@@ -118,29 +120,16 @@ module Fluent
                    method(:receive_data_parser)
                  end
 
-      @loop = Coolio::Loop.new
       @handler = listen(callback)
-      @loop.attach(@handler)
-
-      @thread = Thread.new(&method(:run))
+      event_loop_attach(@handler)
     end
 
     def shutdown
-      @loop.watchers.each {|w| w.detach }
-      @loop.stop
       @handler.close
-      @thread.join
 
       super
     end
 
-    def run
-      @loop.run(@blocking_timeout)
-    rescue
-      log.error "unexpected error", error: $!.to_s
-      log.error_backtrace
-    end
-
     private
 
     def receive_data_parser(data, addr)
@@ -193,11 +182,11 @@ module Fluent
       client = ServerEngine::SocketManager::Client.new(socket_manager_path)
       if @protocol_type == :udp
         @usock = client.listen_udp(@bind, @port)
-        SocketUtil::UdpHandler.new(@usock, log, 2048, callback)
+        Fluent::SocketUtil::UdpHandler.new(@usock, log, @message_length_limit, callback)
       else
         # syslog family add "\n" to each message and this seems only way to split messages in tcp stream
         lsock = client.listen_tcp(@bind, @port)
-        Coolio::TCPServer.new(lsock, nil, SocketUtil::TcpHandler, log, "\n", callback)
+        Coolio::TCPServer.new(lsock, nil, Fluent::SocketUtil::TcpHandler, log, "\n", callback)
       end
     end
 

data/lib/fluent/plugin/in_tail.rb

@@ -56,6 +56,9 @@ module Fluent
     config_param :pos_file, :string, default: nil
     desc 'Start to read the logs from the head of file, not bottom.'
     config_param :read_from_head, :bool, default: false
+    # When the program deletes log file and re-creates log file with same filename after passed refresh_interval,
+    # in_tail may raise a pos_file related error. This is a known issue but there is no such program on production.
+    # If we find such program / application, we will fix the problem.
     desc 'The interval of refreshing the list of watch file.'
     config_param :refresh_interval, :time, default: 60
     desc 'The number of reading lines at each IO.'

data/lib/fluent/plugin/out_buffered_stdout.rb

@@ -20,14 +20,20 @@ module Fluent::Plugin
   class BufferedStdoutOutput < Output
     Fluent::Plugin.register_output('buffered_stdout', self)
 
-    desc 'Output format.(json,hash)'
-    config_param :output_type, default: 'json'
+    helpers :formatter, :inject, :compat_parameters
+
     config_section :buffer do
       config_set_default :chunk_keys, ['tag']
       config_set_default :flush_at_shutdown, true
       config_set_default :chunk_limit_size, 10 * 1024
     end
 
+    DEFAULT_FORMAT_TYPE = 'json'
+
+    config_section :format do
+      config_set_default :@type, DEFAULT_FORMAT_TYPE
+    end
+
     attr_accessor :delayed
 
     def initialize
@@ -40,9 +46,12 @@ module Fluent::Plugin
     end
 
     def configure(conf)
+      if conf['output_type'] && !conf['format']
+        conf['format'] = conf['output_type']
+      end
+      compat_parameters_convert(conf, :inject, :formatter)
       super
-      @formatter = Fluent::Plugin.new_formatter(@output_type, parent: self)
-      @formatter.configure(conf)
+      @formatter = formatter_create(conf: conf.elements('format').first, default_type: DEFAULT_FORMAT_TYPE)
     end
 
     def write(chunk)
@@ -54,6 +63,7 @@ module Fluent::Plugin
     end
 
     def format(tag, time, record)
+      record = inject_values_to_record(tag, time, record)
       "#{Time.at(time).localtime} #{tag}: #{@formatter.format(tag, time, record).chomp}\n"
     end
   end

data/lib/fluent/plugin/out_exec.rb

@@ -39,9 +39,9 @@ module Fluent::Plugin
     config_param :time_key, :string, default: nil
     desc 'The format for event time used when the time_key parameter is specified. The default is UNIX time (integer).'
     config_param :time_format, :string, default: nil
-    desc "The format used to map the incoming events to the program input. (#{ExecUtil::SUPPORTED_FORMAT.keys.join(',')})"
+    desc "The format used to map the incoming events to the program input. (#{Fluent::ExecUtil::SUPPORTED_FORMAT.keys.join(',')})"
     config_param :format, default: :tsv, skip_accessor: true do |val|
-      f = ExecUtil::SUPPORTED_FORMAT[val]
+      f = Fluent::ExecUtil::SUPPORTED_FORMAT[val]
       raise ConfigError, "Unsupported format '#{val}'" unless f
       f
     end
@@ -53,6 +53,8 @@ module Fluent::Plugin
     end
 
     def configure(conf)
+      compat_parameters_convert(conf, :buffer, default_chunk_key: 'time')
+
       super
 
       @formatter = case @format
@@ -60,11 +62,11 @@ module Fluent::Plugin
                      if @keys.empty?
                        raise Fluent::ConfigError, "keys option is required on exec output for tsv format"
                      end
-                     ExecUtil::TSVFormatter.new(@keys)
+                     Fluent::ExecUtil::TSVFormatter.new(@keys)
                    when :json
-                     ExecUtil::JSONFormatter.new
+                     Fluent::ExecUtil::JSONFormatter.new
                    when :msgpack
-                     ExecUtil::MessagePackFormatter.new
+                     Fluent::ExecUtil::MessagePackFormatter.new
                    end
 
       if @time_key

data/lib/fluent/plugin/out_exec_filter.rb

@@ -38,9 +38,9 @@ module Fluent
     config_param :remove_prefix, :string, default: nil
     config_param :add_prefix, :string, default: nil
 
-    desc "The format used to map the incoming event to the program input.(#{ExecUtil::SUPPORTED_FORMAT.keys.join(',')})"
+    desc "The format used to map the incoming event to the program input.(#{Fluent::ExecUtil::SUPPORTED_FORMAT.keys.join(',')})"
     config_param :in_format, default: :tsv do |val|
-      f = ExecUtil::SUPPORTED_FORMAT[val]
+      f = Fluent::ExecUtil::SUPPORTED_FORMAT[val]
       raise ConfigError, "Unsupported in_format '#{val}'" unless f
       f
     end
@@ -55,9 +55,9 @@ module Fluent
     desc 'The format for event time used when the in_time_key parameter is specified.(Defauls is UNIX time)'
     config_param :in_time_format, default: nil
 
-    desc "The format used to process the program output.(#{ExecUtil::SUPPORTED_FORMAT.keys.join(',')})"
+    desc "The format used to process the program output.(#{Fluent::ExecUtil::SUPPORTED_FORMAT.keys.join(',')})"
     config_param :out_format, default: :tsv do |val|
-      f = ExecUtil::SUPPORTED_FORMAT[val]
+      f = Fluent::ExecUtil::SUPPORTED_FORMAT[val]
       raise ConfigError, "Unsupported out_format '#{val}'" unless f
       f
     end
@@ -169,11 +169,11 @@ module Fluent
         if @in_keys.empty?
           raise ConfigError, "in_keys option is required on exec_filter output for tsv in_format"
         end
-        @formatter = ExecUtil::TSVFormatter.new(@in_keys)
+        @formatter = Fluent::ExecUtil::TSVFormatter.new(@in_keys)
       when :json
-        @formatter = ExecUtil::JSONFormatter.new
+        @formatter = Fluent::ExecUtil::JSONFormatter.new
       when :msgpack
-        @formatter = ExecUtil::MessagePackFormatter.new
+        @formatter = Fluent::ExecUtil::MessagePackFormatter.new
       end
 
       case @out_format
@@ -181,11 +181,11 @@ module Fluent
         if @out_keys.empty?
           raise ConfigError, "out_keys option is required on exec_filter output for tsv in_format"
         end
-        @parser = ExecUtil::TSVParser.new(@out_keys, method(:on_message))
+        @parser = Fluent::ExecUtil::TSVParser.new(@out_keys, method(:on_message))
       when :json
-        @parser = ExecUtil::JSONParser.new(method(:on_message))
+        @parser = Fluent::ExecUtil::JSONParser.new(method(:on_message))
       when :msgpack
-        @parser = ExecUtil::MessagePackParser.new(method(:on_message))
+        @parser = Fluent::ExecUtil::MessagePackParser.new(method(:on_message))
       end
 
       @respawns = if @child_respawn.nil? or @child_respawn == 'none' or @child_respawn == '0'

data/lib/fluent/plugin/out_file.rb

@@ -103,9 +103,7 @@ module Fluent
       @formatter.configure(conf)
 
       if @symlink_path && @buffer.respond_to?(:path)
-        (class << @buffer; self; end).module_eval do
-          prepend SymlinkBufferMixin
-        end
+        @buffer.extend SymlinkBufferMixin
         @buffer.symlink_path = @symlink_path
       end
 

data/lib/fluent/plugin/out_forward.rb

@@ -78,8 +78,6 @@ module Fluent
     desc 'Use the "Phi accrual failure detector" to detect server failure.'
     config_param :phi_failure_detector, :bool, default: true
 
-    # if any options added that requires extended forward api, fix @extend_internal_protocol
-
     desc 'Change the protocol to at-least-once.'
     config_param :require_ack_response, :bool, default: false  # require in_forward to respond with ack
     desc 'This option is used when require_ack_response is true.'
@@ -95,8 +93,6 @@ module Fluent
     config_param :port, :integer, default: LISTEN_PORT
     config_param :host, :string, default: nil
 
-    attr_accessor :extend_internal_protocol
-
     def configure(conf)
       super
 
@@ -112,13 +108,6 @@ module Fluent
 
       recover_sample_size = @recover_wait / @heartbeat_interval
 
-      # add options here if any options addes which uses extended protocol
-      @extend_internal_protocol = if @require_ack_response
-                                    true
-                                  else
-                                    false
-                                  end
-
       if @dns_round_robin
         if @heartbeat_type == :udp
           raise ConfigError, "forward output heartbeat type must be 'tcp' or 'none' to use dns_round_robin option"
@@ -276,16 +265,10 @@ module Fluent
       @weight_array = weight_array
     end
 
-    # MessagePack FixArray length = 3 (if @extend_internal_protocol)
-    #                             = 2 (else)
-    FORWARD_HEADER = [0x92].pack('C').freeze
-    FORWARD_HEADER_EXT = [0x93].pack('C').freeze
+    # MessagePack FixArray length is 3
+    FORWARD_HEADER = [0x93].pack('C').freeze
     def forward_header
-      if @extend_internal_protocol
-        FORWARD_HEADER_EXT
-      else
-        FORWARD_HEADER
-      end
+      FORWARD_HEADER
     end
 
     #FORWARD_TCP_HEARTBEAT_DATA = FORWARD_HEADER + ''.to_msgpack + [].to_msgpack
@@ -314,7 +297,7 @@ module Fluent
         opt = [@send_timeout.to_i, 0].pack('L!L!')  # struct timeval
         sock.setsockopt(Socket::SOL_SOCKET, Socket::SO_SNDTIMEO, opt)
 
-        # beginArray(2)
+        # beginArray(3)
         sock.write forward_header
 
         # writeRaw(tag)
@@ -336,48 +319,46 @@ module Fluent
         # writeRawBody(packed_es)
         chunk.write_to(sock)
 
-        if @extend_internal_protocol
-          option = {}
-          option['chunk'] = Base64.encode64(chunk.unique_id) if @require_ack_response
-          sock.write option.to_msgpack
-
-          if @require_ack_response && @ack_response_timeout > 0
-            # Waiting for a response here results in a decrease of throughput because a chunk queue is locked.
-            # To avoid a decrease of troughput, it is necessary to prepare a list of chunks that wait for responses
-            # and process them asynchronously.
-            if IO.select([sock], nil, nil, @ack_response_timeout)
-              raw_data = sock.recv(1024)
-
-              # When connection is closed by remote host, socket is ready to read and #recv returns an empty string that means EOF.
-              # If this happens we assume the data wasn't delivered and retry it.
-              if raw_data.empty?
-                @log.warn "node #{node.host}:#{node.port} closed the connection. regard it as unavailable."
-                node.disable!
-                raise ForwardOutputConnectionClosedError, "node #{node.host}:#{node.port} closed connection"
-              else
-                # Serialization type of the response is same as sent data.
-                res = MessagePack.unpack(raw_data)
-
-                if res['ack'] != option['chunk']
-                  # Some errors may have occured when ack and chunk id is different, so send the chunk again.
-                  raise ForwardOutputResponseError, "ack in response and chunk id in sent data are different"
-                end
-              end
-
-            else
-              # IO.select returns nil on timeout.
-              # There are 2 types of cases when no response has been received:
-              # (1) the node does not support sending responses
-              # (2) the node does support sending response but responses have not arrived for some reasons.
-              @log.warn "no response from #{node.host}:#{node.port}. regard it as unavailable."
+        option = { 'size' => chunk.size_of_events }
+        option['chunk'] = Base64.encode64(chunk.unique_id) if @require_ack_response
+        sock.write option.to_msgpack
+
+        if @require_ack_response && @ack_response_timeout > 0
+          # Waiting for a response here results in a decrease of throughput because a chunk queue is locked.
+          # To avoid a decrease of troughput, it is necessary to prepare a list of chunks that wait for responses
+          # and process them asynchronously.
+          if IO.select([sock], nil, nil, @ack_response_timeout)
+            raw_data = sock.recv(1024)
+
+            # When connection is closed by remote host, socket is ready to read and #recv returns an empty string that means EOF.
+            # If this happens we assume the data wasn't delivered and retry it.
+            if raw_data.empty?
+              @log.warn "node #{node.host}:#{node.port} closed the connection. regard it as unavailable."
               node.disable!
-              raise ForwardOutputACKTimeoutError, "node #{node.host}:#{node.port} does not return ACK"
+              raise ForwardOutputConnectionClosedError, "node #{node.host}:#{node.port} closed connection"
+            else
+              # Serialization type of the response is same as sent data.
+              res = MessagePack.unpack(raw_data)
+
+              if res['ack'] != option['chunk']
+                # Some errors may have occured when ack and chunk id is different, so send the chunk again.
+                raise ForwardOutputResponseError, "ack in response and chunk id in sent data are different"
+              end
             end
+
+          else
+            # IO.select returns nil on timeout.
+            # There are 2 types of cases when no response has been received:
+            # (1) the node does not support sending responses
+            # (2) the node does support sending response but responses have not arrived for some reasons.
+            @log.warn "no response from #{node.host}:#{node.port}. regard it as unavailable."
+            node.disable!
+            raise ForwardOutputACKTimeoutError, "node #{node.host}:#{node.port} does not return ACK"
           end
         end
 
         node.heartbeat(false)
-        return res  # for test
+        res  # for test
       ensure
         sock.close_write
         sock.close

data/lib/fluent/plugin/out_stdout.rb

@@ -20,18 +20,27 @@ module Fluent::Plugin
   class StdoutOutput < Output
     Fluent::Plugin.register_output('stdout', self)
 
-    desc 'Output format.(json,hash)'
-    config_param :output_type, default: 'json'
+    helpers :inject, :formatter, :compat_parameters
+
+    DEFAULT_FORMAT_TYPE = 'json'
+
+    config_section :format do
+      config_set_default :@type, DEFAULT_FORMAT_TYPE
+    end
 
     def configure(conf)
+      if conf['output_type'] && !conf['format']
+        conf['format'] = conf['output_type']
+      end
+      compat_parameters_convert(conf, :inject, :formatter)
       super
-      @formatter = Fluent::Plugin.new_formatter(@output_type, parent: self)
-      @formatter.configure(conf)
+      @formatter = formatter_create(conf: conf.elements('format').first, default_type: DEFAULT_FORMAT_TYPE)
     end
 
     def process(tag, es)
       es.each {|time,record|
-        $log.write "#{Time.at(time).localtime} #{tag}: #{@formatter.format(tag, time, record).chomp}\n"
+        r = inject_values_to_record(tag, time, record)
+        $log.write "#{Time.at(time).localtime} #{tag}: #{@formatter.format(tag, time, r).chomp}\n"
       }
       $log.flush
     end

data/lib/fluent/plugin/out_stream.rb

@@ -25,7 +25,10 @@ module Fluent
   class StreamOutput < BufferedOutput
     config_param :send_timeout, :time, default: 60
 
+    helpers :compat_parameters
+
     def configure(conf)
+      compat_parameters_convert(conf, :buffer)
       super
     end
 

data/lib/fluent/plugin/output.rb

@@ -627,6 +627,18 @@ module Fluent
         end
       end
 
+      FORMAT_MSGPACK_STREAM = ->(e){ e.to_msgpack_stream }
+      FORMAT_MSGPACK_STREAM_TIME_INT = ->(e){ e.to_msgpack_stream(time_int: true) }
+
+      # metadata_and_data is a Hash of:
+      #  (standard format) metadata => event stream
+      #  (custom format)   metadata => array of formatted event
+      # For standard format, formatting should be done for whole event stream, but
+      #   "whole event stream" may be a split of "es" here when it's bigger than chunk_limit_size.
+      #   `@buffer.write` will do this splitting.
+      # For custom format, formatting will be done here. Custom formatting always requires
+      #   iteration of event stream, and it should be done just once even if total event stream size
+      #   is biggar than chunk_limit_size because of performance.
       def handle_stream_with_custom_format(tag, es, enqueue: false)
         meta_and_data = {}
         records = 0
@@ -637,13 +649,14 @@ module Fluent
           records += 1
         end
         write_guard do
-          @buffer.write(meta_and_data, bulk: false, enqueue: enqueue)
+          @buffer.write(meta_and_data, enqueue: enqueue)
         end
         @counters_monitor.synchronize{ @emit_records += records }
         true
       end
 
       def handle_stream_with_standard_format(tag, es, enqueue: false)
+        format_proc = @time_as_integer ? FORMAT_MSGPACK_STREAM_TIME_INT : FORMAT_MSGPACK_STREAM
         meta_and_data = {}
         records = 0
         es.each do |time, record|
@@ -652,41 +665,37 @@ module Fluent
           meta_and_data[meta].add(time, record)
           records += 1
         end
-        meta_and_data_bulk = {}
-        meta_and_data.each_pair do |meta, m_es|
-          meta_and_data_bulk[meta] = [m_es.to_msgpack_stream(time_int: @time_as_integer), m_es.size]
-        end
         write_guard do
-          @buffer.write(meta_and_data_bulk, bulk: true, enqueue: enqueue)
+          @buffer.write(meta_and_data, format: format_proc, enqueue: enqueue)
         end
         @counters_monitor.synchronize{ @emit_records += records }
         true
       end
 
       def handle_stream_simple(tag, es, enqueue: false)
+        format_proc = nil
         meta = metadata((@chunk_key_tag ? tag : nil), nil, nil)
         records = es.size
         if @custom_format
           records = 0
-          es_size = 0
-          es_bulk = ''
-          es.each do |time,record|
-            es_bulk << format(tag, time, record)
-            es_size += 1
+          data = []
+          es.each do |time, record|
+            data << format(tag, time, record)
             records += 1
           end
         else
-          es_size = es.size
-          es_bulk = es.to_msgpack_stream(time_int: @time_as_integer)
+          format_proc = @time_as_integer ? FORMAT_MSGPACK_STREAM_TIME_INT : FORMAT_MSGPACK_STREAM
+          data = es
         end
         write_guard do
-          @buffer.write({meta => [es_bulk, es_size]}, bulk: true, enqueue: enqueue)
+          @buffer.write({meta => data}, format: format_proc, enqueue: enqueue)
         end
         @counters_monitor.synchronize{ @emit_records += records }
         true
       end
 
       def commit_write(chunk_id, delayed: @delayed_commit, secondary: false)
+        log.trace "committing write operation to a chunk", chunk: dump_unique_id_hex(chunk_id), delayed: delayed
         if delayed
           @dequeued_chunks_mutex.synchronize do
             @dequeued_chunks.delete_if{ |info| info.chunk_id == chunk_id }
@@ -762,6 +771,8 @@ module Fluent
         chunk = @buffer.dequeue_chunk
         return unless chunk
 
+        log.debug "trying flush for a chunk", chunk: dump_unique_id_hex(chunk.unique_id)
+
         output = self
         using_secondary = false
         if @retry_mutex.synchronize{ @retry && @retry.secondary? }
@@ -775,6 +786,7 @@ module Fluent
 
         begin
           if output.delayed_commit
+            log.trace "executing delayed write and commit", chunk: dump_unique_id_hex(chunk.unique_id)
             @counters_monitor.synchronize{ @write_count += 1 }
             output.try_write(chunk)
             @dequeued_chunks_mutex.synchronize do
@@ -783,9 +795,14 @@ module Fluent
             end
           else # output plugin without delayed purge
             chunk_id = chunk.unique_id
+            dump_chunk_id = dump_unique_id_hex(chunk_id)
+            log.trace "adding write count", instance: self.object_id
             @counters_monitor.synchronize{ @write_count += 1 }
+            log.trace "executing sync write", chunk: dump_chunk_id
             output.write(chunk)
+            log.trace "write operation done, committing", chunk: dump_chunk_id
             commit_write(chunk_id, secondary: using_secondary)
+            log.trace "done to commit a chunk", chunk: dump_chunk_id
           end
         rescue => e
           log.debug "taking back chunk for errors.", plugin_id: plugin_id, chunk: dump_unique_id_hex(chunk.unique_id)