fluentd 0.14.10-x86-mingw32 → 0.14.11-x86-mingw32

data/lib/fluent/plugin/in_monitor_agent.rb

@@ -35,6 +35,7 @@ module Fluent::Plugin
     config_param :tag, :string, default: nil
     config_param :emit_interval, :time, default: 60
     config_param :include_config, :bool, default: true
+    config_param :include_retry, :bool, default: true
 
     class MonitorServlet < WEBrick::HTTPServlet::AbstractServlet
       def initialize(server, agent)
@@ -78,7 +79,7 @@ module Fluent::Plugin
 
         # if ?debug=1 is set, set :with_debug_info for get_monitor_info
         # and :pretty_json for render_json_error
-        opts = {with_config: @agent.include_config}
+        opts = {with_config: @agent.include_config, with_retry: @agent.include_retry}
         if s = qs['debug'] and s[0]
           opts[:with_debug_info] = true
           opts[:pretty_json] = true
@@ -88,6 +89,10 @@ module Fluent::Plugin
           opts[:with_config] = Fluent::Config.bool_value(with_config)
         end
 
+        if with_retry = get_search_parameter(qs, 'with_retry'.freeze)
+          opts[:with_retry] = Fluent::Config.bool_value(with_retry)
+        end
+
         if tag = get_search_parameter(qs, 'tag'.freeze)
           # ?tag= to search an output plugin by match pattern
           if obj = @agent.plugin_info_by_tag(tag, opts)
@@ -231,7 +236,7 @@ module Fluent::Plugin
       if @tag
         log.debug "tag parameter is specified. Emit plugins info to '#{@tag}'"
 
-        opts = {with_config: false}
+        opts = {with_config: false, with_retry: false}
         timer_execute(:in_monitor_agent_emit, @emit_interval, repeat: true) {
           es = Fluent::MultiEventStream.new
           now = Fluent::Engine.now
@@ -346,6 +351,8 @@ module Fluent::Plugin
         end
       }
 
+      obj['retry'] = get_retry_info(pe.retry) if opts[:with_retry] and pe.instance_variable_defined?(:@retry)
+
       # include all instance variables if :with_debug_info is set
       if opts[:with_debug_info]
         iv = {}
@@ -362,6 +369,24 @@ module Fluent::Plugin
       obj
     end
 
+    RETRY_INFO = {
+        'start' => '@start',
+        'steps' => '@steps',
+        'next_time' => '@next_time',
+    }
+
+    def get_retry_info(pe_retry)
+      retry_variables = {}
+
+      if pe_retry
+        RETRY_INFO.each_pair { |key, param|
+          retry_variables[key] = pe_retry.instance_variable_get(param)
+        }
+      end
+
+      retry_variables
+    end
+
     def plugin_category(pe)
       case pe
       when Fluent::Plugin::Input

data/lib/fluent/plugin/in_syslog.rb

@@ -14,18 +14,17 @@
 #    limitations under the License.
 #
 
-require 'cool.io'
-require 'yajl'
-
 require 'fluent/plugin/input'
 require 'fluent/config/error'
 require 'fluent/plugin/parser'
 
+require 'yajl'
+
 module Fluent::Plugin
   class SyslogInput < Input
     Fluent::Plugin.register_input('syslog', self)
 
-    helpers :parser, :compat_parameters, :event_loop
+    helpers :parser, :compat_parameters, :server
 
     DEFAULT_PARSER = 'syslog'
     SYSLOG_REGEXP = /^\<([0-9]+)\>(.*)/
@@ -68,11 +67,6 @@ module Fluent::Plugin
       7  => 'debug'
     }
 
-    def initialize
-      super
-      require 'fluent/plugin/socket_util'
-    end
-
     desc 'The port to listen to.'
     config_param :port, :integer, default: 5140
     desc 'The bind address to listen to.'
@@ -81,14 +75,22 @@ module Fluent::Plugin
     config_param :tag, :string
     desc 'The transport protocol used to receive logs.(udp, tcp)'
     config_param :protocol_type, :enum, list: [:tcp, :udp], default: :udp
+
     desc 'If true, add source host to event record.'
-    config_param :include_source_host, :bool, default: false
+    config_param :include_source_host, :bool, default: false, deprecated: 'use "source_hostname_key" or "source_address_key" instead.'
     desc 'Specify key of source host when include_source_host is true.'
     config_param :source_host_key, :string, default: 'source_host'.freeze
+
+    desc 'The field name of hostname of sender.'
+    config_param :source_hostname_key, :string, default: nil
+    desc 'The field name of source address of sender.'
+    config_param :source_address_key, :string, default: nil
+
     desc 'The field name of the priority.'
     config_param :priority_key, :string, default: nil
     desc 'The field name of the facility.'
     config_param :facility_key, :string, default: nil
+
     config_param :blocking_timeout, :time, default: 0.5
     config_param :message_length_limit, :size, default: 2048
 
@@ -107,25 +109,57 @@ module Fluent::Plugin
       @parser = parser_create
       @parser_parse_priority = @parser.respond_to?(:with_priority) && @parser.with_priority
 
+      if @include_source_host
+        if @source_address_key
+          raise Fluent::ConfigError, "specify either source_address_key or include_source_host"
+        end
+        @source_address_key = @source_host_key
+      end
+      @resolve_name = !!@source_hostname_key
+
       @_event_loop_run_timeout = @blocking_timeout
     end
 
     def start
       super
 
-      @handler = listen(method(:message_handler))
-      event_loop_attach(@handler)
+      log.info "listening syslog socket on #{@bind}:#{@port} with #{@protocol_type}"
+      case @protocol_type
+      when :udp then start_udp_server
+      when :tcp then start_tcp_server
+      else
+        raise "BUG: invalid protocol_type value:#{@protocol_type}"
+      end
     end
 
-    def shutdown
-      @handler.close
+    def start_udp_server
+      server_create_udp(:in_syslog_udp_server, @port, bind: @bind, max_bytes: @message_length_limit, resolve_name: @resolve_name) do |data, sock|
+        message_handler(data.chomp, sock)
+      end
+    end
 
-      super
+    def start_tcp_server
+      # syslog family add "\n" to each message and this seems only way to split messages in tcp stream
+      delimiter = "\n"
+      delimiter_size = delimiter.size
+      server_create_connection(:in_syslog_tcp_server, @port, bind: @bind, resolve_name: @resolve_name) do |conn|
+        buffer = ""
+        conn.data do |data|
+          buffer << data
+          pos = 0
+          while idx = buffer.index(delimiter, pos)
+            msg = buffer[pos...idx]
+            pos = idx + delimiter_size
+            message_handler(msg, conn)
+          end
+          buffer.slice!(0, pos) if pos > 0
+        end
+      end
     end
 
     private
 
-    def message_handler(data, addr)
+    def message_handler(data, sock)
       pri = nil
       text = data
       unless @parser_parse_priority
@@ -150,7 +184,8 @@ module Fluent::Plugin
 
         record[@priority_key] = priority if @priority_key
         record[@facility_key] = facility if @facility_key
-        record[@source_host_key] = addr[2] if @include_source_host
+        record[@source_address_key] = sock.remote_addr if @source_address_key
+        record[@source_hostname_key] = sock.remote_host if @source_hostname_key
 
         tag = "#{@tag}.#{facility}.#{priority}"
         emit(tag, time, record)
@@ -160,25 +195,6 @@ module Fluent::Plugin
       log.error_backtrace
     end
 
-    private
-
-    def listen(callback)
-      log.info "listening syslog socket on #{@bind}:#{@port} with #{@protocol_type}"
-      socket_manager_path = ENV['SERVERENGINE_SOCKETMANAGER_PATH']
-      if Fluent.windows?
-        socket_manager_path = socket_manager_path.to_i
-      end
-      client = ServerEngine::SocketManager::Client.new(socket_manager_path)
-      if @protocol_type == :udp
-        @usock = client.listen_udp(@bind, @port)
-        Fluent::SocketUtil::UdpHandler.new(@usock, log, @message_length_limit, callback)
-      else
-        # syslog family add "\n" to each message and this seems only way to split messages in tcp stream
-        lsock = client.listen_tcp(@bind, @port)
-        Coolio::TCPServer.new(lsock, nil, Fluent::SocketUtil::TcpHandler, log, "\n", callback)
-      end
-    end
-
     def emit(tag, time, record)
       router.emit(tag, time, record)
     rescue => e

data/lib/fluent/plugin/in_tail.rb

@@ -97,6 +97,7 @@ module Fluent::Plugin
         raise Fluent::ConfigError, "tail: 'path' parameter is required on tail input"
       end
 
+      # TODO: Use plugin_root_dir and storage plugin to store positions if available
       unless @pos_file
         $log.warn "'pos_file PATH' parameter is not set to a 'tail' source."
         $log.warn "this parameter is highly recommended to save the position to resume tailing."

data/lib/fluent/plugin/out_forward.rb

@@ -178,7 +178,7 @@ module Fluent::Plugin
       # But it should be overwritten by ack_response_timeout to rollback chunks after timeout
       if @ack_response_timeout && @delayed_commit_timeout != @ack_response_timeout
         log.info "delayed_commit_timeout is overwritten by ack_response_timeout"
-        @delayed_commit_timeout = @ack_response_timeout
+        @delayed_commit_timeout = @ack_response_timeout + 2 # minimum ack_reader IO.select interval is 1s
       end
 
       @rand_seed = Random.new.seed
@@ -214,22 +214,23 @@ module Fluent::Plugin
       select_a_healthy_node{|node| node.send_data(tag, chunk) }
     end
 
-    ACKWaitingSockInfo = Struct.new(:sock, :chunk_id, :node, :time, :timeout) do
+    ACKWaitingSockInfo = Struct.new(:sock, :chunk_id, :chunk_id_base64, :node, :time, :timeout) do
       def expired?(now)
         time + timeout < now
       end
     end
 
     def try_write(chunk)
+      log.trace "writing a chunk to destination", chunk_id: dump_unique_id_hex(chunk.unique_id)
       if chunk.empty?
         commit_write(chunk.unique_id)
         return
       end
       tag = chunk.metadata.tag
       sock, node = select_a_healthy_node{|n| n.send_data(tag, chunk) }
-      chunk_id = Base64.encode64(chunk.unique_id)
+      chunk_id_base64 = Base64.encode64(chunk.unique_id)
       current_time = Process.clock_gettime(PROCESS_CLOCK_ID)
-      info = ACKWaitingSockInfo.new(sock, chunk_id, node, current_time, @ack_response_timeout)
+      info = ACKWaitingSockInfo.new(sock, chunk.unique_id, chunk_id_base64, node, current_time, @ack_response_timeout)
       @sock_ack_waiting_mutex.synchronize do
         @sock_ack_waiting << info
       end
@@ -341,7 +342,7 @@ module Fluent::Plugin
       end
     end
 
-    # return chunk id when succeeded for tests
+    # return chunk id to be committed
     def read_ack_from_sock(sock, unpacker)
       begin
         raw_data = sock.recv(@read_length)
@@ -359,11 +360,14 @@ module Fluent::Plugin
       else
         unpacker.feed(raw_data)
         res = unpacker.read
-        if res['ack'] != info.chunk_id
+        log.trace "getting response from destination", host: info.node.host, port: info.node.port, chunk_id: dump_unique_id_hex(info.chunk_id), response: res
+        if res['ack'] != info.chunk_id_base64
           # Some errors may have occured when ack and chunk id is different, so send the chunk again.
-          log.warn "ack in response and chunk id in sent data are different", chunk_id: info.chunk_id, ack: res['ack']
+          log.warn "ack in response and chunk id in sent data are different", chunk_id: dump_unique_id_hex(info.chunk_id), ack: res['ack']
           rollback_write(info.chunk_id)
           return nil
+        else
+          log.trace "got a correct ack response", chunk_id: dump_unique_id_hex(info.chunk_id)
         end
         return info.chunk_id
       end
@@ -378,9 +382,9 @@ module Fluent::Plugin
 
     def ack_reader
       select_interval = if @delayed_commit_timeout > 3
-                          2
+                          1
                         else
-                          @delayed_commit_timeout / 2.0
+                          @delayed_commit_timeout / 3.0
                         end
 
       unpacker = Fluent::Engine.msgpack_unpacker
@@ -388,30 +392,36 @@ module Fluent::Plugin
       while thread_current_running?
         now = Process.clock_gettime(PROCESS_CLOCK_ID)
         sockets = []
-        @sock_ack_waiting_mutex.synchronize do
-          new_list = []
-          @sock_ack_waiting.each do |info|
-            if info.expired?(now)
-              # There are 2 types of cases when no response has been received from socket:
-              # (1) the node does not support sending responses
-              # (2) the node does support sending response but responses have not arrived for some reasons.
-              log.warn "no response from node. regard it as unavailable.", host: info.node.host, port: info.node.port
-              info.node.disable!
-              info.sock.close rescue nil
-              rollback_write(info.chunk_id)
-            else
-              sockets << info.sock
-              new_list << info
+        begin
+          @sock_ack_waiting_mutex.synchronize do
+            new_list = []
+            @sock_ack_waiting.each do |info|
+              if info.expired?(now)
+                # There are 2 types of cases when no response has been received from socket:
+                # (1) the node does not support sending responses
+                # (2) the node does support sending response but responses have not arrived for some reasons.
+                log.warn "no response from node. regard it as unavailable.", host: info.node.host, port: info.node.port
+                info.node.disable!
+                info.sock.close rescue nil
+                rollback_write(info.chunk_id)
+              else
+                sockets << info.sock
+                new_list << info
+              end
             end
+            @sock_ack_waiting = new_list
           end
-          @sock_ack_waiting = new_list
-        end
 
-        readable_sockets, _, _ = IO.select(sockets, nil, nil, select_interval)
-        next unless readable_sockets
+          readable_sockets, _, _ = IO.select(sockets, nil, nil, select_interval)
+          next unless readable_sockets
 
-        readable_sockets.each do |sock|
-          read_ack_from_sock(sock, unpacker)
+          readable_sockets.each do |sock|
+            chunk_id = read_ack_from_sock(sock, unpacker)
+            commit_write(chunk_id)
+          end
+        rescue => e
+          log.error "unexpected error while receiving ack", error: e
+          log.error_backtrace
         end
       end
     end

data/lib/fluent/plugin/output.rb

@@ -43,6 +43,8 @@ module Fluent
       PROCESS_CLOCK_ID = Process::CLOCK_MONOTONIC_RAW rescue Process::CLOCK_MONOTONIC
 
       config_param :time_as_integer, :bool, default: false
+      desc 'The threshold to show slow flush logs'
+      config_param :slow_flush_log_threshold, :float, default: 20.0
 
       # `<buffer>` and `<secondary>` sections are available only when '#format' and '#write' are implemented
       config_section :buffer, param_name: :buffer_config, init: true, required: false, multi: false, final: true do
@@ -985,6 +987,8 @@ module Fluent
         end
 
         begin
+          chunk_write_start = Process.clock_gettime(PROCESS_CLOCK_ID)
+
           if output.delayed_commit
             log.trace "executing delayed write and commit", chunk: dump_unique_id_hex(chunk.unique_id)
             @counters_monitor.synchronize{ @write_count += 1 }
@@ -992,14 +996,19 @@ module Fluent
               # delayed_commit_timeout for secondary is configured in <buffer> of primary (<secondary> don't get <buffer>)
               @dequeued_chunks << DequeuedChunkInfo.new(chunk.unique_id, Time.now, self.delayed_commit_timeout)
             end
+
             output.try_write(chunk)
+            check_slow_flush(chunk_write_start)
           else # output plugin without delayed purge
             chunk_id = chunk.unique_id
             dump_chunk_id = dump_unique_id_hex(chunk_id)
             log.trace "adding write count", instance: self.object_id
             @counters_monitor.synchronize{ @write_count += 1 }
             log.trace "executing sync write", chunk: dump_chunk_id
+
             output.write(chunk)
+            check_slow_flush(chunk_write_start)
+
             log.trace "write operation done, committing", chunk: dump_chunk_id
             commit_write(chunk_id, delayed: false, secondary: using_secondary)
             log.trace "done to commit a chunk", chunk: dump_chunk_id
@@ -1019,6 +1028,14 @@ module Fluent
         end
       end
 
+      def check_slow_flush(start)
+        elapsed_time = Process.clock_gettime(PROCESS_CLOCK_ID) - start
+        if elapsed_time > @slow_flush_log_threshold
+          log.warn "buffer flush took longer time than slow_flush_log_threshold:",
+                   elapsed_time: elapsed_time, slow_flush_log_threshold: @slow_flush_log_threshold, plugin_id: self.plugin_id
+        end
+      end
+
       def update_retry_state(chunk_id, using_secondary, error = nil)
         @retry_mutex.synchronize do
           @counters_monitor.synchronize{ @num_errors += 1 }

data/lib/fluent/plugin/storage_local.rb

@@ -29,10 +29,16 @@ module Fluent
       DEFAULT_FILE_MODE = 0644
 
       config_param :path, :string, default: nil
-      config_param :mode, :integer, default: DEFAULT_FILE_MODE
-      config_param :dir_mode, :integer, default: DEFAULT_DIR_MODE
+      config_param :mode, default: DEFAULT_FILE_MODE do |v|
+        v.to_i(8)
+      end
+      config_param :dir_mode, default: DEFAULT_DIR_MODE do |v|
+        v.to_i(8)
+      end
       config_param :pretty_print, :bool, default: false
 
+      attr_reader :store # for test
+
       def initialize
         super
         @store = {}
@@ -42,9 +48,13 @@ module Fluent
         super
 
         @on_memory = false
-        if !@path && !@_plugin_id_configured
+        if @path
+          # use it
+        elsif root_dir = owner.plugin_root_dir
+          @path = File.join(root_dir, 'storage.json')
+        else
           if @persistent
-            raise Fluent::ConfigError, "Plugin @id or path for <storage> required to save data"
+            raise Fluent::ConfigError, "Plugin @id or path for <storage> required when 'persistent' is true"
           else
             if @autosave
               log.warn "both of Plugin @id and path for <storage> are not specified. Using on-memory store."
@@ -53,18 +63,11 @@ module Fluent
             end
             @on_memory = true
           end
-        elsif @path
-          # ok
-        else # @_plugin_id_configured is true
-          log.warn "path for <storage> is not specified. Using on-memory store temporarily, but will use file store after support global storage path"
-          @on_memory = true
-          ## TODO: get process-wide directory for plugin storage, and generate path for this plugin storage instance
-          # path = 
         end
 
         if !@on_memory
           dir = File.dirname(@path)
-          FileUtils.mkdir_p(dir, mode: @dir_mode) unless File.exist?(dir)
+          FileUtils.mkdir_p(dir, mode: @dir_mode) unless Dir.exist?(dir)
           if File.exist?(@path)
             raise Fluent::ConfigError, "Plugin storage path '#{@path}' is not readable/writable" unless File.readable?(@path) && File.writable?(@path)
             begin
@@ -75,7 +78,7 @@ module Fluent
               raise Fluent::ConfigError, "Unexpected error: failed to read data from plugin storage file: '#{@path}'"
             end
           else
-            raise Fluent::ConfigError, "Directory is not writable for plugin storage file '#{dir}'" unless File.writable?(dir)
+            raise Fluent::ConfigError, "Directory is not writable for plugin storage file '#{@path}'" unless File.stat(dir).writable?
           end
         end
       end