fluentd 0.14.17-x64-mingw32 → 1.3.1-x64-mingw32

data/lib/fluent/plugin_helper/retry_state.rb

@@ -165,18 +165,26 @@ module Fluent
         end
 
         def calc_interval(num)
-          # make it infinite if calculated "interval" is too big
-          interval = @constant_factor.to_f * (@backoff_base ** (num - 1))
-          if interval.finite?
-            if @max_interval && interval > @max_interval
-              @max_interval
+          interval = raw_interval(num - 1)
+          if @max_interval && interval > @max_interval
+            @max_interval
+          else
+            if interval.finite?
+              interval
             else
+              # Calculate previous finite value to avoid inf related errors. If this re-computing is heavy, use cache.
+              until interval.finite?
+                num -= 1
+                interval = raw_interval(num - 1)
+              end
               interval
             end
-          else
-            interval
           end
         end
+
+        def raw_interval(num)
+          @constant_factor.to_f * (@backoff_base ** (num))
+        end
       end
 
       class PeriodicRetry < RetryStateMachine

data/lib/fluent/plugin_helper/server.rb

@@ -213,8 +213,10 @@ module Fluent
         socket_option_setter.call(sock)
         close_callback = ->(conn){ @_server_mutex.synchronize{ @_server_connections.delete(conn) } }
         server = Coolio::TCPServer.new(sock, nil, EventHandler::TCPServer, socket_option_setter, close_callback, @log, @under_plugin_development, block) do |conn|
-          @_server_mutex.synchronize do
-            @_server_connections << conn
+          unless conn.closing
+            @_server_mutex.synchronize do
+              @_server_connections << conn
+            end
           end
         end
         server.listen(backlog) if backlog
@@ -227,8 +229,10 @@ module Fluent
         socket_option_setter.call(sock)
         close_callback = ->(conn){ @_server_mutex.synchronize{ @_server_connections.delete(conn) } }
         server = Coolio::TCPServer.new(sock, nil, EventHandler::TLSServer, context, socket_option_setter, close_callback, @log, @under_plugin_development, block) do |conn|
-          @_server_mutex.synchronize do
-            @_server_connections << conn
+          unless conn.closing
+            @_server_mutex.synchronize do
+              @_server_connections << conn
+            end
           end
         end
         server.listen(backlog) if backlog
@@ -237,7 +241,7 @@ module Fluent
 
       SERVER_TRANSPORT_PARAMS = [
         :protocol, :version, :ciphers, :insecure,
-        :cert_path, :private_key_path, :private_key_passphrase,
+        :ca_path, :cert_path, :private_key_path, :private_key_passphrase, :client_cert_auth,
         :ca_cert_path, :ca_private_key_path, :ca_private_key_passphrase,
         :generate_private_key_length,
         :generate_cert_country, :generate_cert_state, :generate_cert_state,
@@ -271,9 +275,11 @@ module Fluent
           config_param :insecure, :bool, default: false
 
           # Cert signed by public CA
+          config_param :ca_path, :string, default: nil
           config_param :cert_path, :string, default: nil
           config_param :private_key_path, :string, default: nil
           config_param :private_key_passphrase, :string, default: nil, secret: true
+          config_param :client_cert_auth, :bool, default: false
 
           # Cert generated and signed by private CA Certificate
           config_param :ca_cert_path, :string, default: nil
@@ -370,6 +376,9 @@ module Fluent
         sock
       end
 
+      # Use string "?" for port, not integer or nil. "?" is clear than -1 or nil in the log.
+      PEERADDR_FAILED = ["?", "?", "name resolusion failed", "?"]
+
       class CallbackSocket
         def initialize(server_type, sock, enabled_events = [], close_socket: true)
           @server_type = server_type
@@ -399,6 +408,10 @@ module Fluent
           raise "not implemented here"
         end
 
+        def close_after_write_complete
+          @sock.close_after_write_complete = true
+        end
+
         def close
           @sock.close if @close_socket
         end
@@ -425,9 +438,14 @@ module Fluent
       end
 
       class TCPCallbackSocket < CallbackSocket
+        ENABLED_EVENTS = [:data, :write_complete, :close]
+
+        attr_accessor :buffer
+
         def initialize(sock)
-          super("tcp", sock, [:data, :write_complete, :close])
-          @peeraddr = @sock.peeraddr
+          super("tcp", sock, ENABLED_EVENTS)
+          @peeraddr = (@sock.peeraddr rescue PEERADDR_FAILED)
+          @buffer = ''
         end
 
         def write(data)
@@ -436,9 +454,14 @@ module Fluent
       end
 
       class TLSCallbackSocket < CallbackSocket
+        ENABLED_EVENTS = [:data, :write_complete, :close]
+
+        attr_accessor :buffer
+
         def initialize(sock)
-          super("tls", sock, [:data, :write_complete, :close])
-          @peeraddr = @sock.to_io.peeraddr
+          super("tls", sock, ENABLED_EVENTS)
+          @peeraddr = (@sock.to_io.peeraddr rescue PEERADDR_FAILED)
+          @buffer = ''
         end
 
         def write(data)
@@ -447,8 +470,10 @@ module Fluent
       end
 
       class UDPCallbackSocket < CallbackSocket
+        ENABLED_EVENTS = []
+
         def initialize(sock, peeraddr, **kwargs)
-          super("udp", sock, [], **kwargs)
+          super("udp", sock, ENABLED_EVENTS, **kwargs)
           @peeraddr = peeraddr
         end
 
@@ -471,6 +496,8 @@ module Fluent
 
       module EventHandler
         class UDPServer < Coolio::IO
+          attr_writer :close_after_write_complete # dummy for consistent method call in callbacks
+
           def initialize(sock, max_bytes, flags, close_socket, log, under_plugin_development, &callback)
             raise ArgumentError, "socket must be a UDPSocket: sock = #{sock}" unless sock.is_a?(UDPSocket)
 
@@ -521,6 +548,9 @@ module Fluent
         end
 
         class TCPServer < Coolio::TCPSocket
+          attr_reader :closing
+          attr_writer :close_after_write_complete
+
           def initialize(sock, socket_option_setter, close_callback, log, under_plugin_development, connect_callback)
             raise ArgumentError, "socket must be a TCPSocket: sock=#{sock}" unless sock.is_a?(TCPSocket)
 
@@ -537,6 +567,7 @@ module Fluent
             @close_callback = close_callback
 
             @callback_connection = nil
+            @close_after_write_complete = false
             @closing = false
 
             @mutex = Mutex.new # to serialize #write and #close
@@ -564,6 +595,11 @@ module Fluent
             end
           end
 
+          def on_writable
+            super
+            close if @close_after_write_complete
+          end
+
           def on_connect
             @callback_connection = TCPCallbackSocket.new(self)
             @connect_callback.call(@callback_connection)
@@ -577,7 +613,7 @@ module Fluent
           rescue => e
             @log.error "unexpected error on reading data", host: @callback_connection.remote_host, port: @callback_connection.remote_port, error: e
             @log.error_backtrace
-            close(true) rescue nil
+            close rescue nil
             raise if @under_plugin_development
           end
 
@@ -586,7 +622,7 @@ module Fluent
           rescue => e
             @log.error "unexpected error on reading data", host: @callback_connection.remote_host, port: @callback_connection.remote_port, error: e
             @log.error_backtrace
-            close(true) rescue nil
+            close rescue nil
             raise if @under_plugin_development
           end
 
@@ -601,6 +637,9 @@ module Fluent
         end
 
         class TLSServer < Coolio::Socket
+          attr_reader :closing
+          attr_writer :close_after_write_complete
+
           # It can't use Coolio::TCPSocket, because Coolio::TCPSocket checks that underlying socket (1st argument of super) is TCPSocket.
           def initialize(sock, context, socket_option_setter, close_callback, log, under_plugin_development, connect_callback)
             raise ArgumentError, "socket must be a TCPSocket: sock=#{sock}" unless sock.is_a?(TCPSocket)
@@ -620,6 +659,7 @@ module Fluent
             @close_callback = close_callback
 
             @callback_connection = nil
+            @close_after_write_complete = false
             @closing = false
 
             @mutex = Mutex.new # to serialize #write and #close
@@ -649,22 +689,39 @@ module Fluent
             end
           end
 
+          if RUBY_VERSION.to_f >= 2.3
+            NONBLOCK_ARG = {exception: false}
+            def try_handshake
+              @_handler_socket.accept_nonblock(NONBLOCK_ARG)
+            end
+          else
+            def try_handshake
+              @_handler_socket.accept_nonblock
+            rescue IO::WaitReadable
+              :wait_readable
+            rescue IO::WaitWritable
+              :wait_writable
+            end
+          end
+
           def try_tls_accept
             return true if @_handler_accepted
 
             begin
-              @_handler_socket.accept_nonblock # this method call actually try to do handshake via TLS
-              @_handler_accepted = true
-
-              @callback_connection = TLSCallbackSocket.new(self)
-              @connect_callback.call(@callback_connection)
-              unless @data_callback
-                raise "connection callback must call #data to set data callback"
+              result = try_handshake # this method call actually try to do handshake via TLS
+              if result == :wait_readable || result == :wait_writable
+                # retry accept_nonblock: there aren't enough data in underlying socket buffer
+              else
+                @_handler_accepted = true
+
+                @callback_connection = TLSCallbackSocket.new(self)
+                @connect_callback.call(@callback_connection)
+                unless @data_callback
+                  raise "connection callback must call #data to set data callback"
+                end
+
+                return true
               end
-              return true
-
-            rescue IO::WaitReadable, IO::WaitWritable
-              # retry accept_nonblock: there aren't enough data in underlying socket buffer
             rescue OpenSSL::SSL::SSLError => e
               @log.trace "unexpected error before accepting TLS connection", error: e
               close rescue nil
@@ -683,15 +740,20 @@ module Fluent
             end
           rescue IO::WaitReadable, IO::WaitWritable
             # ignore and return with doing nothing
+          rescue OpenSSL::SSL::SSLError => e
+            @log.warn "close socket due to unexpected ssl error: #{e}"
+            close rescue nil
           end
 
           def on_writable
             begin
               @mutex.synchronize do
+                # Consider write_nonblock with {exception: false} when IO::WaitWritable error happens frequently.
                 written_bytes = @_handler_socket.write_nonblock(@_handler_write_buffer)
                 @_handler_write_buffer.slice!(0, written_bytes)
                 super
               end
+              close if @close_after_write_complete
             rescue IO::WaitWritable, IO::WaitReadable
               return
             rescue Errno::EINTR
@@ -710,7 +772,7 @@ module Fluent
           rescue => e
             @log.error "unexpected error on reading data", host: @callback_connection.remote_host, port: @callback_connection.remote_port, error: e
             @log.error_backtrace
-            close(true) rescue nil
+            close rescue nil
             raise if @under_plugin_development
           end
 
@@ -719,7 +781,7 @@ module Fluent
           rescue => e
             @log.error "unexpected error on reading data", host: @callback_connection.remote_host, port: @callback_connection.remote_port, error: e
             @log.error_backtrace
-            close(true) rescue nil
+            close rescue nil
             raise if @under_plugin_development
           end
 

data/lib/fluent/plugin_helper/socket_option.rb

@@ -24,7 +24,7 @@ module Fluent
       FORMAT_STRUCT_LINGER  = 'I!I!' # { int l_onoff; int l_linger; }
       FORMAT_STRUCT_TIMEVAL = 'L!L!' # { time_t tv_sec; suseconds_t tv_usec; }
 
-      def socket_option_validate!(protocol, resolve_name: nil, linger_timeout: nil, recv_timeout: nil, send_timeout: nil)
+      def socket_option_validate!(protocol, resolve_name: nil, linger_timeout: nil, recv_timeout: nil, send_timeout: nil, receive_buffer_size: nil)
         unless resolve_name.nil?
           if protocol != :tcp && protocol != :udp && protocol != :tls
             raise ArgumentError, "BUG: resolve_name in available for tcp/udp/tls"
@@ -37,7 +37,7 @@ module Fluent
         end
       end
 
-      def socket_option_set(sock, resolve_name: nil, nonblock: false, linger_timeout: nil, recv_timeout: nil, send_timeout: nil)
+      def socket_option_set(sock, resolve_name: nil, nonblock: false, linger_timeout: nil, recv_timeout: nil, send_timeout: nil, receive_buffer_size: nil)
         unless resolve_name.nil?
           sock.do_not_reverse_lookup = !resolve_name
         end
@@ -56,6 +56,9 @@ module Fluent
           optval = [send_timeout.to_i, 0].pack(FORMAT_STRUCT_TIMEVAL)
           socket_option_set_one(sock, :SO_SNDTIMEO, optval)
         end
+        if receive_buffer_size
+          socket_option_set_one(sock, :SO_RCVBUF, receive_buffer_size.to_i)
+        end
         sock
       end
 

data/lib/fluent/plugin_helper/timer.rb

@@ -15,6 +15,7 @@
 #
 
 require 'fluent/plugin_helper/event_loop'
+require 'set'
 
 module Fluent
   module PluginHelper
@@ -33,7 +34,8 @@ module Fluent
         raise ArgumentError, "BUG: title must be a symbol" unless title.is_a? Symbol
         raise ArgumentError, "BUG: block not specified for callback" unless block_given?
         checker = ->(){ @_timer_running }
-        timer = TimerWatcher.new(title, interval, repeat, log, checker, &block)
+        detacher = ->(watcher){ event_loop_detach(watcher) }
+        timer = TimerWatcher.new(title, interval, repeat, log, checker, detacher, &block)
         @_timers << title
         event_loop_attach(timer)
         timer
@@ -45,7 +47,7 @@ module Fluent
 
       def initialize
         super
-        @_timers = []
+        @_timers ||= Set.new
       end
 
       def start
@@ -60,16 +62,17 @@ module Fluent
 
       def terminate
         super
-        @_timers = []
+        @_timers = nil
       end
 
       class TimerWatcher < Coolio::TimerWatcher
-        def initialize(title, interval, repeat, log, checker, &callback)
+        def initialize(title, interval, repeat, log, checker, detacher, &callback)
           @title = title
           @callback = callback
           @repeat = repeat
           @log = log
           @checker = checker
+          @detacher = detacher
           super(interval, repeat)
         end
 
@@ -81,9 +84,7 @@ module Fluent
           detach
           @log.error "Timer detached.", title: @title
         ensure
-          if attached?
-            detach unless @repeat
-          end
+          @detacher.call(self) unless @repeat
         end
       end
     end

data/lib/fluent/root_agent.rb

@@ -212,7 +212,6 @@ module Fluent
 
       lifecycle_unsafe_sequence = ->(method, checker) {
         operation = case method
-                    when :before_shutdown then "preparing shutdown"
                     when :shutdown then "shutting down"
                     when :close    then "closing"
                     else
@@ -228,11 +227,23 @@ module Fluent
             Thread.current.abort_on_exception = true
             begin
               if method == :shutdown
+                # To avoid Input#shutdown and Output#before_shutdown mismatch problem, combine before_shutdown and shutdown call in one sequence.
+                # The problem is in_tail flushes buffered multiline in shutdown but output's flush_at_shutdown is invoked in before_shutdown
+                operation = "preparing shutdown" # for logging
+                log.debug "#{operation} #{kind} plugin", type: Plugin.lookup_type_from_class(instance.class), plugin_id: instance.plugin_id
+                begin
+                  instance.send(:before_shutdown) unless instance.send(:before_shutdown?)
+                rescue Exception => e
+                  log.warn "unexpected error while #{operation} on #{kind} plugin", plugin: instance.class, plugin_id: instance.plugin_id, error: e
+                  log.warn_backtrace
+                end
+                operation = "shutting down"
                 log.info "#{operation} #{kind} plugin", type: Plugin.lookup_type_from_class(instance.class), plugin_id: instance.plugin_id
+                instance.send(:shutdown) unless instance.send(:shutdown?)
               else
                 log.debug "#{operation} #{kind} plugin", type: Plugin.lookup_type_from_class(instance.class), plugin_id: instance.plugin_id
+                instance.send(method) unless instance.send(checker)
               end
-              instance.send(method) unless instance.send(checker)
             rescue Exception => e
               log.warn "unexpected error while #{operation} on #{kind} plugin", plugin: instance.class, plugin_id: instance.plugin_id, error: e
               log.warn_backtrace
@@ -245,8 +256,6 @@ module Fluent
       lifecycle_safe_sequence.call(:stop, :stopped?)
 
       # before_shutdown does force_flush for output plugins: it should block, so it's unsafe operation
-      lifecycle_unsafe_sequence.call(:before_shutdown, :before_shutdown?)
-
       lifecycle_unsafe_sequence.call(:shutdown, :shutdown?)
 
       lifecycle_safe_sequence.call(:after_shutdown, :after_shutdown?)
@@ -292,7 +301,7 @@ module Fluent
     end
 
     def emit_error_event(tag, time, record, error)
-      error_info = {error: error, tag: tag, time: time}
+      error_info = {error: error, location: (error.backtrace ? error.backtrace.first : nil), tag: tag, time: time}
       if @error_collector
         # A record is not included in the logs because <@ERROR> handles it. This warn is for the notification
         log.warn "send an error event to @ERROR:", error_info
@@ -304,12 +313,12 @@ module Fluent
     end
 
     def handle_emits_error(tag, es, error)
-      error_info = {error: error, tag: tag}
+      error_info = {error: error, location: (error.backtrace ? error.backtrace.first : nil), tag: tag}
       if @error_collector
         log.warn "send an error event stream to @ERROR:", error_info
         @error_collector.emit_stream(tag, es)
       else
-        now = Time.now
+        now = Time.now.to_i
         if @suppress_emit_error_log_interval.zero? || now > @next_emit_error_log_time
           log.warn "emit transaction failed:", error_info
           log.warn_backtrace
@@ -333,12 +342,12 @@ module Fluent
       end
 
       def emit_error_event(tag, time, record, error)
-        error_info = {error: error, tag: tag, time: time, record: record}
+        error_info = {error: error, location: (error.backtrace ? error.backtrace.first : nil), tag: tag, time: time, record: record}
         log.warn "dump an error event in @ERROR:", error_info
       end
 
       def handle_emits_error(tag, es, e)
-        now = EventTime.now
+        now = EventTime.now.to_i
         if @suppress_emit_error_log_interval.zero? || now > @next_emit_error_log_time
           log.warn "emit transaction failed in @ERROR:", error: e, tag: tag
           log.warn_backtrace