fluentd 0.14.17-x64-mingw32 → 1.3.1-x64-mingw32

data/lib/fluent/plugin/in_tcp.rb

@@ -55,13 +55,12 @@ module Fluent::Plugin
     def start
       super
 
-      @buffer = ''
-      server_create(:in_tcp_server, @port, bind: @bind) do |data, conn|
-        @buffer << data
+      server_create(:in_tcp_server, @port, bind: @bind, resolve_name: !!@source_hostname_key) do |data, conn|
+        conn.buffer << data
         begin
           pos = 0
-          while i = @buffer.index(@delimiter, pos)
-            msg = @buffer[pos...i]
+          while i = conn.buffer.index(@delimiter, pos)
+            msg = conn.buffer[pos...i]
             pos = i + @delimiter.length
 
             @parser.parse(msg) do |time, record|
@@ -77,7 +76,7 @@ module Fluent::Plugin
               router.emit(tag, time, record)
             end
           end
-          @buffer.slice!(0, pos) if pos > 0
+          conn.buffer.slice!(0, pos) if pos > 0
         end
       end
     end

data/lib/fluent/plugin/in_udp.rb

@@ -38,6 +38,10 @@ module Fluent::Plugin
     config_param :body_size_limit, :size, default: nil, deprecated: "use message_length_limit instead."
     desc "The max bytes of message"
     config_param :message_length_limit, :size, default: 4096
+    desc "Remove newline from the end of incoming payload"
+    config_param :remove_newline, :bool, default: true
+    desc "The max size of socket receive buffer. SO_RCVBUF"
+    config_param :receive_buffer_size, :size, default: nil
 
     config_param :blocking_timeout, :time, default: 0.5
 
@@ -59,8 +63,8 @@ module Fluent::Plugin
       super
 
       log.info "listening udp socket", bind: @bind, port: @port
-      server_create(:in_udp_server, @port, proto: :udp, bind: @bind, max_bytes: @message_length_limit) do |data, sock|
-        data.chomp!
+      server_create(:in_udp_server, @port, proto: :udp, bind: @bind, resolve_name: !!@source_hostname_key, max_bytes: @message_length_limit, receive_buffer_size: @receive_buffer_size) do |data, sock|
+        data.chomp! if @remove_newline
         begin
           @parser.parse(data) do |time, record|
             unless time && record

data/lib/fluent/plugin/in_unix.rb

@@ -146,7 +146,7 @@ module Fluent
           @u = Fluent::Engine.msgpack_factory.unpacker
         end
 
-        (class << self; self; end).module_eval do
+        singleton_class.module_eval do
           define_method(:on_read, m)
         end
         m.call(data)

data/lib/fluent/plugin/multi_output.rb

@@ -29,6 +29,7 @@ module Fluent
       helpers :event_emitter # to get router from agent, which will be supplied to child plugins
 
       config_section :store, param_name: :stores, multi: true, required: true do
+        config_argument :arg, :string, default: ''
         config_param :@type, :string, default: nil
       end
 

data/lib/fluent/plugin/out_copy.rb

@@ -25,6 +25,21 @@ module Fluent::Plugin
     desc 'If true, pass different record to each `store` plugin.'
     config_param :deep_copy, :bool, default: false
 
+    attr_reader :ignore_errors
+
+    def initialize
+      super
+      @ignore_errors = []
+    end
+
+    def configure(conf)
+      super
+
+      @stores.each { |store|
+        @ignore_errors << (store.arg == 'ignore_error')
+      }
+    end
+
     def multi_workers_ready?
       true
     end
@@ -38,8 +53,16 @@ module Fluent::Plugin
         es = m
       end
 
-      outputs.each do |output|
-        output.emit_events(tag, @deep_copy ? es.dup : es)
+      outputs.each_with_index do |output, i|
+        begin
+          output.emit_events(tag, @deep_copy ? es.dup : es)
+        rescue => e
+          if @ignore_errors[i]
+            log.error "ignore emit error", error: e
+          else
+            raise e
+          end
+        end
       end
     end
   end

data/lib/fluent/plugin/out_file.rb

@@ -17,6 +17,7 @@
 require 'fileutils'
 require 'zlib'
 require 'time'
+require 'tempfile'
 
 require 'fluent/plugin/output'
 require 'fluent/config/error'
@@ -141,8 +142,8 @@ module Fluent::Plugin
         dummy_record_keys = get_placeholders_keys(@path_template) || ['message']
         dummy_record = Hash[dummy_record_keys.zip(['data'] * dummy_record_keys.size)]
 
-        test_meta1 = metadata_for_test(dummy_tag, Fluent::Engine.now, dummy_record)
-        test_path = extract_placeholders(@path_template, test_meta1)
+        test_chunk1 = chunk_for_test(dummy_tag, Fluent::Engine.now, dummy_record)
+        test_path = extract_placeholders(@path_template, test_chunk1)
         unless ::Fluent::FileUtil.writable_p?(test_path)
           raise Fluent::ConfigError, "out_file: `#{test_path}` is not writable"
         end
@@ -178,7 +179,7 @@ module Fluent::Plugin
     end
 
     def write(chunk)
-      path = extract_placeholders(@path_template, chunk.metadata)
+      path = extract_placeholders(@path_template, chunk)
       FileUtils.mkdir_p File.dirname(path), mode: @dir_perm
 
       writer = case
@@ -213,10 +214,28 @@ module Fluent::Plugin
     end
 
     def write_gzip_with_compression(path, chunk)
-      File.open(path, "ab", @file_perm) do |f|
-        gz = Zlib::GzipWriter.new(f)
-        chunk.write_to(gz, compressed: :text)
-        gz.close
+      if @append
+        # This code will be removed after zlib/multithread bug is fixed.
+        # Use Tempfile to avoid broken gzip files: https://github.com/fluent/fluentd/issues/1903
+        Tempfile.create('out_file-gzip-append') { |temp|
+          begin
+            writer = Zlib::GzipWriter.new(temp)
+            chunk.write_to(writer, compressed: :text)
+          ensure
+            writer.finish # avoid zlib finalizer warning
+          end
+          temp.rewind
+
+          File.open(path, "ab", @file_perm) do |f|
+            IO.copy_stream(temp, f)
+          end
+        }
+      else
+        File.open(path, "ab", @file_perm) do |f|
+          gz = Zlib::GzipWriter.new(f)
+          chunk.write_to(gz, compressed: :text)
+          gz.close
+        end
       end
     end
 

data/lib/fluent/plugin/out_forward.rb

@@ -77,6 +77,9 @@ module Fluent::Plugin
     desc 'Ignore DNS resolution and errors at startup time.'
     config_param :ignore_network_errors_at_startup, :bool, default: false
 
+    desc 'Verify that a connection can be made with one of out_forward nodes at the time of startup.'
+    config_param :verify_connection_at_startup, :bool, default: false
+
     desc 'Compress buffered data.'
     config_param :compress, :enum, list: [:text, :gzip], default: :text
 
@@ -91,6 +94,8 @@ module Fluent::Plugin
     desc 'Verify hostname of servers and certificates or not in TLS transport.'
     config_param :tls_verify_hostname, :bool, default: true
     desc 'The additional CA certificate path for TLS.'
+    config_param :tls_ca_cert_path, :array, value_type: :string, default: nil
+    desc 'The additional certificate path for TLS.'
     config_param :tls_cert_path, :array, value_type: :string, default: nil
 
     config_section :security, required: false, multi: false do
@@ -166,8 +171,12 @@ module Fluent::Plugin
       end
 
       if @transport == :tls
+        # socket helper adds CA cert or signed certificate to same cert store internally so unify it in this place.
         if @tls_cert_path && !@tls_cert_path.empty?
-          @tls_cert_path.each do |path|
+          @tls_ca_cert_path = @tls_cert_path
+        end
+        if @tls_ca_cert_path && !@tls_ca_cert_path.empty?
+          @tls_ca_cert_path.each do |path|
             raise Fluent::ConfigError, "specified cert path does not exist:#{path}" unless File.exist?(path)
             raise Fluent::ConfigError, "specified cert path is not readable:#{path}" unless File.readable?(path)
           end
@@ -253,6 +262,17 @@ module Fluent::Plugin
         @sock_ack_waiting = []
         thread_create(:out_forward_receiving_ack, &method(:ack_reader))
       end
+
+      if @verify_connection_at_startup
+        @nodes.each do |node|
+          begin
+            node.verify_connection
+          rescue StandardError => e
+            log.fatal "forward's connection setting error: #{e.message}"
+            raise Fluent::UnrecoverableError, e.message
+          end
+        end
+      end
     end
 
     def close
@@ -324,7 +344,7 @@ module Fluent::Plugin
           verify_fqdn: @tls_verify_hostname,
           fqdn: hostname,
           allow_self_signed_cert: @tls_allow_self_signed_cert,
-          cert_paths: @tls_cert_path,
+          cert_paths: @tls_ca_cert_path,
           linger_timeout: @send_timeout,
           send_timeout: @send_timeout,
           recv_timeout: @ack_response_timeout,
@@ -395,15 +415,19 @@ module Fluent::Plugin
 
     def on_timer
       @nodes.each {|n|
-        if n.tick
-          rebuild_weight_array
-        end
         begin
           log.trace "sending heartbeat", host: n.host, port: n.port, heartbeat_type: @heartbeat_type
           n.usock = @usock if @usock
-          n.send_heartbeat
-        rescue Errno::EAGAIN, Errno::EWOULDBLOCK, Errno::EINTR, Errno::ECONNREFUSED
-          log.debug "failed to send heartbeat packet", host: n.host, port: n.port, heartbeat_type: @heartbeat_type, error: $!
+          if n.send_heartbeat
+            rebuild_weight_array
+          end
+        rescue Errno::EAGAIN, Errno::EWOULDBLOCK, Errno::EINTR, Errno::ECONNREFUSED, Errno::ETIMEDOUT => e
+          log.debug "failed to send heartbeat packet", host: n.host, port: n.port, heartbeat_type: @heartbeat_type, error: e
+        rescue => e
+          log.debug "unexpected error happen during heartbeat", host: n.host, port: n.port, heartbeat_type: @heartbeat_type, error: e
+        end
+        if n.tick
+          rebuild_weight_array
         end
       }
     end
@@ -421,7 +445,7 @@ module Fluent::Plugin
     def read_ack_from_sock(sock, unpacker)
       begin
         raw_data = sock.instance_of?(Fluent::PluginHelper::Socket::WrappedSocket::TLS) ? sock.readpartial(@read_length) : sock.recv(@read_length)
-      rescue Errno::ECONNRESET
+      rescue Errno::ECONNRESET, EOFError # ECONNRESET for #recv, #EOFError for #readpartial
         raw_data = ""
       end
       info = @sock_ack_waiting_mutex.synchronize{ @sock_ack_waiting.find{|i| i.sock == sock } }
@@ -431,16 +455,16 @@ module Fluent::Plugin
       if raw_data.empty?
         log.warn "destination node closed the connection. regard it as unavailable.", host: info.node.host, port: info.node.port
         info.node.disable!
-        rollback_write(info.chunk_id)
+        rollback_write(info.chunk_id, update_retry: false)
         return nil
       else
         unpacker.feed(raw_data)
         res = unpacker.read
         log.trace "getting response from destination", host: info.node.host, port: info.node.port, chunk_id: dump_unique_id_hex(info.chunk_id), response: res
         if res['ack'] != info.chunk_id_base64
-          # Some errors may have occured when ack and chunk id is different, so send the chunk again.
+          # Some errors may have occurred when ack and chunk id is different, so send the chunk again.
           log.warn "ack in response and chunk id in sent data are different", chunk_id: dump_unique_id_hex(info.chunk_id), ack: res['ack']
-          rollback_write(info.chunk_id)
+          rollback_write(info.chunk_id, update_retry: false)
           return nil
         else
           log.trace "got a correct ack response", chunk_id: dump_unique_id_hex(info.chunk_id)
@@ -481,7 +505,7 @@ module Fluent::Plugin
                 log.warn "no response from node. regard it as unavailable.", host: info.node.host, port: info.node.port
                 info.node.disable!
                 info.sock.close rescue nil
-                rollback_write(info.chunk_id)
+                rollback_write(info.chunk_id, update_retry: false)
               else
                 sockets << info.sock
                 new_list << info
@@ -517,7 +541,6 @@ module Fluent::Plugin
         @standby = server.standby
         @failure = failure
         @available = true
-        @state = nil
 
         # @hostname is used for certificate verification & TLS SNI
         host_is_hostname = !(IPAddr.new(@host) rescue false)
@@ -533,7 +556,6 @@ module Fluent::Plugin
         @password = server.password
         @shared_key = server.shared_key || (sender.security && sender.security.shared_key) || ""
         @shared_key_salt = generate_salt
-        @shared_key_nonce = ""
 
         @unpacker = Fluent::Engine.msgpack_unpacker
 
@@ -548,6 +570,8 @@ module Fluent::Plugin
       attr_reader :sockaddr  # used by on_heartbeat
       attr_reader :failure, :available # for test
 
+      RequestInfo = Struct.new(:state, :shared_key_nonce, :auth)
+
       def validate_host_resolution!
         resolved_host
       end
@@ -564,8 +588,21 @@ module Fluent::Plugin
         @standby
       end
 
-      def establish_connection(sock)
-        while available? && @state != :established
+      def verify_connection
+        sock = @sender.create_transfer_socket(resolved_host, port, @hostname)
+        begin
+          ri = RequestInfo.new(@sender.security ? :helo : :established)
+          if ri.state != :established
+            establish_connection(sock, ri)
+            raise if ri.state != :established
+          end
+        ensure
+          sock.close
+        end
+      end
+
+      def establish_connection(sock, ri)
+        while available? && ri.state != :established
           begin
             # TODO: On Ruby 2.2 or earlier, read_nonblock doesn't work expectedly.
             # We need rewrite around here using new socket/server plugin helper.
@@ -575,13 +612,13 @@ module Fluent::Plugin
               next
             end
             @unpacker.feed_each(buf) do |data|
-              on_read(sock, data)
+              on_read(sock, ri, data)
             end
           rescue IO::WaitReadable
             # If the exception is Errno::EWOULDBLOCK or Errno::EAGAIN, it is extended by IO::WaitReadable.
             # So IO::WaitReadable can be used to rescue the exceptions for retrying read_nonblock.
-            # http://docs.ruby-lang.org/en/2.3.0/IO.html#method-i-read_nonblock
-            sleep @sender.read_interval unless @state == :established
+            # https//docs.ruby-lang.org/en/2.3.0/IO.html#method-i-read_nonblock
+            sleep @sender.read_interval unless ri.state == :established
           rescue SystemCallError => e
             @log.warn "disconnected by error", host: @host, port: @port, error: e
             disable!
@@ -595,9 +632,9 @@ module Fluent::Plugin
       end
 
       def send_data_actual(sock, tag, chunk)
-        @state = @sender.security ? :helo : :established
-        if @state != :established
-          establish_connection(sock)
+        ri = RequestInfo.new(@sender.security ? :helo : :established)
+        if ri.state != :established
+          establish_connection(sock, ri)
         end
 
         unless available?
@@ -663,11 +700,13 @@ module Fluent::Plugin
             ## don't send any data to not cause a compatibility problem
             # sock.write FORWARD_TCP_HEARTBEAT_DATA
 
-            # successful tcp connection establishment is considered as valid heartbeat
+            # successful tcp connection establishment is considered as valid heartbeat.
+            # When heartbeat is succeeded after detached, return true. It rebuilds weight array.
             heartbeat(true)
           end
         when :udp
           @usock.send "\0", 0, Socket.pack_sockaddr_in(@port, resolved_host)
+          nil
         when :none # :none doesn't use this class
           raise "BUG: heartbeat_type none must not use Node"
         else
@@ -750,7 +789,7 @@ module Fluent::Plugin
         SecureRandom.hex(16)
       end
 
-      def check_helo(message)
+      def check_helo(ri, message)
         @log.debug "checking helo"
         # ['HELO', options(hash)]
         unless message.size == 2 && message[0] == 'HELO'
@@ -758,23 +797,23 @@ module Fluent::Plugin
         end
         opts = message[1] || {}
         # make shared_key_check failed (instead of error) if protocol version mismatch exist
-        @shared_key_nonce = opts['nonce'] || ''
-        @authentication = opts['auth'] || ''
+        ri.shared_key_nonce = opts['nonce'] || ''
+        ri.auth = opts['auth'] || ''
         true
       end
 
-      def generate_ping
+      def generate_ping(ri)
         @log.debug "generating ping"
         # ['PING', self_hostname, sharedkey\_salt, sha512\_hex(sharedkey\_salt + self_hostname + nonce + shared_key),
         #  username || '', sha512\_hex(auth\_salt + username + password) || '']
         shared_key_hexdigest = Digest::SHA512.new.update(@shared_key_salt)
           .update(@sender.security.self_hostname)
-          .update(@shared_key_nonce)
+          .update(ri.shared_key_nonce)
           .update(@shared_key)
           .hexdigest
         ping = ['PING', @sender.security.self_hostname, @shared_key_salt, shared_key_hexdigest]
-        if !@authentication.empty?
-          password_hexdigest = Digest::SHA512.new.update(@authentication).update(@username).update(@password).hexdigest
+        if !ri.auth.empty?
+          password_hexdigest = Digest::SHA512.new.update(ri.auth).update(@username).update(@password).hexdigest
           ping.push(@username, password_hexdigest)
         else
           ping.push('','')
@@ -782,7 +821,7 @@ module Fluent::Plugin
         ping
       end
 
-      def check_pong(message)
+      def check_pong(ri, message)
         @log.debug "checking pong"
         # ['PONG', bool(authentication result), 'reason if authentication failed',
         #  self_hostname, sha512\_hex(salt + self_hostname + nonce + sharedkey)]
@@ -799,7 +838,7 @@ module Fluent::Plugin
           return false, 'same hostname between input and output: invalid configuration'
         end
 
-        clientside = Digest::SHA512.new.update(@shared_key_salt).update(hostname).update(@shared_key_nonce).update(@shared_key).hexdigest
+        clientside = Digest::SHA512.new.update(@shared_key_salt).update(hostname).update(ri.shared_key_nonce).update(@shared_key).hexdigest
         unless shared_key_hexdigest == clientside
           return false, 'shared key mismatch'
         end
@@ -807,29 +846,29 @@ module Fluent::Plugin
         return true, nil
       end
 
-      def on_read(sock, data)
+      def on_read(sock, ri, data)
         @log.trace __callee__
 
-        case @state
+        case ri.state
         when :helo
-          unless check_helo(data)
+          unless check_helo(ri, data)
             @log.warn "received invalid helo message from #{@name}"
             disable! # shutdown
             return
           end
-          sock.write(generate_ping.to_msgpack)
-          @state = :pingpong
+          sock.write(generate_ping(ri).to_msgpack)
+          ri.state = :pingpong
         when :pingpong
-          succeeded, reason = check_pong(data)
+          succeeded, reason = check_pong(ri, data)
           unless succeeded
-            @log.warn "connection refused to #{@name}: #{reason}"
+            @log.warn "connection refused to #{@name || @host}: #{reason}"
             disable! # shutdown
             return
           end
-          @state = :established
+          ri.state = :established
           @log.debug "connection established", host: @host, port: @port
         else
-          raise "BUG: unknown session state: #{@state}"
+          raise "BUG: unknown session state: #{ri.state}"
         end
       end
     end