fluentd 0.14.11 → 0.14.12

data/lib/fluent/plugin/in_tcp.rb

@@ -48,11 +48,15 @@ module Fluent::Plugin
       @parser = parser_create
     end
 
+    def multi_workers_ready?
+      true
+    end
+
     def start
       super
 
       @buffer = ''
-      server_create(:in_tcp_server, @port, proto: :tcp, bind: @bind) do |data, conn|
+      server_create(:in_tcp_server, @port, bind: @bind) do |data, conn|
         @buffer << data
         begin
           pos = 0

data/lib/fluent/plugin/in_udp.rb

@@ -47,6 +47,10 @@ module Fluent::Plugin
       @parser = parser_create
     end
 
+    def multi_workers_ready?
+      true
+    end
+
     def start
       super
 

data/lib/fluent/plugin/input.rb

@@ -28,6 +28,10 @@ module Fluent
       include PluginHelper::Mixin
 
       helpers :event_emitter
+
+      def multi_workers_ready?
+        false
+      end
     end
   end
 end

data/lib/fluent/plugin/out_copy.rb

@@ -25,6 +25,10 @@ module Fluent::Plugin
     desc 'If true, pass different record to each `store` plugin.'
     config_param :deep_copy, :bool, default: false
 
+    def multi_workers_ready?
+      true
+    end
+
     def process(tag, es)
       unless es.repeatable?
         m = Fluent::MultiEventStream.new

data/lib/fluent/plugin/out_exec.rb

@@ -51,6 +51,10 @@ module Fluent::Plugin
       @formatter = formatter_create
     end
 
+    def multi_workers_ready?
+      true
+    end
+
     NEWLINE = "\n"
 
     def format(tag, time, record)

data/lib/fluent/plugin/out_exec_filter.rb

@@ -172,6 +172,10 @@ module Fluent::Plugin
       @next_log_time = Time.now.to_i
     end
 
+    def multi_workers_ready?
+      true
+    end
+
     ExecutedProcess = Struct.new(:mutex, :pid, :respawns, :readio, :writeio)
 
     def start

data/lib/fluent/plugin/out_file.rb

@@ -93,14 +93,15 @@ module Fluent::Plugin
 
       configured_time_slice_format = conf['time_slice_format']
 
-      # v0.14 file buffer handles path as directory if '*' is missing
-      # 'dummy_path' is not to raise configuration error for 'path' in file buffer plugin,
-      # but raise it in this plugin.
       if conf.elements(name: 'buffer').empty?
         conf.add_element('buffer', 'time')
       end
       buffer_conf = conf.elements(name: 'buffer').first
-      unless buffer_conf.has_key?('path')
+      # Fluent::PluginId#configure is not called yet, so we can't use #plugin_root_dir here.
+      if !buffer_conf.has_key?('path') && !(conf['@id'] && system_config.root_dir)
+        # v0.14 file buffer handles path as directory if '*' is missing
+        # 'dummy_path' is not to raise configuration error for 'path' in file buffer plugin,
+        # but raise it in this plugin.
         buffer_conf['path'] = conf['path'] || '/tmp/dummy_path'
       end
 
@@ -159,6 +160,11 @@ module Fluent::Plugin
 
       @dir_perm = system_config.dir_permission || DIR_PERMISSION
       @file_perm = system_config.file_permission || FILE_PERMISSION
+      @need_lock = system_config.workers > 1
+    end
+
+    def multi_workers_ready?
+      true
     end
 
     def format(tag, time, record)
@@ -170,34 +176,51 @@ module Fluent::Plugin
       path = extract_placeholders(@path_template, chunk.metadata)
       FileUtils.mkdir_p File.dirname(path), mode: @dir_perm
 
-      unless @append
-        path = find_filepath_available(path)
-      end
-
-      case @compress_method
-      when nil
-        File.open(path, "ab", @file_perm) do |f|
-          chunk.write_to(f)
-        end
-      when :gzip
-        if @buffer.compress != :gzip || @recompress
-          File.open(path, "ab", @file_perm) do |f|
-            gz = Zlib::GzipWriter.new(f)
-            chunk.write_to(gz, compressed: :text)
-            gz.close
-          end
-        else
-          File.open(path, "ab", @file_perm) do |f|
-            chunk.write_to(f, compressed: :gzip)
-          end
-        end
+      writer = case
+               when @compress_method.nil?
+                 method(:write_without_compression)
+               when @compress_method == :gzip
+                 if @buffer.compress != :gzip || @recompress
+                   method(:write_gzip_with_compression)
+                 else
+                   method(:write_gzip_from_gzipped_chunk)
+                 end
+               else
+                 raise "BUG: unknown compression method #{@compress_method}"
+               end
+
+      if @append
+        writer.call(path, chunk)
       else
-        raise "BUG: unknown compression method #{@compress_method}"
+        find_filepath_available(path, with_lock: @need_lock) do |actual_path|
+          writer.call(actual_path, chunk)
+          path = actual_path
+        end
       end
 
       @last_written_path = path
     end
 
+    def write_without_compression(path, chunk)
+      File.open(path, "ab", @file_perm) do |f|
+        chunk.write_to(f)
+      end
+    end
+
+    def write_gzip_with_compression(path, chunk)
+      File.open(path, "ab", @file_perm) do |f|
+        gz = Zlib::GzipWriter.new(f)
+        chunk.write_to(gz, compressed: :text)
+        gz.close
+      end
+    end
+
+    def write_gzip_from_gzipped_chunk(path, chunk)
+      File.open(path, "ab", @file_perm) do |f|
+        chunk.write_to(f, compressed: :gzip)
+      end
+    end
+
     def timekey_to_timeformat(timekey)
       case timekey
       when nil          then ''
@@ -249,14 +272,31 @@ module Fluent::Plugin
       end
     end
 
-    def find_filepath_available(path_with_placeholder) # for non-append
+    def find_filepath_available(path_with_placeholder, with_lock: false) # for non-append
       raise "BUG: index placeholder not found in path: #{path_with_placeholder}" unless path_with_placeholder.index('_**')
       i = 0
-      while path = path_with_placeholder.sub('_**', "_#{i}")
-        break unless File.exist?(path)
+      dir_path = locked = nil
+      while true
+        path = path_with_placeholder.sub('_**', "_#{i}")
         i += 1
+        next if File.exist?(path)
+
+        if with_lock
+          dir_path = path + '.lock'
+          locked = Dir.mkdir(dir_path) rescue false
+          next unless locked
+          # ensure that other worker doesn't create a file (and release lock)
+          # between previous File.exist? and Dir.mkdir
+          next if File.exist?(path)
+        end
+
+        break
+      end
+      yield path
+    ensure
+      if dir_path && locked && Dir.exist?(dir_path)
+        Dir.rmdir(dir_path) rescue nil
       end
-      path
     end
   end
 end

data/lib/fluent/plugin/out_forward.rb

@@ -16,6 +16,7 @@
 
 require 'fluent/output'
 require 'fluent/config/error'
+require 'fluent/clock'
 require 'base64'
 
 require 'fluent/compat/socket_util'
@@ -32,14 +33,17 @@ module Fluent::Plugin
 
     LISTEN_PORT = 24224
 
-    PROCESS_CLOCK_ID = Process::CLOCK_MONOTONIC_RAW rescue Process::CLOCK_MONOTONIC
+    desc 'The transport protocol.'
+    config_param :transport, :enum, list: [:tcp, :tls], default: :tcp
+    # TODO: TLS session cache/tickets
+    # TODO: Connection keepalive
 
     desc 'The timeout time when sending event logs.'
     config_param :send_timeout, :time, default: 60
     # TODO: add linger_timeout, recv_timeout
 
-    desc 'The transport protocol to use for heartbeats.(udp,tcp,none)'
-    config_param :heartbeat_type, :enum, list: [:tcp, :udp, :none], default: :tcp
+    desc 'The protocol to use for heartbeats (default is the same with "transport").'
+    config_param :heartbeat_type, :enum, list: [:transport, :tcp, :udp, :none], default: :transport
     desc 'The interval of the heartbeat packer.'
     config_param :heartbeat_interval, :time, default: 1
     desc 'The wait time before accepting a server fault recovery.'
@@ -70,9 +74,25 @@ module Fluent::Plugin
     desc 'Enable client-side DNS round robin.'
     config_param :dns_round_robin, :bool, default: false # heartbeat_type 'udp' is not available for this
 
+    desc 'Ignore DNS resolution and errors at startup time.'
+    config_param :ignore_network_errors_at_startup, :bool, default: false
+
     desc 'Compress buffered data.'
     config_param :compress, :enum, list: [:text, :gzip], default: :text
 
+    desc 'The default version of TLS transport.'
+    config_param :tls_version, :enum, list: Fluent::PluginHelper::Socket::TLS_SUPPORTED_VERSIONS, default: Fluent::PluginHelper::Socket::TLS_DEFAULT_VERSION
+    desc 'The cipher configuration of TLS transport.'
+    config_param :tls_ciphers, :string, default: Fluent::PluginHelper::Socket::CIPHERS_DEFAULT
+    desc 'Skip all verification of certificates or not.'
+    config_param :tls_insecure_mode, :bool, default: false
+    desc 'Allow self signed certificates or not.'
+    config_param :tls_allow_self_signed_cert, :bool, default: false
+    desc 'Verify hostname of servers and certificates or not in TLS transport.'
+    config_param :tls_verify_hostname, :bool, default: true
+    desc 'The additional CA certificate path for TLS.'
+    config_param :tls_cert_path, :array, value_type: :string, default: nil
+
     config_section :security, required: false, multi: false do
       desc 'The hostname'
       config_param :self_hostname, :string
@@ -83,7 +103,7 @@ module Fluent::Plugin
     config_section :server, param_name: :servers do
       desc "The IP address or host name of the server."
       config_param :host, :string
-      desc "The name of the server. Used in log messages."
+      desc "The name of the server. Used for logging and certificate verification in TLS transport (when host is address)."
       config_param :name, :string, default: nil
       desc "The port number of the host."
       config_param :port, :integer, default: LISTEN_PORT
@@ -134,9 +154,29 @@ module Fluent::Plugin
       @read_interval = @read_interval_msec / 1000.0
       @recover_sample_size = @recover_wait / @heartbeat_interval
 
+      if @heartbeat_type == :tcp
+        log.warn "'heartbeat_type tcp' is deprecated. use 'transport' instead."
+        @heartbeat_type = :transport
+      end
+
       if @dns_round_robin
         if @heartbeat_type == :udp
-          raise Fluent::ConfigError, "forward output heartbeat type must be 'tcp' or 'none' to use dns_round_robin option"
+          raise Fluent::ConfigError, "forward output heartbeat type must be 'transport' or 'none' to use dns_round_robin option"
+        end
+      end
+
+      if @transport == :tls
+        if @tls_cert_path && !@tls_cert_path.empty?
+          @tls_cert_path.each do |path|
+            raise Fluent::ConfigError, "specified cert path does not exist:#{path}" unless File.exist?(path)
+            raise Fluent::ConfigError, "specified cert path is not readable:#{path}" unless File.readable?(path)
+          end
+        end
+
+        if @tls_insecure_mode
+          log.warn "TLS transport is configured in insecure way"
+          @tls_verify_hostname = false
+          @tls_allow_self_signed_cert = true
         end
       end
 
@@ -148,7 +188,15 @@ module Fluent::Plugin
         if @heartbeat_type == :none
           @nodes << NoneHeartbeatNode.new(self, server, failure: failure)
         else
-          @nodes << Node.new(self, server, failure: failure)
+          node = Node.new(self, server, failure: failure)
+          begin
+            node.validate_host_resolution!
+          rescue => e
+            raise unless @ignore_network_errors_at_startup
+            log.warn "failed to resolve node name when configured", server: (server.name || server.host), error: e
+            node.disable!
+          end
+          @nodes << node
         end
       end
 
@@ -167,6 +215,10 @@ module Fluent::Plugin
       raise Fluent::ConfigError, "ack_response_timeout must be a positive integer" if @ack_response_timeout < 1
     end
 
+    def multi_workers_ready?
+      true
+    end
+
     def prefer_delayed_commit
       @require_ack_response
     end
@@ -204,7 +256,10 @@ module Fluent::Plugin
     end
 
     def close
-      @usock.close if @usock
+      if @usock
+        # close socket and ignore errors: this socket will not be used anyway.
+        @usock.close rescue nil
+      end
       super
     end
 
@@ -229,7 +284,7 @@ module Fluent::Plugin
       tag = chunk.metadata.tag
       sock, node = select_a_healthy_node{|n| n.send_data(tag, chunk) }
       chunk_id_base64 = Base64.encode64(chunk.unique_id)
-      current_time = Process.clock_gettime(PROCESS_CLOCK_ID)
+      current_time = Fluent::Clock.now
       info = ACKWaitingSockInfo.new(sock, chunk.unique_id, chunk_id_base64, node, current_time, @ack_response_timeout)
       @sock_ack_waiting_mutex.synchronize do
         @sock_ack_waiting << info
@@ -258,14 +313,34 @@ module Fluent::Plugin
       raise NoNodesAvailable, "no nodes are available"
     end
 
-    def create_transfer_socket(host, port, &block)
-      socket_create_tcp(
-        host, port,
-        linger_timeout: @send_timeout,
-        send_timeout: @send_timeout,
-        recv_timeout: @ack_response_timeout,
-        &block
-      )
+    def create_transfer_socket(host, port, hostname, &block)
+      case @transport
+      when :tls
+        socket_create_tls(
+          host, port,
+          version: @tls_version,
+          ciphers: @tls_ciphers,
+          insecure: @tls_insecure_mode,
+          verify_fqdn: @tls_verify_hostname,
+          fqdn: hostname,
+          allow_self_signed_cert: @tls_allow_self_signed_cert,
+          cert_paths: @tls_cert_path,
+          linger_timeout: @send_timeout,
+          send_timeout: @send_timeout,
+          recv_timeout: @ack_response_timeout,
+          &block
+        )
+      when :tcp
+        socket_create_tcp(
+          host, port,
+          linger_timeout: @send_timeout,
+          send_timeout: @send_timeout,
+          recv_timeout: @ack_response_timeout,
+          &block
+        )
+      else
+        raise "BUG: unknown transport protocol #{@transport}"
+      end
     end
 
     # MessagePack FixArray length is 3
@@ -390,7 +465,7 @@ module Fluent::Plugin
       unpacker = Fluent::Engine.msgpack_unpacker
 
       while thread_current_running?
-        now = Process.clock_gettime(PROCESS_CLOCK_ID)
+        now = Fluent::Clock.now
         sockets = []
         begin
           @sock_ack_waiting_mutex.synchronize do
@@ -441,6 +516,14 @@ module Fluent::Plugin
         @available = true
         @state = nil
 
+        # @hostname is used for certificate verification & TLS SNI
+        host_is_hostname = !(IPAddr.new(@host) rescue false)
+        @hostname = case
+                    when host_is_hostname then @host
+                    when @name then @name
+                    else nil
+                    end
+
         @usock = nil
 
         @username = server.username
@@ -453,7 +536,7 @@ module Fluent::Plugin
 
         @resolved_host = nil
         @resolved_time = 0
-        resolved_host  # check dns
+        @resolved_once = false
       end
 
       attr_accessor :usock
@@ -462,6 +545,10 @@ module Fluent::Plugin
       attr_reader :sockaddr  # used by on_heartbeat
       attr_reader :failure, :available # for test
 
+      def validate_host_resolution!
+        resolved_host
+      end
+
       def available?
         @available
       end
@@ -517,20 +604,26 @@ module Fluent::Plugin
         option = { 'size' => chunk.size, 'compressed' => @compress }
         option['chunk'] = Base64.encode64(chunk.unique_id) if @sender.require_ack_response
 
-        # out_forward always uses Raw32 type for content.
-        # Raw16 can store only 64kbytes, and it should be much smaller than buffer chunk size.
+        # https://github.com/fluent/fluentd/wiki/Forward-Protocol-Specification-v1#packedforward-mode
+        # out_forward always uses str32 type for entries.
+        # str16 can store only 64kbytes, and it should be much smaller than buffer chunk size.
+
+        tag = tag.dup.force_encoding(Encoding::UTF_8)
 
-        sock.write @sender.forward_header        # beginArray(3)
-        sock.write tag.to_msgpack                # 1. writeRaw(tag)
+        sock.write @sender.forward_header                    # array, size=3
+        sock.write tag.to_msgpack                            # 1. tag: String (str)
         chunk.open(compressed: @compress) do |chunk_io|
-          sock.write [0xdb, chunk_io.size].pack('CN') # 2. beginRaw(size) raw32
-          IO.copy_stream(chunk_io, sock)              # writeRawBody(packed_es)
+          entries = [0xdb, chunk_io.size].pack('CN')
+          sock.write entries.force_encoding(Encoding::UTF_8) # 2. entries: String (str32)
+          IO.copy_stream(chunk_io, sock)                     #    writeRawBody(packed_es)
         end
-        sock.write option.to_msgpack             # 3. writeOption(option)
+        sock.write option.to_msgpack                         # 3. option: Hash(map)
+
+        # TODO: use bin32 for non-utf8 content(entries) when old msgpack-ruby (0.5.x or earlier) not supported
       end
 
       def send_data(tag, chunk)
-        sock = @sender.create_transfer_socket(resolved_host, port)
+        sock = @sender.create_transfer_socket(resolved_host, port, @hostname)
         begin
           send_data_actual(sock, tag, chunk)
         rescue
@@ -550,9 +643,20 @@ module Fluent::Plugin
 
       # FORWARD_TCP_HEARTBEAT_DATA = FORWARD_HEADER + ''.to_msgpack + [].to_msgpack
       def send_heartbeat
+        begin
+          dest_addr = resolved_host
+          @resolved_once = true
+        rescue ::SocketError => e
+          if !@resolved_once && @sender.ignore_network_errors_at_startup
+            @log.warn "failed to resolve node name in heartbeating", server: @name || @host, error: e
+            return
+          end
+          raise
+        end
+
         case @sender.heartbeat_type
-        when :tcp
-          @sender.create_transfer_socket(resolved_host, port) do |sock|
+        when :transport
+          @sender.create_transfer_socket(dest_addr, port, @hostname) do |sock|
             ## don't send any data to not cause a compatibility problem
             # sock.write FORWARD_TCP_HEARTBEAT_DATA