fluentd 0.14.11 → 0.14.12

data/lib/fluent/plugin_helper/compat_parameters.rb

@@ -36,7 +36,7 @@ module Fluent
         "retry_limit"         => "retry_max_times",
         "max_retry_wait"      => "retry_max_interval",
         "buffer_chunk_limit"  => "chunk_limit_size",
-        "buffer_queue_limit"  => "queue_length_limit",
+        "buffer_queue_limit"  => "queue_limit_length",
         "buffer_queue_full_action" => "overflow_action",
         "flush_at_shutdown" => "flush_at_shutdown",
       }

data/lib/fluent/plugin_helper/event_loop.rb

@@ -16,6 +16,7 @@
 
 require 'cool.io'
 require 'fluent/plugin_helper/thread'
+require 'fluent/clock'
 
 module Fluent
   module PluginHelper
@@ -31,7 +32,6 @@ module Fluent
 
       EVENT_LOOP_RUN_DEFAULT_TIMEOUT = 0.5
       EVENT_LOOP_SHUTDOWN_TIMEOUT = 5
-      EVENT_LOOP_CLOCK_ID = Process::CLOCK_MONOTONIC_RAW rescue Process::CLOCK_MONOTONIC
 
       attr_reader :_event_loop # for tests
 
@@ -48,7 +48,14 @@ module Fluent
       end
 
       def event_loop_wait_until_stop
-        sleep(0.1) while event_loop_running?
+        timeout_at = Fluent::Clock.now + EVENT_LOOP_SHUTDOWN_TIMEOUT
+        sleep(0.1) while event_loop_running? && Fluent::Clock.now < timeout_at
+        if @_event_loop_running
+          puts "terminating event_loop forcedly"
+          caller.each{|bt| puts "\t#{bt}" }
+          @_event_loop.stop rescue nil
+          @_event_loop_running = true
+        end
       end
 
       def event_loop_running?
@@ -85,13 +92,31 @@ module Fluent
         @_event_loop_mutex.synchronize do
           @_event_loop_attached_watchers.reverse.each do |w|
             if w.attached?
+              begin
+                w.detach
+              rescue => e
+                log.warn "unexpected error while detaching event loop watcher", error: e
+              end
+            end
+          end
+        end
+
+        super
+      end
+
+      def after_shutdown
+        timeout_at = Fluent::Clock.now + EVENT_LOOP_SHUTDOWN_TIMEOUT
+        @_event_loop_mutex.synchronize do
+          @_event_loop.watchers.reverse.each do |w|
+            begin
               w.detach
+            rescue => e
+              log.warn "unexpected error while detaching event loop watcher", error: e
             end
           end
         end
-        timeout_at = Process.clock_gettime(EVENT_LOOP_CLOCK_ID) + EVENT_LOOP_SHUTDOWN_TIMEOUT
         while @_event_loop_running
-          if Process.clock_gettime(EVENT_LOOP_CLOCK_ID) >= timeout_at
+          if Fluent::Clock.now >= timeout_at
             log.warn "event loop does NOT exit until hard timeout."
             raise "event loop does NOT exit until hard timeout." if @under_plugin_development
             break

data/lib/fluent/plugin_helper/inject.rb

@@ -29,6 +29,9 @@ module Fluent
         if @_inject_hostname_key
           r[@_inject_hostname_key] = @_inject_hostname
         end
+        if @_inject_worker_id_key
+          r[@_inject_worker_id_key] = @_inject_worker_id
+        end
         if @_inject_tag_key
           r[@_inject_tag_key] = tag
         end
@@ -48,6 +51,9 @@ module Fluent
           if @_inject_hostname_key
             r[@_inject_hostname_key] = @_inject_hostname
           end
+          if @_inject_worker_id_key
+            r[@_inject_worker_id_key] = @_inject_worker_id
+          end
           if @_inject_tag_key
             r[@_inject_tag_key] = tag
           end
@@ -65,6 +71,7 @@ module Fluent
         config_section :inject, required: false, multi: false, param_name: :inject_config do
           config_param :hostname_key, :string, default: nil
           config_param :hostname, :string, default: nil
+          config_param :worker_id_key, :string, default: nil
           config_param :tag_key, :string, default: nil
           config_param :time_key, :string, default: nil
 
@@ -86,6 +93,8 @@ module Fluent
         @_inject_enabled = false
         @_inject_hostname_key = nil
         @_inject_hostname = nil
+        @_inject_worker_id_key = nil
+        @_inject_worker_id = nil
         @_inject_tag_key = nil
         @_inject_time_key = nil
         @_inject_time_formatter = nil
@@ -114,6 +123,10 @@ module Fluent
               log.info "using hostname for specified field", host_key: @_inject_hostname_key, host_name: @_inject_hostname
             end
           end
+          @_inject_worker_id_key = @inject_config.worker_id_key
+          if @_inject_worker_id_key
+            @_inject_worker_id = fluentd_worker_id # get id here, because #with_worker_config method may be used only for #configure in tests
+          end
           @_inject_tag_key = @inject_config.tag_key
           @_inject_time_key = @inject_config.time_key
           if @_inject_time_key
@@ -130,7 +143,7 @@ module Fluent
             end
           end
 
-          @_inject_enabled = @_inject_hostname_key || @_inject_tag_key || @_inject_time_key
+          @_inject_enabled = @_inject_hostname_key || @_inject_worker_id_key || @_inject_tag_key || @_inject_time_key
         end
       end
     end

data/lib/fluent/plugin_helper/server.rb

@@ -21,18 +21,22 @@ require 'cool.io'
 require 'socket'
 require 'ipaddr'
 require 'fcntl'
+require 'openssl'
 
 require_relative 'socket_option'
+require_relative 'cert_option'
 
 module Fluent
   module PluginHelper
     module Server
       include Fluent::PluginHelper::EventLoop
       include Fluent::PluginHelper::SocketOption
+      include Fluent::PluginHelper::CertOption
 
       # This plugin helper doesn't support these things for now:
-      # * SSL/TLS (TBD)
       # * TCP/TLS keepalive
+      # * TLS session cache/tickets
+      # * unix domain sockets
 
       # stop     : [-]
       # shutdown : detach server event handler from event loop (event_loop)
@@ -63,12 +67,16 @@ module Fluent
       #     conn.close
       #   end
       # end
-      def server_create_connection(title, port, proto: :tcp, bind: '0.0.0.0', shared: true, backlog: nil, **socket_options, &block)
+      def server_create_connection(title, port, proto: nil, bind: '0.0.0.0', shared: true, backlog: nil, tls_options: nil, **socket_options, &block)
+        proto ||= (@transport_config && @transport_config.protocol == :tls) ? :tls : :tcp
+
         raise ArgumentError, "BUG: title must be a symbol" unless title && title.is_a?(Symbol)
         raise ArgumentError, "BUG: port must be an integer" unless port && port.is_a?(Integer)
         raise ArgumentError, "BUG: invalid protocol name" unless PROTOCOLS.include?(proto)
         raise ArgumentError, "BUG: cannot create connection for UDP" unless CONNECTION_PROTOCOLS.include?(proto)
 
+        raise ArgumentError, "BUG: tls_options is available only for tls" if tls_options && proto != :tls
+
         raise ArgumentError, "BUG: block not specified which handles connection" unless block_given?
         raise ArgumentError, "BUG: block must have just one argument" unless block.arity == 1
 
@@ -83,11 +91,14 @@ module Fluent
         when :tcp
           server = server_create_for_tcp_connection(shared, bind, port, backlog, socket_option_setter, &block)
         when :tls
-          raise ArgumentError, "BUG: certopts (certificate options) not specified for TLS" unless certopts
-          # server_certopts_validate!(certopts)
-          # sock = server_create_tls_socket(shared, bind, port)
-          # server = nil # ...
-          raise "not implemented yet"
+          transport_config = if tls_options
+                               server_create_transport_section_object(tls_options)
+                             elsif @transport_config && @transport_config.protocol == :tls
+                               @transport_config
+                             else
+                               raise ArgumentError, "BUG: TLS transport specified, but certification options are not specified"
+                             end
+          server = server_create_for_tls_connection(shared, bind, port, transport_config, backlog, socket_option_setter, &block)
         when :unix
           raise "not implemented yet"
         else
@@ -108,12 +119,15 @@ module Fluent
       #   sock.remote_port
       #   # ...
       # end
-      def server_create(title, port, proto: :tcp, bind: '0.0.0.0', shared: true, socket: nil, backlog: nil, max_bytes: nil, flags: 0, **socket_options, &callback)
+      def server_create(title, port, proto: nil, bind: '0.0.0.0', shared: true, socket: nil, backlog: nil, tls_options: nil, max_bytes: nil, flags: 0, **socket_options, &callback)
+        proto ||= (@transport_config && @transport_config.protocol == :tls) ? :tls : :tcp
+
         raise ArgumentError, "BUG: title must be a symbol" unless title && title.is_a?(Symbol)
         raise ArgumentError, "BUG: port must be an integer" unless port && port.is_a?(Integer)
         raise ArgumentError, "BUG: invalid protocol name" unless PROTOCOLS.include?(proto)
 
         raise ArgumentError, "BUG: socket option is available only for udp" if socket && proto != :udp
+        raise ArgumentError, "BUG: tls_options is available only for tls" if tls_options && proto != :tls
 
         raise ArgumentError, "BUG: block not specified which handles received data" unless block_given?
         raise ArgumentError, "BUG: block must have 1 or 2 arguments" unless callback.arity == 1 || callback.arity == 2
@@ -141,7 +155,16 @@ module Fluent
             conn.data(&callback)
           end
         when :tls
-          raise "not implemented yet"
+          transport_config = if tls_options
+                               server_create_transport_section_object(tls_options)
+                             elsif @transport_config && @transport_config.protocol == :tls
+                               @transport_config
+                             else
+                               raise ArgumentError, "BUG: TLS transport specified, but certification options are not specified"
+                             end
+          server = server_create_for_tls_connection(shared, bind, port, transport_config, backlog, socket_option_setter) do |conn|
+            conn.data(&callback)
+          end
         when :udp
           raise ArgumentError, "BUG: max_bytes must be specified for UDP" unless max_bytes
           if socket
@@ -198,6 +221,80 @@ module Fluent
         server
       end
 
+      def server_create_for_tls_connection(shared, bind, port, conf, backlog, socket_option_setter, &block)
+        context = cert_option_create_context(conf.version, conf.insecure, conf.ciphers, conf)
+        sock = server_create_tcp_socket(shared, bind, port)
+        socket_option_setter.call(sock)
+        close_callback = ->(conn){ @_server_mutex.synchronize{ @_server_connections.delete(conn) } }
+        server = Coolio::TCPServer.new(sock, nil, EventHandler::TLSServer, context, socket_option_setter, close_callback, @log, @under_plugin_development, block) do |conn|
+          @_server_mutex.synchronize do
+            @_server_connections << conn
+          end
+        end
+        server.listen(backlog) if backlog
+        server
+      end
+
+      SERVER_TRANSPORT_PARAMS = [
+        :protocol, :version, :ciphers, :insecure,
+        :cert_path, :private_key_path, :private_key_passphrase,
+        :ca_cert_path, :ca_private_key_path, :ca_private_key_passphrase,
+        :generate_private_key_length,
+        :generate_cert_country, :generate_cert_state, :generate_cert_state,
+        :generate_cert_locality, :generate_cert_common_name,
+        :generate_cert_expiration, :generate_cert_digest,
+      ]
+
+      def server_create_transport_section_object(opts)
+        transport_section = configured_section_create(:transport)
+        SERVER_TRANSPORT_PARAMS.each do |param|
+          if opts.has_key?(param)
+            transport_section[param] = opts[param]
+          end
+        end
+        transport_section
+      end
+
+      module ServerTransportParams
+        TLS_DEFAULT_VERSION = :'TLSv1_2'
+        TLS_SUPPORTED_VERSIONS = [:'TLSv1_1', :'TLSv1_2']
+        ### follow httpclient configuration by nahi
+        # OpenSSL 0.9.8 default: "ALL:!ADH:!LOW:!EXP:!MD5:+SSLv2:@STRENGTH"
+        CIPHERS_DEFAULT = "ALL:!aNULL:!eNULL:!SSLv2" # OpenSSL >1.0.0 default
+
+        include Fluent::Configurable
+        config_section :transport, required: false, multi: false, init: true, param_name: :transport_config do
+          config_argument :protocol, :enum, list: [:tcp, :tls], default: :tcp
+          config_param :version, :enum, list: TLS_SUPPORTED_VERSIONS, default: TLS_DEFAULT_VERSION
+
+          config_param :ciphers, :string, default: CIPHERS_DEFAULT
+          config_param :insecure, :bool, default: false
+
+          # Cert signed by public CA
+          config_param :cert_path, :string, default: nil
+          config_param :private_key_path, :string, default: nil
+          config_param :private_key_passphrase, :string, default: nil, secret: true
+
+          # Cert generated and signed by private CA Certificate
+          config_param :ca_cert_path, :string, default: nil
+          config_param :ca_private_key_path, :string, default: nil
+          config_param :ca_private_key_passphrase, :string, default: nil, secret: true
+
+          # Options for generating certs by private CA certs or self-signed
+          config_param :generate_private_key_length, :integer, default: 2048
+          config_param :generate_cert_country, :string, default: 'US'
+          config_param :generate_cert_state, :string, default: 'CA'
+          config_param :generate_cert_locality, :string, default: 'Mountain View'
+          config_param :generate_cert_common_name, :string, default: nil
+          config_param :generate_cert_expiration, :time, default: 10 * 365 * 86400 # 10years later
+          config_param :generate_cert_digest, :enum, list: [:sha1, :sha256, :sha384, :sha512], default: :sha256
+        end
+      end
+
+      def self.included(mod)
+        mod.include ServerTransportParams
+      end
+
       def initialize
         super
         @_servers = []
@@ -205,28 +302,33 @@ module Fluent
         @_server_mutex = Mutex.new
       end
 
-      def shutdown
-        @_server_connections.each do |conn|
-          conn.close rescue nil
+      def configure(conf)
+        super
+
+        if @transport_config
+          if @transport_config.protocol == :tls
+            cert_option_server_validate!(@transport_config)
+          end
         end
+      end
+
+      def stop
         @_server_mutex.synchronize do
           @_servers.each do |si|
             si.server.detach if si.server.attached?
+            # to refuse more connections: (connected sockets are still alive here)
+            si.server.close rescue nil
           end
         end
 
         super
       end
 
-      def close
+      def shutdown
         @_server_connections.each do |conn|
           conn.close rescue nil
         end
-        @_server_mutex.synchronize do
-          @_servers.each do |si|
-            si.server.close rescue nil
-          end
-        end
+
         super
       end
 
@@ -235,10 +337,6 @@ module Fluent
         super
       end
 
-      def server_certopts_validate!(certopts)
-        raise "not implemented yet"
-      end
-
       def server_socket_manager_client
         socket_manager_path = ENV['SERVERENGINE_SOCKETMANAGER_PATH']
         if Fluent.windows?
@@ -272,28 +370,25 @@ module Fluent
         sock
       end
 
-      def server_create_tls_socket(shared, bind, port)
-        raise "not implemented yet"
-      end
-
       class CallbackSocket
         def initialize(server_type, sock, enabled_events = [], close_socket: true)
           @server_type = server_type
           @sock = sock
+          @peeraddr = nil
           @enabled_events = enabled_events
           @close_socket = close_socket
         end
 
         def remote_addr
-          @sock.peeraddr[3]
+          @peeraddr[3]
         end
 
         def remote_host
-          @sock.peeraddr[2]
+          @peeraddr[2]
         end
 
         def remote_port
-          @sock.peeraddr[1]
+          @peeraddr[1]
         end
 
         def send(data, flags = 0)
@@ -332,6 +427,18 @@ module Fluent
       class TCPCallbackSocket < CallbackSocket
         def initialize(sock)
           super("tcp", sock, [:data, :write_complete, :close])
+          @peeraddr = @sock.peeraddr
+        end
+
+        def write(data)
+          @sock.write(data)
+        end
+      end
+
+      class TLSCallbackSocket < CallbackSocket
+        def initialize(sock)
+          super("tls", sock, [:data, :write_complete, :close])
+          @peeraddr = @sock.to_io.peeraddr
         end
 
         def write(data)
@@ -435,6 +542,10 @@ module Fluent
             @mutex = Mutex.new # to serialize #write and #close
           end
 
+          def to_io
+            @_handler_socket
+          end
+
           def data(&callback)
             raise "data callback can be registered just once, but registered twice" if self.singleton_methods.include?(:on_read)
             @data_callback = callback
@@ -464,7 +575,140 @@ module Fluent
           def on_read_without_connection(data)
             @data_callback.call(data)
           rescue => e
-            @log.error "unexpected error on reading data", host: remote_host, port: remote_port, error: e
+            @log.error "unexpected error on reading data", host: @callback_connection.remote_host, port: @callback_connection.remote_port, error: e
+            @log.error_backtrace
+            close(true) rescue nil
+            raise if @under_plugin_development
+          end
+
+          def on_read_with_connection(data)
+            @data_callback.call(data, @callback_connection)
+          rescue => e
+            @log.error "unexpected error on reading data", host: @callback_connection.remote_host, port: @callback_connection.remote_port, error: e
+            @log.error_backtrace
+            close(true) rescue nil
+            raise if @under_plugin_development
+          end
+
+          def close
+            @mutex.synchronize do
+              return if @closing
+              @closing = true
+              @close_callback.call(self)
+              super
+            end
+          end
+        end
+
+        class TLSServer < Coolio::Socket
+          # It can't use Coolio::TCPSocket, because Coolio::TCPSocket checks that underlying socket (1st argument of super) is TCPSocket.
+          def initialize(sock, context, socket_option_setter, close_callback, log, under_plugin_development, connect_callback)
+            raise ArgumentError, "socket must be a TCPSocket: sock=#{sock}" unless sock.is_a?(TCPSocket)
+
+            socket_option_setter.call(sock)
+            @_handler_socket = OpenSSL::SSL::SSLSocket.new(sock, context)
+            @_handler_socket.sync_close = true
+            @_handler_write_buffer = ''.force_encoding('ascii-8bit')
+            @_handler_accepted = false
+            super(@_handler_socket)
+
+            @log = log
+            @under_plugin_development = under_plugin_development
+
+            @connect_callback = connect_callback
+            @data_callback = nil
+            @close_callback = close_callback
+
+            @callback_connection = nil
+            @closing = false
+
+            @mutex = Mutex.new # to serialize #write and #close
+          end
+
+          def to_io
+            @_handler_socket.to_io
+          end
+
+          def data(&callback)
+            raise "data callback can be registered just once, but registered twice" if self.singleton_methods.include?(:on_read)
+            @data_callback = callback
+            on_read_impl = case callback.arity
+                           when 1 then :on_read_without_connection
+                           when 2 then :on_read_with_connection
+                           else
+                             raise "BUG: callback block must have 1 or 2 arguments"
+                           end
+            self.define_singleton_method(:on_read, method(on_read_impl))
+          end
+
+          def write(data)
+            @mutex.synchronize do
+              @_handler_write_buffer << data
+              schedule_write
+              data.bytesize
+            end
+          end
+
+          def try_tls_accept
+            return true if @_handler_accepted
+
+            begin
+              @_handler_socket.accept_nonblock # this method call actually try to do handshake via TLS
+              @_handler_accepted = true
+
+              @callback_connection = TLSCallbackSocket.new(self)
+              @connect_callback.call(@callback_connection)
+              unless @data_callback
+                raise "connection callback must call #data to set data callback"
+              end
+              return true
+
+            rescue IO::WaitReadable, IO::WaitWritable
+              # retry accept_nonblock: there aren't enough data in underlying socket buffer
+            rescue OpenSSL::SSL::SSLError => e
+              @log.trace "unexpected error before accepting TLS connection", error: e
+              close rescue nil
+            end
+
+            false
+          end
+
+          def on_connect
+            try_tls_accept
+          end
+
+          def on_readable
+            if try_tls_accept
+              super
+            end
+          rescue IO::WaitReadable, IO::WaitWritable
+            # ignore and return with doing nothing
+          end
+
+          def on_writable
+            begin
+              @mutex.synchronize do
+                written_bytes = @_handler_socket.write_nonblock(@_handler_write_buffer)
+                @_handler_write_buffer.slice!(0, written_bytes)
+                super
+              end
+            rescue IO::WaitWritable, IO::WaitReadable
+              return
+            rescue Errno::EINTR
+              return
+            rescue SystemCallError, IOError, SocketError
+              # SystemCallError catches Errno::EPIPE & Errno::ECONNRESET amongst others.
+              close rescue nil
+              return
+            rescue OpenSSL::SSL::SSLError => e
+              @log.debug "unexpected SSLError while writing data into socket connected via TLS", error: e
+            end
+          end
+
+          def on_read_without_connection(data)
+            @data_callback.call(data)
+          rescue => e
+            @log.error "unexpected error on reading data", host: @callback_connection.remote_host, port: @callback_connection.remote_port, error: e
             @log.error_backtrace
             close(true) rescue nil
             raise if @under_plugin_development
@@ -473,7 +717,7 @@ module Fluent
           def on_read_with_connection(data)
             @data_callback.call(data, @callback_connection)
           rescue => e
-            @log.error "unexpected error on reading data", host: remote_host, port: remote_port, error: e
+            @log.error "unexpected error on reading data", host: @callback_connection.remote_host, port: @callback_connection.remote_port, error: e
             @log.error_backtrace
             close(true) rescue nil
             raise if @under_plugin_development