fluentd 0.14.7-x86-mingw32 → 0.14.10-x86-mingw32

data/lib/fluent/plugin_helper/compat_parameters.rb

@@ -48,10 +48,19 @@ module Fluent
       }
 
       PARSER_PARAMS = {
-        "format" => "@type",
+        "format" => nil,
+        "types" => nil,
+        "types_delimiter" => nil,
+        "types_label_delimiter" => nil,
+        "keys" => "keys", # CSVParser, TSVParser (old ValuesParser)
         "time_key"    => "time_key",
         "time_format" => "time_format",
+        "localtim" => nil,
+        "utc" => nil,
         "delimiter"   => "delimiter",
+        "keep_time_key" => "keep_time_key",
+        "null_empty_string" => "null_empty_string",
+        "null_value_pattern" => "null_value_pattern",
         "json_parser"      => "json_parser", # JSONParser
         "label_delimiter"  => "label_delimiter", # LabeledTSVParser
         "format_firstline" => "format_firstline", # MultilineParser
@@ -62,19 +71,29 @@ module Fluent
 
       INJECT_PARAMS = {
         "include_time_key" => nil,
-        "time_key"      => "time_key",
-        "time_format"   => "time_format",
-        "timezone"      => "timezone",
+        "time_key"    => "time_key",
+        "time_format" => "time_format",
+        "timezone"    => "timezone",
         "include_tag_key" => nil,
         "tag_key" => "tag_key",
         "localtime" => nil,
         "utc" => nil,
       }
 
+      EXTRACT_PARAMS = {
+        "time_key"    => "time_key",
+        "time_format" => "time_format",
+        "timezone"    => "timezone",
+        "tag_key" => "tag_key",
+        "localtime" => nil,
+        "utc" => nil,
+      }
+
       FORMATTER_PARAMS = {
         "format" => "@type",
         "delimiter" => "delimiter",
         "force_quotes" => "force_quotes", # CsvFormatter
+        "keys" => "keys", # TSVFormatter
         "fields" => "fields", # CsvFormatter
         "json_parser" => "json_parser", # JSONFormatter
         "label_delimiter" => "label_delimiter", # LabeledTSVFormatter
@@ -95,6 +114,8 @@ module Fluent
             compat_parameters_buffer(conf, **kwargs)
           when :inject
             compat_parameters_inject(conf)
+          when :extract
+            compat_parameters_extract(conf)
           when :parser
             compat_parameters_parser(conf)
           when :formatter
@@ -167,6 +188,7 @@ module Fluent
           hash['time_type'] ||= 'string'
         end
         if conf.has_key?('time_as_epoch') && Fluent::Config.bool_value(conf['time_as_epoch'])
+          hash['time_key'] ||= 'time'
           hash['time_type'] = 'unixtime'
         end
         if conf.has_key?('localtime') || conf.has_key?('utc')
@@ -193,6 +215,40 @@ module Fluent
         conf
       end
 
+      def compat_parameters_extract(conf)
+        return unless conf.elements('extract').empty?
+        return if EXTRACT_PARAMS.keys.all?{|k| !conf.has_key?(k) } && !conf.has_key?('format')
+
+        # TODO: warn obsolete parameters if these are deprecated
+        hash = compat_parameters_copy_to_subsection_attributes(conf, EXTRACT_PARAMS)
+
+        if conf.has_key?('time_as_epoch') && Fluent::Config.bool_value(conf['time_as_epoch'])
+          hash['time_key'] ||= 'time'
+          hash['time_type'] = 'unixtime'
+        elsif conf.has_key?('format') && conf["format"].start_with?("/") && conf["format"].end_with?("/") # old-style regexp parser
+          hash['time_key'] ||= 'time'
+          hash['time_type'] ||= 'string'
+        end
+        if conf.has_key?('localtime') || conf.has_key?('utc')
+          if conf.has_key?('localtime') && conf.has_key?('utc')
+            raise Fluent::ConfigError, "both of utc and localtime are specified, use only one of them"
+          elsif conf.has_key?('localtime')
+            hash['localtime'] = Fluent::Config.bool_value(conf['localtime'])
+          elsif conf.has_key?('utc')
+            hash['localtime'] = !(Fluent::Config.bool_value(conf['utc']))
+            # Specifying "localtime false" means using UTC in TimeFormatter
+            # And specifying "utc" is different from specifying "timezone +0000"(it's not always UTC).
+            # There are difference between "Z" and "+0000" in timezone formatting.
+            # TODO: add kwargs to TimeFormatter to specify "using localtime", "using UTC" or "using specified timezone" in more explicit way
+          end
+        end
+
+        e = Fluent::Config::Element.new('extract', '', hash, [])
+        conf.elements << e
+
+        conf
+      end
+
       def compat_parameters_parser(conf)
         return unless conf.elements('parse').empty?
         return if PARSER_PARAMS.keys.all?{|k| !conf.has_key?(k) }
@@ -200,6 +256,39 @@ module Fluent
         # TODO: warn obsolete parameters if these are deprecated
         hash = compat_parameters_copy_to_subsection_attributes(conf, PARSER_PARAMS)
 
+        if conf["format"]
+          if conf["format"].start_with?("/") && conf["format"].end_with?("/")
+            hash["@type"] = "regexp"
+            hash["expression"] = conf["format"][1..-2]
+          else
+            hash["@type"] = conf["format"]
+          end
+        end
+
+        if conf["types"]
+          delimiter = conf["types_delimiter"] || ','
+          label_delimiter = conf["types_label_delimiter"] || ':'
+          types = {}
+          conf['types'].split(delimiter).each do |pair|
+            key, value = pair.split(label_delimiter, 2)
+            types[key] = value
+          end
+          hash["types"] = JSON.dump(types)
+        end
+        if conf.has_key?('localtime') || conf.has_key?('utc')
+          if conf.has_key?('localtime') && conf.has_key?('utc')
+            raise Fluent::ConfigError, "both of utc and localtime are specified, use only one of them"
+          elsif conf.has_key?('localtime')
+            hash['localtime'] = Fluent::Config.bool_value(conf['localtime'])
+          elsif conf.has_key?('utc')
+            hash['localtime'] = !(Fluent::Config.bool_value(conf['utc']))
+            # Specifying "localtime false" means using UTC in TimeFormatter
+            # And specifying "utc" is different from specifying "timezone +0000"(it's not always UTC).
+            # There are difference between "Z" and "+0000" in timezone formatting.
+            # TODO: add kwargs to TimeFormatter to specify "using localtime", "using UTC" or "using specified timezone" in more explicit way
+          end
+        end
+
         e = Fluent::Config::Element.new('parse', '', hash, [])
         conf.elements << e
 

data/lib/fluent/plugin_helper/event_emitter.rb

@@ -26,10 +26,14 @@ module Fluent
 
       def router
         @_event_emitter_used_actually = true
+        if @_event_emitter_lazy_init
+          @router = @primary_instance.router
+        end
         @router
       end
 
       def router=(r)
+        # not recommended now...
         @router = r
       end
 
@@ -44,14 +48,23 @@ module Fluent
       def event_emitter_router(label_name)
         if label_name
           Engine.root_agent.find_label(label_name).event_router
+        elsif self.respond_to?(:as_secondary) && self.as_secondary
+          if @primary_instance.has_router?
+            @_event_emitter_lazy_init = true
+            nil # primary plugin's event router is not initialized yet, here.
+          else
+            @primary_instance.context_router
+          end
         else
-          Engine.root_agent.event_router
+          # `Engine.root_agent.event_router` is for testing
+          self.context_router || Engine.root_agent.event_router
         end
       end
 
       def initialize
         super
         @_event_emitter_used_actually = false
+        @_event_emitter_lazy_init = false
         @router = nil
       end
 

data/lib/fluent/plugin_helper/event_loop.rb

@@ -30,12 +30,16 @@ module Fluent
       # terminate: initialize internal state
 
       EVENT_LOOP_RUN_DEFAULT_TIMEOUT = 0.5
+      EVENT_LOOP_SHUTDOWN_TIMEOUT = 5
+      EVENT_LOOP_CLOCK_ID = Process::CLOCK_MONOTONIC_RAW rescue Process::CLOCK_MONOTONIC
 
       attr_reader :_event_loop # for tests
 
       def event_loop_attach(watcher)
         @_event_loop_mutex.synchronize do
           @_event_loop.attach(watcher)
+          @_event_loop_attached_watchers << watcher
+          watcher
         end
       end
 
@@ -58,6 +62,7 @@ module Fluent
         @_event_loop_mutex = Mutex.new
         # plugin MAY configure loop run timeout in #configure
         @_event_loop_run_timeout = EVENT_LOOP_RUN_DEFAULT_TIMEOUT
+        @_event_loop_attached_watchers = []
       end
 
       def start
@@ -65,19 +70,32 @@ module Fluent
 
         # event loop does not run here, so mutex lock is not required
         thread_create :event_loop do
-          default_watcher = DefaultWatcher.new
-          @_event_loop.attach(default_watcher)
-          @_event_loop_running = true
-          @_event_loop.run(@_event_loop_run_timeout) # this method blocks
-          @_event_loop_running = false
+          begin
+            default_watcher = DefaultWatcher.new
+            event_loop_attach(default_watcher)
+            @_event_loop_running = true
+            @_event_loop.run(@_event_loop_run_timeout) # this method blocks
+          ensure
+            @_event_loop_running = false
+          end
         end
       end
 
       def shutdown
         @_event_loop_mutex.synchronize do
-          @_event_loop.watchers.each {|w| w.detach if w.attached? }
+          @_event_loop_attached_watchers.reverse.each do |w|
+            if w.attached?
+              w.detach
+            end
+          end
         end
+        timeout_at = Process.clock_gettime(EVENT_LOOP_CLOCK_ID) + EVENT_LOOP_SHUTDOWN_TIMEOUT
         while @_event_loop_running
+          if Process.clock_gettime(EVENT_LOOP_CLOCK_ID) >= timeout_at
+            log.warn "event loop does NOT exit until hard timeout."
+            raise "event loop does NOT exit until hard timeout." if @under_plugin_development
+            break
+          end
           sleep 0.1
         end
 

data/lib/fluent/plugin_helper/extract.rb

@@ -25,7 +25,8 @@ module Fluent
         return nil unless @_extract_enabled
 
         if @_extract_tag_key && record.has_key?(@_extract_tag_key)
-          return record[@_extract_tag_key].to_s
+          v = @_extract_keep_tag_key ? record[@_extract_tag_key] : record.delete(@_extract_tag_key)
+          return v.to_s
         end
 
         nil
@@ -35,7 +36,8 @@ module Fluent
         return nil unless @_extract_enabled
 
         if @_extract_time_key && record.has_key?(@_extract_time_key)
-          return @_extract_time_parser.call(record[@_extract_time_key])
+          v = @_extract_keep_time_key ? record[@_extract_time_key] : record.delete(@_extract_time_key)
+          return @_extract_time_parser.call(v)
         end
 
         nil
@@ -45,7 +47,9 @@ module Fluent
         include Fluent::Configurable
         config_section :extract, required: false, multi: false, param_name: :extract_config do
           config_param :tag_key, :string, default: nil
+          config_param :keep_tag_key, :bool, default: false
           config_param :time_key, :string, default: nil
+          config_param :keep_time_key, :bool, default: false
 
           # To avoid defining :time_type twice
           config_param :time_type, :enum, list: [:float, :unixtime, :string], default: :float
@@ -64,7 +68,9 @@ module Fluent
         super
         @_extract_enabled = false
         @_extract_tag_key = nil
+        @_extract_keep_tag_key = nil
         @_extract_time_key = nil
+        @_extract_keep_time_key = nil
         @_extract_time_parser = nil
       end
 
@@ -73,15 +79,21 @@ module Fluent
 
         if @extract_config
           @_extract_tag_key = @extract_config.tag_key
+          @_extract_keep_tag_key = @extract_config.keep_tag_key
           @_extract_time_key = @extract_config.time_key
           if @_extract_time_key
+            @_extract_keep_time_key = @extract_config.keep_time_key
             @_extract_time_parser = case @extract_config.time_type
-                                    when :float then ->(v){ Fluent::EventTime.new(v.to_i, ((v.to_f - v.to_i) * 1_000_000_000).to_i) }
-                                    when :unixtime then ->(v){ Fluent::EventTime.new(v.to_i, 0) }
+                                    when :float then Fluent::NumericTimeParser.new(:float)
+                                    when :unixtime then Fluent::NumericTimeParser.new(:unixtime)
                                     else
                                       localtime = @extract_config.localtime && !@extract_config.utc
                                       Fluent::TimeParser.new(@extract_config.time_format, localtime, @extract_config.timezone)
                                     end
+          else
+            if @extract_config.time_format
+              log.warn "'time_format' specified without 'time_key', will be ignored"
+            end
           end
 
           @_extract_enabled = @_extract_tag_key || @_extract_time_key

data/lib/fluent/plugin_helper/formatter.rb

@@ -24,7 +24,7 @@ module Fluent
     module Formatter
       def formatter_create(usage: '', type: nil, conf: nil, default_type: nil)
         formatter = @_formatters[usage]
-        return formatter if formatter
+        return formatter if formatter && !type && !conf
 
         type = if type
                  type
@@ -61,9 +61,9 @@ module Fluent
       module FormatterParams
         include Fluent::Configurable
         # minimum section definition to instantiate formatter plugin instances
-        config_section :format, required: false, multi: true, param_name: :formatter_configs do
+        config_section :format, required: false, multi: true, init: true, param_name: :formatter_configs do
           config_argument :usage, :string, default: ''
-          config_param    :@type, :string
+          config_param    :@type, :string # config_set_default required for :@type
         end
       end
 
@@ -82,15 +82,13 @@ module Fluent
       def configure(conf)
         super
 
-        if @formatter_configs
-          @formatter_configs.each do |section|
-            if @_formatters[section.usage]
-              raise Fluent::ConfigError, "duplicated formatter configured: #{section.usage}"
-            end
-            formatter = Plugin.new_formatter(section[:@type], parent: self)
-            formatter.configure(section.corresponding_config_element)
-            @_formatters[section.usage] = formatter
+        @formatter_configs.each do |section|
+          if @_formatters[section.usage]
+            raise Fluent::ConfigError, "duplicated formatter configured: #{section.usage}"
           end
+          formatter = Plugin.new_formatter(section[:@type], parent: self)
+          formatter.configure(section.corresponding_config_element)
+          @_formatters[section.usage] = formatter
         end
       end
 

data/lib/fluent/plugin_helper/inject.rb

@@ -97,9 +97,20 @@ module Fluent
         if @inject_config
           @_inject_hostname_key = @inject_config.hostname_key
           if @_inject_hostname_key
+            if self.respond_to?(:buffer_config)
+              # Output plugin cannot use "hostname"(specified by @hostname_key),
+              # injected by this plugin helper, in chunk keys.
+              # This plugin helper works in `#format` (in many cases), but modified record
+              # don't have any side effect in chunking of output plugin.
+              if self.buffer_config.chunk_keys.include?(@_inject_hostname_key)
+                log.error "Use filters to inject hostname to use it in buffer chunking."
+                raise Fluent::ConfigError, "the key specified by 'hostname_key' in <inject> cannot be used in buffering chunk key."
+              end
+            end
+
             @_inject_hostname =  @inject_config.hostname
             unless @_inject_hostname
-              @_inject_hostname = Socket.gethostname
+              @_inject_hostname = ::Socket.gethostname
               log.info "using hostname for specified field", host_key: @_inject_hostname_key, host_name: @_inject_hostname
             end
           end
@@ -113,6 +124,10 @@ module Fluent
                                         localtime = @inject_config.localtime && !@inject_config.utc
                                         Fluent::TimeFormatter.new(@inject_config.time_format, localtime, @inject_config.timezone)
                                       end
+          else
+            if @inject_config.time_format
+              log.warn "'time_format' specified without 'time_key', will be ignored"
+            end
           end
 
           @_inject_enabled = @_inject_hostname_key || @_inject_tag_key || @_inject_time_key

data/lib/fluent/plugin_helper/parser.rb

@@ -24,7 +24,7 @@ module Fluent
     module Parser
       def parser_create(usage: '', type: nil, conf: nil, default_type: nil)
         parser = @_parsers[usage]
-        return parser if parser
+        return parser if parser && !type && !conf
 
         type = if type
                  type
@@ -61,9 +61,9 @@ module Fluent
       module ParserParams
         include Fluent::Configurable
         # minimum section definition to instantiate parser plugin instances
-        config_section :parse, required: false, multi: true, param_name: :parser_configs do
+        config_section :parse, required: false, multi: true, init: true, param_name: :parser_configs do
           config_argument :usage, :string, default: ''
-          config_param    :@type, :string
+          config_param    :@type, :string # config_set_default required for :@type
         end
       end
 

data/lib/fluent/plugin_helper/server.rb

@@ -0,0 +1,494 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin_helper/event_loop'
+
+require 'serverengine'
+require 'cool.io'
+require 'socket'
+require 'ipaddr'
+require 'fcntl'
+
+require_relative 'socket_option'
+
+module Fluent
+  module PluginHelper
+    module Server
+      include Fluent::PluginHelper::EventLoop
+      include Fluent::PluginHelper::SocketOption
+
+      # This plugin helper doesn't support these things for now:
+      # * SSL/TLS (TBD)
+      # * TCP/TLS keepalive
+
+      # stop     : [-]
+      # shutdown : detach server event handler from event loop (event_loop)
+      # close    : close listening sockets
+      # terminate: remote all server instances
+
+      attr_reader :_servers # for tests
+
+      def server_wait_until_start
+        # event_loop_wait_until_start works well for this
+      end
+
+      def server_wait_until_stop
+        sleep 0.1 while @_servers.any?{|si| si.server.attached? }
+        @_servers.each{|si| si.server.close rescue nil }
+      end
+
+      PROTOCOLS = [:tcp, :udp, :tls, :unix]
+      CONNECTION_PROTOCOLS = [:tcp, :tls, :unix]
+
+      # server_create_connection(:title, @port) do |conn|
+      #   # on connection
+      #   source_addr = conn.remote_host
+      #   source_port = conn.remote_port
+      #   conn.data do |data|
+      #     # on data
+      #     conn.write resp # ...
+      #     conn.close
+      #   end
+      # end
+      def server_create_connection(title, port, proto: :tcp, bind: '0.0.0.0', shared: true, backlog: nil, **socket_options, &block)
+        raise ArgumentError, "BUG: title must be a symbol" unless title && title.is_a?(Symbol)
+        raise ArgumentError, "BUG: port must be an integer" unless port && port.is_a?(Integer)
+        raise ArgumentError, "BUG: invalid protocol name" unless PROTOCOLS.include?(proto)
+        raise ArgumentError, "BUG: cannot create connection for UDP" unless CONNECTION_PROTOCOLS.include?(proto)
+
+        raise ArgumentError, "BUG: block not specified which handles connection" unless block_given?
+        raise ArgumentError, "BUG: block must have just one argument" unless block.arity == 1
+
+        if proto == :tcp || proto == :tls # default linger_timeout only for server
+          socket_options[:linger_timeout] ||= 0
+        end
+
+        socket_option_validate!(proto, **socket_options)
+        socket_option_setter = ->(sock){ socket_option_set(sock, **socket_options) }
+
+        case proto
+        when :tcp
+          server = server_create_for_tcp_connection(shared, bind, port, backlog, socket_option_setter, &block)
+        when :tls
+          raise ArgumentError, "BUG: certopts (certificate options) not specified for TLS" unless certopts
+          # server_certopts_validate!(certopts)
+          # sock = server_create_tls_socket(shared, bind, port)
+          # server = nil # ...
+          raise "not implemented yet"
+        when :unix
+          raise "not implemented yet"
+        else
+          raise "unknown protocol #{proto}"
+        end
+
+        server_attach(title, proto, port, bind, shared, server)
+      end
+
+      # server_create(:title, @port) do |data|
+      #   # ...
+      # end
+      # server_create(:title, @port) do |data, conn|
+      #   # ...
+      # end
+      # server_create(:title, @port, proto: :udp, max_bytes: 2048) do |data, sock|
+      #   sock.remote_host
+      #   sock.remote_port
+      #   # ...
+      # end
+      def server_create(title, port, proto: :tcp, bind: '0.0.0.0', shared: true, socket: nil, backlog: nil, max_bytes: nil, flags: 0, **socket_options, &callback)
+        raise ArgumentError, "BUG: title must be a symbol" unless title && title.is_a?(Symbol)
+        raise ArgumentError, "BUG: port must be an integer" unless port && port.is_a?(Integer)
+        raise ArgumentError, "BUG: invalid protocol name" unless PROTOCOLS.include?(proto)
+
+        raise ArgumentError, "BUG: socket option is available only for udp" if socket && proto != :udp
+
+        raise ArgumentError, "BUG: block not specified which handles received data" unless block_given?
+        raise ArgumentError, "BUG: block must have 1 or 2 arguments" unless callback.arity == 1 || callback.arity == 2
+
+        if proto == :tcp || proto == :tls # default linger_timeout only for server
+          socket_options[:linger_timeout] ||= 0
+        end
+
+        unless socket
+          socket_option_validate!(proto, **socket_options)
+          socket_option_setter = ->(sock){ socket_option_set(sock, **socket_options) }
+        end
+
+        if proto != :tcp && proto != :tls && proto != :unix # options to listen/accept connections
+          raise ArgumentError, "BUG: backlog is available for tcp/tls" if backlog
+        end
+        if proto != :udp # UDP options
+          raise ArgumentError, "BUG: max_bytes is available only for udp" if max_bytes
+          raise ArgumentError, "BUG: flags is available only for udp" if flags != 0
+        end
+
+        case proto
+        when :tcp
+          server = server_create_for_tcp_connection(shared, bind, port, backlog, socket_option_setter) do |conn|
+            conn.data(&callback)
+          end
+        when :tls
+          raise "not implemented yet"
+        when :udp
+          raise ArgumentError, "BUG: max_bytes must be specified for UDP" unless max_bytes
+          if socket
+            sock = socket
+            close_socket = false
+          else
+            sock = server_create_udp_socket(shared, bind, port)
+            socket_option_setter.call(sock)
+            close_socket = true
+          end
+          server = EventHandler::UDPServer.new(sock, max_bytes, flags, close_socket, @log, @under_plugin_development, &callback)
+        when :unix
+          raise "not implemented yet"
+        else
+          raise "BUG: unknown protocol #{proto}"
+        end
+
+        server_attach(title, proto, port, bind, shared, server)
+      end
+
+      def server_create_tcp(title, port, **kwargs, &callback)
+        server_create(title, port, proto: :tcp, **kwargs, &callback)
+      end
+
+      def server_create_udp(title, port, **kwargs, &callback)
+        server_create(title, port, proto: :udp, **kwargs, &callback)
+      end
+
+      def server_create_tls(title, port, **kwargs, &callback)
+        server_create(title, port, proto: :tls, **kwargs, &callback)
+      end
+
+      def server_create_unix(title, port, **kwargs, &callback)
+        server_create(title, port, proto: :unix, **kwargs, &callback)
+      end
+
+      ServerInfo = Struct.new(:title, :proto, :port, :bind, :shared, :server)
+
+      def server_attach(title, proto, port, bind, shared, server)
+        @_servers << ServerInfo.new(title, proto, port, bind, shared, server)
+        event_loop_attach(server)
+      end
+
+      def server_create_for_tcp_connection(shared, bind, port, backlog, socket_option_setter, &block)
+        sock = server_create_tcp_socket(shared, bind, port)
+        socket_option_setter.call(sock)
+        close_callback = ->(conn){ @_server_mutex.synchronize{ @_server_connections.delete(conn) } }
+        server = Coolio::TCPServer.new(sock, nil, EventHandler::TCPServer, socket_option_setter, close_callback, @log, @under_plugin_development, block) do |conn|
+          @_server_mutex.synchronize do
+            @_server_connections << conn
+          end
+        end
+        server.listen(backlog) if backlog
+        server
+      end
+
+      def initialize
+        super
+        @_servers = []
+        @_server_connections = []
+        @_server_mutex = Mutex.new
+      end
+
+      def shutdown
+        @_server_connections.each do |conn|
+          conn.close rescue nil
+        end
+        @_server_mutex.synchronize do
+          @_servers.each do |si|
+            si.server.detach if si.server.attached?
+          end
+        end
+
+        super
+      end
+
+      def close
+        @_server_connections.each do |conn|
+          conn.close rescue nil
+        end
+        @_server_mutex.synchronize do
+          @_servers.each do |si|
+            si.server.close rescue nil
+          end
+        end
+        super
+      end
+
+      def terminate
+        @_servers = []
+        super
+      end
+
+      def server_certopts_validate!(certopts)
+        raise "not implemented yet"
+      end
+
+      def server_socket_manager_client
+        socket_manager_path = ENV['SERVERENGINE_SOCKETMANAGER_PATH']
+        if Fluent.windows?
+          socket_manager_path = socket_manager_path.to_i
+        end
+        ServerEngine::SocketManager::Client.new(socket_manager_path)
+      end
+
+      def server_create_tcp_socket(shared, bind, port)
+        sock = if shared
+                 server_socket_manager_client.listen_tcp(bind, port)
+               else
+                 TCPServer.new(bind, port) # this method call can create sockets for AF_INET6
+               end
+        # close-on-exec is set by default in Ruby 2.0 or later (, and it's unavailable on Windows)
+        sock.fcntl(Fcntl::F_SETFL, Fcntl::O_NONBLOCK) # nonblock
+        sock
+      end
+
+      def server_create_udp_socket(shared, bind, port)
+        sock = if shared
+                 server_socket_manager_client.listen_udp(bind, port)
+               else
+                 family = IPAddr.new(IPSocket.getaddress(bind)).ipv4? ? ::Socket::AF_INET : ::Socket::AF_INET6
+                 usock = UDPSocket.new(family)
+                 usock.bind(bind, port)
+                 usock
+               end
+        # close-on-exec is set by default in Ruby 2.0 or later (, and it's unavailable on Windows)
+        sock.fcntl(Fcntl::F_SETFL, Fcntl::O_NONBLOCK) # nonblock
+        sock
+      end
+
+      def server_create_tls_socket(shared, bind, port)
+        raise "not implemented yet"
+      end
+
+      class CallbackSocket
+        def initialize(server_type, sock, enabled_events = [], close_socket: true)
+          @server_type = server_type
+          @sock = sock
+          @enabled_events = enabled_events
+          @close_socket = close_socket
+        end
+
+        def remote_addr
+          @sock.peeraddr[3]
+        end
+
+        def remote_host
+          @sock.peeraddr[2]
+        end
+
+        def remote_port
+          @sock.peeraddr[1]
+        end
+
+        def send(data, flags = 0)
+          @sock.send(data, flags)
+        end
+
+        def write(data)
+          raise "not implemented here"
+        end
+
+        def close
+          @sock.close if @close_socket
+        end
+
+        def data(&callback)
+          on(:data, &callback)
+        end
+
+        def on(event, &callback)
+          raise "BUG: this event is disabled for #{@server_type}: #{event}" unless @enabled_events.include?(event)
+          case event
+          when :data
+            @sock.data(&callback)
+          when :write_complete
+            cb = ->(){ callback.call(self) }
+            @sock.on_write_complete(&cb)
+          when :close
+            cb = ->(){ callback.call(self) }
+            @sock.on_close(&cb)
+          else
+            raise "BUG: unknown event: #{event}"
+          end
+        end
+      end
+
+      class TCPCallbackSocket < CallbackSocket
+        def initialize(sock)
+          super("tcp", sock, [:data, :write_complete, :close])
+        end
+
+        def write(data)
+          @sock.write(data)
+        end
+      end
+
+      class UDPCallbackSocket < CallbackSocket
+        def initialize(sock, peeraddr, **kwargs)
+          super("udp", sock, [], **kwargs)
+          @peeraddr = peeraddr
+        end
+
+        def remote_addr
+          @peeraddr[3]
+        end
+
+        def remote_host
+          @peeraddr[2]
+        end
+
+        def remote_port
+          @peeraddr[1]
+        end
+
+        def write(data)
+          @sock.send(data, 0, @peeraddr[3], @peeraddr[1])
+        end
+      end
+
+      module EventHandler
+        class UDPServer < Coolio::IO
+          def initialize(sock, max_bytes, flags, close_socket, log, under_plugin_development, &callback)
+            raise ArgumentError, "socket must be a UDPSocket: sock = #{sock}" unless sock.is_a?(UDPSocket)
+
+            super(sock)
+
+            @sock = sock
+            @max_bytes = max_bytes
+            @flags = flags
+            @close_socket = close_socket
+            @log = log
+            @under_plugin_development = under_plugin_development
+            @callback = callback
+
+            on_readable_impl = case @callback.arity
+                               when 1 then :on_readable_without_sock
+                               when 2 then :on_readable_with_sock
+                               else
+                                 raise "BUG: callback block must have 1 or 2 arguments"
+                               end
+            self.define_singleton_method(:on_readable, method(on_readable_impl))
+          end
+
+          def on_readable_without_sock
+            begin
+              data = @sock.recv(@max_bytes, @flags)
+            rescue Errno::EAGAIN, Errno::EWOULDBLOCK, Errno::EINTR, Errno::ECONNRESET
+              return
+            end
+            @callback.call(data)
+          rescue => e
+            @log.error "unexpected error in processing UDP data", error: e
+            @log.error_backtrace
+            raise if @under_plugin_development
+          end
+
+          def on_readable_with_sock
+            begin
+              data, addr = @sock.recvfrom(@max_bytes)
+            rescue Errno::EAGAIN, Errno::EWOULDBLOCK, Errno::EINTR, Errno::ECONNRESET
+              return
+            end
+            @callback.call(data, UDPCallbackSocket.new(@sock, addr, close_socket: @close_socket))
+          rescue => e
+            @log.error "unexpected error in processing UDP data", error: e
+            @log.error_backtrace
+            raise if @under_plugin_development
+          end
+        end
+
+        class TCPServer < Coolio::TCPSocket
+          def initialize(sock, socket_option_setter, close_callback, log, under_plugin_development, connect_callback)
+            raise ArgumentError, "socket must be a TCPSocket: sock=#{sock}" unless sock.is_a?(TCPSocket)
+
+            socket_option_setter.call(sock)
+
+            @_handler_socket = sock
+            super(sock)
+
+            @log = log
+            @under_plugin_development = under_plugin_development
+
+            @connect_callback = connect_callback
+            @data_callback = nil
+            @close_callback = close_callback
+
+            @callback_connection = nil
+            @closing = false
+
+            @mutex = Mutex.new # to serialize #write and #close
+          end
+
+          def data(&callback)
+            raise "data callback can be registered just once, but registered twice" if self.singleton_methods.include?(:on_read)
+            @data_callback = callback
+            on_read_impl = case callback.arity
+                           when 1 then :on_read_without_connection
+                           when 2 then :on_read_with_connection
+                           else
+                             raise "BUG: callback block must have 1 or 2 arguments"
+                           end
+            self.define_singleton_method(:on_read, method(on_read_impl))
+          end
+
+          def write(data)
+            @mutex.synchronize do
+              super
+            end
+          end
+
+          def on_connect
+            @callback_connection = TCPCallbackSocket.new(self)
+            @connect_callback.call(@callback_connection)
+            unless @data_callback
+              raise "connection callback must call #data to set data callback"
+            end
+          end
+
+          def on_read_without_connection(data)
+            @data_callback.call(data)
+          rescue => e
+            @log.error "unexpected error on reading data", host: remote_host, port: remote_port, error: e
+            @log.error_backtrace
+            close(true) rescue nil
+            raise if @under_plugin_development
+          end
+
+          def on_read_with_connection(data)
+            @data_callback.call(data, @callback_connection)
+          rescue => e
+            @log.error "unexpected error on reading data", host: remote_host, port: remote_port, error: e
+            @log.error_backtrace
+            close(true) rescue nil
+            raise if @under_plugin_development
+          end
+
+          def close
+            @mutex.synchronize do
+              return if @closing
+              @closing = true
+              @close_callback.call(self)
+              super
+            end
+          end
+        end
+      end
+    end
+  end
+end