fluentd 0.14.9 → 0.14.10

data/lib/fluent/plugin/output.rb

@@ -40,6 +40,8 @@ module Fluent
 
       CHUNKING_FIELD_WARN_NUM = 4
 
+      PROCESS_CLOCK_ID = Process::CLOCK_MONOTONIC_RAW rescue Process::CLOCK_MONOTONIC
+
       config_param :time_as_integer, :bool, default: false
 
       # `<buffer>` and `<secondary>` sections are available only when '#format' and '#write' are implemented
@@ -138,7 +140,7 @@ module Fluent
       end
 
       # Internal states
-      FlushThreadState = Struct.new(:thread, :next_time)
+      FlushThreadState = Struct.new(:thread, :next_clock)
       DequeuedChunkInfo = Struct.new(:chunk_id, :time, :timeout) do
         def expired?
           time + timeout < Time.now
@@ -898,9 +900,9 @@ module Fluent
         @retry_mutex.synchronize do
           if @retry # success to flush chunks in retries
             if secondary
-              log.warn "retry succeeded by secondary.", plugin_id: plugin_id, chunk_id: dump_unique_id_hex(chunk_id)
+              log.warn "retry succeeded by secondary.", chunk_id: dump_unique_id_hex(chunk_id)
             else
-              log.warn "retry succeeded.", plugin_id: plugin_id, chunk_id: dump_unique_id_hex(chunk_id)
+              log.warn "retry succeeded.", chunk_id: dump_unique_id_hex(chunk_id)
             end
             @retry = nil
           end
@@ -918,6 +920,8 @@ module Fluent
         # in many cases, false can be just ignored
         if @buffer.takeback_chunk(chunk_id)
           @counters_monitor.synchronize{ @rollback_count += 1 }
+          primary = @as_secondary ? @primary_instance : self
+          primary.update_retry_state(chunk_id, @as_secondary)
           true
         else
           false
@@ -930,7 +934,9 @@ module Fluent
             info = @dequeued_chunks.shift
             if @buffer.takeback_chunk(info.chunk_id)
               @counters_monitor.synchronize{ @rollback_count += 1 }
-              log.warn "failed to flush the buffer chunk, timeout to commit.", plugin_id: plugin_id, chunk_id: dump_unique_id_hex(info.chunk_id), flushed_at: info.time
+              log.warn "failed to flush the buffer chunk, timeout to commit.", chunk_id: dump_unique_id_hex(info.chunk_id), flushed_at: info.time
+              primary = @as_secondary ? @primary_instance : self
+              primary.update_retry_state(info.chunk_id, @as_secondary)
             end
           end
         end
@@ -943,7 +949,9 @@ module Fluent
             info = @dequeued_chunks.shift
             if @buffer.takeback_chunk(info.chunk_id)
               @counters_monitor.synchronize{ @rollback_count += 1 }
-              log.info "delayed commit for buffer chunks was cancelled in shutdown", plugin_id: plugin_id, chunk_id: dump_unique_id_hex(info.chunk_id)
+              log.info "delayed commit for buffer chunks was cancelled in shutdown", chunk_id: dump_unique_id_hex(info.chunk_id)
+              primary = @as_secondary ? @primary_instance : self
+              primary.update_retry_state(info.chunk_id, @as_secondary)
             end
           end
         end
@@ -997,7 +1005,7 @@ module Fluent
             log.trace "done to commit a chunk", chunk: dump_chunk_id
           end
         rescue => e
-          log.debug "taking back chunk for errors.", plugin_id: plugin_id, chunk: dump_unique_id_hex(chunk.unique_id)
+          log.debug "taking back chunk for errors.", chunk: dump_unique_id_hex(chunk.unique_id)
           if output.delayed_commit
             @dequeued_chunks_mutex.synchronize do
               @dequeued_chunks.delete_if{|d| d.chunk_id == chunk.unique_id }
@@ -1005,35 +1013,52 @@ module Fluent
           end
           @buffer.takeback_chunk(chunk.unique_id)
 
-          @retry_mutex.synchronize do
-            if @retry
-              @counters_monitor.synchronize{ @num_errors += 1 }
-              if @retry.limit?
-                records = @buffer.queued_records
-                log.error "failed to flush the buffer, and hit limit for retries. dropping all chunks in the buffer queue.", plugin_id: plugin_id, retry_times: @retry.steps, records: records, error: e
-                log.error_backtrace e.backtrace
-                @buffer.clear_queue!
-                log.debug "buffer queue cleared", plugin_id: plugin_id
-                @retry = nil
-              else
-                @retry.step
-                msg = if using_secondary
-                        "failed to flush the buffer with secondary output."
-                      else
-                        "failed to flush the buffer."
-                      end
-                log.warn msg, plugin_id: plugin_id, retry_time: @retry.steps, next_retry: @retry.next_time, chunk: dump_unique_id_hex(chunk.unique_id), error: e
-                log.warn_backtrace e.backtrace
-              end
+          update_retry_state(chunk.unique_id, using_secondary, e)
+
+          raise if @under_plugin_development && !@retry_for_error_chunk
+        end
+      end
+
+      def update_retry_state(chunk_id, using_secondary, error = nil)
+        @retry_mutex.synchronize do
+          @counters_monitor.synchronize{ @num_errors += 1 }
+          chunk_id_hex = dump_unique_id_hex(chunk_id)
+
+          unless @retry
+            @retry = retry_state(@buffer_config.retry_randomize)
+            if error
+              log.warn "failed to flush the buffer.", retry_time: @retry.steps, next_retry_seconds: @retry.next_time, chunk: chunk_id_hex, error: error
+              log.warn_backtrace error.backtrace
+            end
+            return
+          end
+
+          # @retry exists
+
+          if error
+            if @retry.limit?
+              records = @buffer.queued_records
+              msg = "failed to flush the buffer, and hit limit for retries. dropping all chunks in the buffer queue."
+              log.error msg, retry_times: @retry.steps, records: records, error: error
+              log.error_backtrace error.backtrace
+            elsif using_secondary
+              msg = "failed to flush the buffer with secondary output."
+              log.warn msg, retry_time: @retry.steps, next_retry_seconds: @retry.next_time, chunk: chunk_id_hex, error: error
+              log.warn_backtrace error.backtrace
             else
-              @retry = retry_state(@buffer_config.retry_randomize)
-              @counters_monitor.synchronize{ @num_errors += 1 }
-              log.warn "failed to flush the buffer.", plugin_id: plugin_id, retry_time: @retry.steps, next_retry: @retry.next_time, chunk: dump_unique_id_hex(chunk.unique_id), error: e
-              log.warn_backtrace e.backtrace
+              msg = "failed to flush the buffer."
+              log.warn msg, retry_time: @retry.steps, next_retry_seconds: @retry.next_time, chunk: chunk_id_hex, error: error
+              log.warn_backtrace error.backtrace
             end
           end
 
-          raise if @under_plugin_development && !@retry_for_error_chunk
+          if @retry.limit?
+            @buffer.clear_queue!
+            log.debug "buffer queue cleared"
+            @retry = nil
+          else
+            @retry.step
+          end
         end
       end
 
@@ -1060,7 +1085,7 @@ module Fluent
         # Without locks: it is rough but enough to select "next" writer selection
         @output_flush_thread_current_position = (@output_flush_thread_current_position + 1) % @buffer_config.flush_thread_count
         state = @output_flush_threads[@output_flush_thread_current_position]
-        state.next_time = 0
+        state.next_clock = 0
         if state.thread && state.thread.status # "run"/"sleep"/"aborting" or false(successfully stop) or nil(killed by exception)
           state.thread.run
         else
@@ -1102,7 +1127,7 @@ module Fluent
       # only for tests of output plugin
       def flush_thread_wakeup
         @output_flush_threads.each do |state|
-          state.next_time = 0
+          state.next_clock = 0
           state.thread.run
         end
       end
@@ -1156,7 +1181,7 @@ module Fluent
               end
             rescue => e
               raise if @under_plugin_development
-              log.error "unexpected error while checking flushed chunks. ignored.", plugin_id: plugin_id, error: e
+              log.error "unexpected error while checking flushed chunks. ignored.", error: e
               log.error_backtrace
             ensure
               @output_enqueue_thread_waiting = false
@@ -1166,7 +1191,7 @@ module Fluent
           end
         rescue => e
           # normal errors are rescued by inner begin-rescue clause.
-          log.error "error on enqueue thread", plugin_id: plugin_id, error: e
+          log.error "error on enqueue thread", error: e
           log.error_backtrace
           raise
         end
@@ -1175,9 +1200,7 @@ module Fluent
       def flush_thread_run(state)
         flush_thread_interval = @buffer_config.flush_thread_interval
 
-        # If the given clock_id is not supported, Errno::EINVAL is raised.
-        clock_id = Process::CLOCK_MONOTONIC rescue Process::CLOCK_MONOTONIC_RAW
-        state.next_time = Process.clock_gettime(clock_id) + flush_thread_interval
+        state.next_clock = Process.clock_gettime(PROCESS_CLOCK_ID) + flush_thread_interval
 
         while !self.after_started? && !self.stopped?
           sleep 0.5
@@ -1187,16 +1210,18 @@ module Fluent
         begin
           # This thread don't use `thread_current_running?` because this thread should run in `before_shutdown` phase
           while @output_flush_threads_running
-            time = Process.clock_gettime(clock_id)
-            interval = state.next_time - time
+            current_clock = Process.clock_gettime(PROCESS_CLOCK_ID)
+            interval = state.next_clock - current_clock
 
-            if state.next_time <= time
+            if state.next_clock <= current_clock && (!@retry || @retry_mutex.synchronize{ @retry.next_time } <= Time.now)
               try_flush
-              # next_flush_interval uses flush_thread_interval or flush_thread_burst_interval (or retrying)
+
+              # next_flush_time uses flush_thread_interval or flush_thread_burst_interval (or retrying)
               interval = next_flush_time.to_f - Time.now.to_f
-              # TODO: if secondary && delayed-commit, next_flush_time will be much longer than expected (because @retry still exists)
-              #   @retry should be cleared if delayed commit is enabled? Or any other solution?
-              state.next_time = Process.clock_gettime(clock_id) + interval
+              # TODO: if secondary && delayed-commit, next_flush_time will be much longer than expected
+              #       because @retry still exists (#commit_write is not called yet in #try_flush)
+              #       @retry should be cleared if delayed commit is enabled? Or any other solution?
+              state.next_clock = Process.clock_gettime(PROCESS_CLOCK_ID) + interval
             end
 
             if @dequeued_chunks_mutex.synchronize{ !@dequeued_chunks.empty? && @dequeued_chunks.first.expired? }
@@ -1210,7 +1235,7 @@ module Fluent
         rescue => e
           # normal errors are rescued by output plugins in #try_flush
           # so this rescue section is for critical & unrecoverable errors
-          log.error "error on output thread", plugin_id: plugin_id, error: e
+          log.error "error on output thread", error: e
           log.error_backtrace
           raise
         end

data/lib/fluent/plugin/parser_json.rb

@@ -53,7 +53,7 @@ module Fluent
         end
       rescue LoadError
         name = :yajl
-        log.info "Oj is not installed, and failing back to Yajl for json parser"
+        log.info "Oj is not installed, and failing back to Yajl for json parser" if log
         retry
       end
 

data/lib/fluent/plugin_helper.rb

@@ -24,6 +24,8 @@ require 'fluent/plugin_helper/parser'
 require 'fluent/plugin_helper/formatter'
 require 'fluent/plugin_helper/inject'
 require 'fluent/plugin_helper/extract'
+require 'fluent/plugin_helper/socket'
+require 'fluent/plugin_helper/server'
 require 'fluent/plugin_helper/retry_state'
 require 'fluent/plugin_helper/compat_parameters'
 

data/lib/fluent/plugin_helper/event_loop.rb

@@ -30,12 +30,16 @@ module Fluent
       # terminate: initialize internal state
 
       EVENT_LOOP_RUN_DEFAULT_TIMEOUT = 0.5
+      EVENT_LOOP_SHUTDOWN_TIMEOUT = 5
+      EVENT_LOOP_CLOCK_ID = Process::CLOCK_MONOTONIC_RAW rescue Process::CLOCK_MONOTONIC
 
       attr_reader :_event_loop # for tests
 
       def event_loop_attach(watcher)
         @_event_loop_mutex.synchronize do
           @_event_loop.attach(watcher)
+          @_event_loop_attached_watchers << watcher
+          watcher
         end
       end
 
@@ -58,6 +62,7 @@ module Fluent
         @_event_loop_mutex = Mutex.new
         # plugin MAY configure loop run timeout in #configure
         @_event_loop_run_timeout = EVENT_LOOP_RUN_DEFAULT_TIMEOUT
+        @_event_loop_attached_watchers = []
       end
 
       def start
@@ -65,19 +70,32 @@ module Fluent
 
         # event loop does not run here, so mutex lock is not required
         thread_create :event_loop do
-          default_watcher = DefaultWatcher.new
-          @_event_loop.attach(default_watcher)
-          @_event_loop_running = true
-          @_event_loop.run(@_event_loop_run_timeout) # this method blocks
-          @_event_loop_running = false
+          begin
+            default_watcher = DefaultWatcher.new
+            event_loop_attach(default_watcher)
+            @_event_loop_running = true
+            @_event_loop.run(@_event_loop_run_timeout) # this method blocks
+          ensure
+            @_event_loop_running = false
+          end
         end
       end
 
       def shutdown
         @_event_loop_mutex.synchronize do
-          @_event_loop.watchers.each {|w| w.detach if w.attached? }
+          @_event_loop_attached_watchers.reverse.each do |w|
+            if w.attached?
+              w.detach
+            end
+          end
         end
+        timeout_at = Process.clock_gettime(EVENT_LOOP_CLOCK_ID) + EVENT_LOOP_SHUTDOWN_TIMEOUT
         while @_event_loop_running
+          if Process.clock_gettime(EVENT_LOOP_CLOCK_ID) >= timeout_at
+            log.warn "event loop does NOT exit until hard timeout."
+            raise "event loop does NOT exit until hard timeout." if @under_plugin_development
+            break
+          end
           sleep 0.1
         end
 

data/lib/fluent/plugin_helper/inject.rb

@@ -97,9 +97,20 @@ module Fluent
         if @inject_config
           @_inject_hostname_key = @inject_config.hostname_key
           if @_inject_hostname_key
+            if self.respond_to?(:buffer_config)
+              # Output plugin cannot use "hostname"(specified by @hostname_key),
+              # injected by this plugin helper, in chunk keys.
+              # This plugin helper works in `#format` (in many cases), but modified record
+              # don't have any side effect in chunking of output plugin.
+              if self.buffer_config.chunk_keys.include?(@_inject_hostname_key)
+                log.error "Use filters to inject hostname to use it in buffer chunking."
+                raise Fluent::ConfigError, "the key specified by 'hostname_key' in <inject> cannot be used in buffering chunk key."
+              end
+            end
+
             @_inject_hostname =  @inject_config.hostname
             unless @_inject_hostname
-              @_inject_hostname = Socket.gethostname
+              @_inject_hostname = ::Socket.gethostname
               log.info "using hostname for specified field", host_key: @_inject_hostname_key, host_name: @_inject_hostname
             end
           end

data/lib/fluent/plugin_helper/server.rb

@@ -0,0 +1,494 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin_helper/event_loop'
+
+require 'serverengine'
+require 'cool.io'
+require 'socket'
+require 'ipaddr'
+require 'fcntl'
+
+require_relative 'socket_option'
+
+module Fluent
+  module PluginHelper
+    module Server
+      include Fluent::PluginHelper::EventLoop
+      include Fluent::PluginHelper::SocketOption
+
+      # This plugin helper doesn't support these things for now:
+      # * SSL/TLS (TBD)
+      # * TCP/TLS keepalive
+
+      # stop     : [-]
+      # shutdown : detach server event handler from event loop (event_loop)
+      # close    : close listening sockets
+      # terminate: remote all server instances
+
+      attr_reader :_servers # for tests
+
+      def server_wait_until_start
+        # event_loop_wait_until_start works well for this
+      end
+
+      def server_wait_until_stop
+        sleep 0.1 while @_servers.any?{|si| si.server.attached? }
+        @_servers.each{|si| si.server.close rescue nil }
+      end
+
+      PROTOCOLS = [:tcp, :udp, :tls, :unix]
+      CONNECTION_PROTOCOLS = [:tcp, :tls, :unix]
+
+      # server_create_connection(:title, @port) do |conn|
+      #   # on connection
+      #   source_addr = conn.remote_host
+      #   source_port = conn.remote_port
+      #   conn.data do |data|
+      #     # on data
+      #     conn.write resp # ...
+      #     conn.close
+      #   end
+      # end
+      def server_create_connection(title, port, proto: :tcp, bind: '0.0.0.0', shared: true, backlog: nil, **socket_options, &block)
+        raise ArgumentError, "BUG: title must be a symbol" unless title && title.is_a?(Symbol)
+        raise ArgumentError, "BUG: port must be an integer" unless port && port.is_a?(Integer)
+        raise ArgumentError, "BUG: invalid protocol name" unless PROTOCOLS.include?(proto)
+        raise ArgumentError, "BUG: cannot create connection for UDP" unless CONNECTION_PROTOCOLS.include?(proto)
+
+        raise ArgumentError, "BUG: block not specified which handles connection" unless block_given?
+        raise ArgumentError, "BUG: block must have just one argument" unless block.arity == 1
+
+        if proto == :tcp || proto == :tls # default linger_timeout only for server
+          socket_options[:linger_timeout] ||= 0
+        end
+
+        socket_option_validate!(proto, **socket_options)
+        socket_option_setter = ->(sock){ socket_option_set(sock, **socket_options) }
+
+        case proto
+        when :tcp
+          server = server_create_for_tcp_connection(shared, bind, port, backlog, socket_option_setter, &block)
+        when :tls
+          raise ArgumentError, "BUG: certopts (certificate options) not specified for TLS" unless certopts
+          # server_certopts_validate!(certopts)
+          # sock = server_create_tls_socket(shared, bind, port)
+          # server = nil # ...
+          raise "not implemented yet"
+        when :unix
+          raise "not implemented yet"
+        else
+          raise "unknown protocol #{proto}"
+        end
+
+        server_attach(title, proto, port, bind, shared, server)
+      end
+
+      # server_create(:title, @port) do |data|
+      #   # ...
+      # end
+      # server_create(:title, @port) do |data, conn|
+      #   # ...
+      # end
+      # server_create(:title, @port, proto: :udp, max_bytes: 2048) do |data, sock|
+      #   sock.remote_host
+      #   sock.remote_port
+      #   # ...
+      # end
+      def server_create(title, port, proto: :tcp, bind: '0.0.0.0', shared: true, socket: nil, backlog: nil, max_bytes: nil, flags: 0, **socket_options, &callback)
+        raise ArgumentError, "BUG: title must be a symbol" unless title && title.is_a?(Symbol)
+        raise ArgumentError, "BUG: port must be an integer" unless port && port.is_a?(Integer)
+        raise ArgumentError, "BUG: invalid protocol name" unless PROTOCOLS.include?(proto)
+
+        raise ArgumentError, "BUG: socket option is available only for udp" if socket && proto != :udp
+
+        raise ArgumentError, "BUG: block not specified which handles received data" unless block_given?
+        raise ArgumentError, "BUG: block must have 1 or 2 arguments" unless callback.arity == 1 || callback.arity == 2
+
+        if proto == :tcp || proto == :tls # default linger_timeout only for server
+          socket_options[:linger_timeout] ||= 0
+        end
+
+        unless socket
+          socket_option_validate!(proto, **socket_options)
+          socket_option_setter = ->(sock){ socket_option_set(sock, **socket_options) }
+        end
+
+        if proto != :tcp && proto != :tls && proto != :unix # options to listen/accept connections
+          raise ArgumentError, "BUG: backlog is available for tcp/tls" if backlog
+        end
+        if proto != :udp # UDP options
+          raise ArgumentError, "BUG: max_bytes is available only for udp" if max_bytes
+          raise ArgumentError, "BUG: flags is available only for udp" if flags != 0
+        end
+
+        case proto
+        when :tcp
+          server = server_create_for_tcp_connection(shared, bind, port, backlog, socket_option_setter) do |conn|
+            conn.data(&callback)
+          end
+        when :tls
+          raise "not implemented yet"
+        when :udp
+          raise ArgumentError, "BUG: max_bytes must be specified for UDP" unless max_bytes
+          if socket
+            sock = socket
+            close_socket = false
+          else
+            sock = server_create_udp_socket(shared, bind, port)
+            socket_option_setter.call(sock)
+            close_socket = true
+          end
+          server = EventHandler::UDPServer.new(sock, max_bytes, flags, close_socket, @log, @under_plugin_development, &callback)
+        when :unix
+          raise "not implemented yet"
+        else
+          raise "BUG: unknown protocol #{proto}"
+        end
+
+        server_attach(title, proto, port, bind, shared, server)
+      end
+
+      def server_create_tcp(title, port, **kwargs, &callback)
+        server_create(title, port, proto: :tcp, **kwargs, &callback)
+      end
+
+      def server_create_udp(title, port, **kwargs, &callback)
+        server_create(title, port, proto: :udp, **kwargs, &callback)
+      end
+
+      def server_create_tls(title, port, **kwargs, &callback)
+        server_create(title, port, proto: :tls, **kwargs, &callback)
+      end
+
+      def server_create_unix(title, port, **kwargs, &callback)
+        server_create(title, port, proto: :unix, **kwargs, &callback)
+      end
+
+      ServerInfo = Struct.new(:title, :proto, :port, :bind, :shared, :server)
+
+      def server_attach(title, proto, port, bind, shared, server)
+        @_servers << ServerInfo.new(title, proto, port, bind, shared, server)
+        event_loop_attach(server)
+      end
+
+      def server_create_for_tcp_connection(shared, bind, port, backlog, socket_option_setter, &block)
+        sock = server_create_tcp_socket(shared, bind, port)
+        socket_option_setter.call(sock)
+        close_callback = ->(conn){ @_server_mutex.synchronize{ @_server_connections.delete(conn) } }
+        server = Coolio::TCPServer.new(sock, nil, EventHandler::TCPServer, socket_option_setter, close_callback, @log, @under_plugin_development, block) do |conn|
+          @_server_mutex.synchronize do
+            @_server_connections << conn
+          end
+        end
+        server.listen(backlog) if backlog
+        server
+      end
+
+      def initialize
+        super
+        @_servers = []
+        @_server_connections = []
+        @_server_mutex = Mutex.new
+      end
+
+      def shutdown
+        @_server_connections.each do |conn|
+          conn.close rescue nil
+        end
+        @_server_mutex.synchronize do
+          @_servers.each do |si|
+            si.server.detach if si.server.attached?
+          end
+        end
+
+        super
+      end
+
+      def close
+        @_server_connections.each do |conn|
+          conn.close rescue nil
+        end
+        @_server_mutex.synchronize do
+          @_servers.each do |si|
+            si.server.close rescue nil
+          end
+        end
+        super
+      end
+
+      def terminate
+        @_servers = []
+        super
+      end
+
+      def server_certopts_validate!(certopts)
+        raise "not implemented yet"
+      end
+
+      def server_socket_manager_client
+        socket_manager_path = ENV['SERVERENGINE_SOCKETMANAGER_PATH']
+        if Fluent.windows?
+          socket_manager_path = socket_manager_path.to_i
+        end
+        ServerEngine::SocketManager::Client.new(socket_manager_path)
+      end
+
+      def server_create_tcp_socket(shared, bind, port)
+        sock = if shared
+                 server_socket_manager_client.listen_tcp(bind, port)
+               else
+                 TCPServer.new(bind, port) # this method call can create sockets for AF_INET6
+               end
+        # close-on-exec is set by default in Ruby 2.0 or later (, and it's unavailable on Windows)
+        sock.fcntl(Fcntl::F_SETFL, Fcntl::O_NONBLOCK) # nonblock
+        sock
+      end
+
+      def server_create_udp_socket(shared, bind, port)
+        sock = if shared
+                 server_socket_manager_client.listen_udp(bind, port)
+               else
+                 family = IPAddr.new(IPSocket.getaddress(bind)).ipv4? ? ::Socket::AF_INET : ::Socket::AF_INET6
+                 usock = UDPSocket.new(family)
+                 usock.bind(bind, port)
+                 usock
+               end
+        # close-on-exec is set by default in Ruby 2.0 or later (, and it's unavailable on Windows)
+        sock.fcntl(Fcntl::F_SETFL, Fcntl::O_NONBLOCK) # nonblock
+        sock
+      end
+
+      def server_create_tls_socket(shared, bind, port)
+        raise "not implemented yet"
+      end
+
+      class CallbackSocket
+        def initialize(server_type, sock, enabled_events = [], close_socket: true)
+          @server_type = server_type
+          @sock = sock
+          @enabled_events = enabled_events
+          @close_socket = close_socket
+        end
+
+        def remote_addr
+          @sock.peeraddr[3]
+        end
+
+        def remote_host
+          @sock.peeraddr[2]
+        end
+
+        def remote_port
+          @sock.peeraddr[1]
+        end
+
+        def send(data, flags = 0)
+          @sock.send(data, flags)
+        end
+
+        def write(data)
+          raise "not implemented here"
+        end
+
+        def close
+          @sock.close if @close_socket
+        end
+
+        def data(&callback)
+          on(:data, &callback)
+        end
+
+        def on(event, &callback)
+          raise "BUG: this event is disabled for #{@server_type}: #{event}" unless @enabled_events.include?(event)
+          case event
+          when :data
+            @sock.data(&callback)
+          when :write_complete
+            cb = ->(){ callback.call(self) }
+            @sock.on_write_complete(&cb)
+          when :close
+            cb = ->(){ callback.call(self) }
+            @sock.on_close(&cb)
+          else
+            raise "BUG: unknown event: #{event}"
+          end
+        end
+      end
+
+      class TCPCallbackSocket < CallbackSocket
+        def initialize(sock)
+          super("tcp", sock, [:data, :write_complete, :close])
+        end
+
+        def write(data)
+          @sock.write(data)
+        end
+      end
+
+      class UDPCallbackSocket < CallbackSocket
+        def initialize(sock, peeraddr, **kwargs)
+          super("udp", sock, [], **kwargs)
+          @peeraddr = peeraddr
+        end
+
+        def remote_addr
+          @peeraddr[3]
+        end
+
+        def remote_host
+          @peeraddr[2]
+        end
+
+        def remote_port
+          @peeraddr[1]
+        end
+
+        def write(data)
+          @sock.send(data, 0, @peeraddr[3], @peeraddr[1])
+        end
+      end
+
+      module EventHandler
+        class UDPServer < Coolio::IO
+          def initialize(sock, max_bytes, flags, close_socket, log, under_plugin_development, &callback)
+            raise ArgumentError, "socket must be a UDPSocket: sock = #{sock}" unless sock.is_a?(UDPSocket)
+
+            super(sock)
+
+            @sock = sock
+            @max_bytes = max_bytes
+            @flags = flags
+            @close_socket = close_socket
+            @log = log
+            @under_plugin_development = under_plugin_development
+            @callback = callback
+
+            on_readable_impl = case @callback.arity
+                               when 1 then :on_readable_without_sock
+                               when 2 then :on_readable_with_sock
+                               else
+                                 raise "BUG: callback block must have 1 or 2 arguments"
+                               end
+            self.define_singleton_method(:on_readable, method(on_readable_impl))
+          end
+
+          def on_readable_without_sock
+            begin
+              data = @sock.recv(@max_bytes, @flags)
+            rescue Errno::EAGAIN, Errno::EWOULDBLOCK, Errno::EINTR, Errno::ECONNRESET
+              return
+            end
+            @callback.call(data)
+          rescue => e
+            @log.error "unexpected error in processing UDP data", error: e
+            @log.error_backtrace
+            raise if @under_plugin_development
+          end
+
+          def on_readable_with_sock
+            begin
+              data, addr = @sock.recvfrom(@max_bytes)
+            rescue Errno::EAGAIN, Errno::EWOULDBLOCK, Errno::EINTR, Errno::ECONNRESET
+              return
+            end
+            @callback.call(data, UDPCallbackSocket.new(@sock, addr, close_socket: @close_socket))
+          rescue => e
+            @log.error "unexpected error in processing UDP data", error: e
+            @log.error_backtrace
+            raise if @under_plugin_development
+          end
+        end
+
+        class TCPServer < Coolio::TCPSocket
+          def initialize(sock, socket_option_setter, close_callback, log, under_plugin_development, connect_callback)
+            raise ArgumentError, "socket must be a TCPSocket: sock=#{sock}" unless sock.is_a?(TCPSocket)
+
+            socket_option_setter.call(sock)
+
+            @_handler_socket = sock
+            super(sock)
+
+            @log = log
+            @under_plugin_development = under_plugin_development
+
+            @connect_callback = connect_callback
+            @data_callback = nil
+            @close_callback = close_callback
+
+            @callback_connection = nil
+            @closing = false
+
+            @mutex = Mutex.new # to serialize #write and #close
+          end
+
+          def data(&callback)
+            raise "data callback can be registered just once, but registered twice" if self.singleton_methods.include?(:on_read)
+            @data_callback = callback
+            on_read_impl = case callback.arity
+                           when 1 then :on_read_without_connection
+                           when 2 then :on_read_with_connection
+                           else
+                             raise "BUG: callback block must have 1 or 2 arguments"
+                           end
+            self.define_singleton_method(:on_read, method(on_read_impl))
+          end
+
+          def write(data)
+            @mutex.synchronize do
+              super
+            end
+          end
+
+          def on_connect
+            @callback_connection = TCPCallbackSocket.new(self)
+            @connect_callback.call(@callback_connection)
+            unless @data_callback
+              raise "connection callback must call #data to set data callback"
+            end
+          end
+
+          def on_read_without_connection(data)
+            @data_callback.call(data)
+          rescue => e
+            @log.error "unexpected error on reading data", host: remote_host, port: remote_port, error: e
+            @log.error_backtrace
+            close(true) rescue nil
+            raise if @under_plugin_development
+          end
+
+          def on_read_with_connection(data)
+            @data_callback.call(data, @callback_connection)
+          rescue => e
+            @log.error "unexpected error on reading data", host: remote_host, port: remote_port, error: e
+            @log.error_backtrace
+            close(true) rescue nil
+            raise if @under_plugin_development
+          end
+
+          def close
+            @mutex.synchronize do
+              return if @closing
+              @closing = true
+              @close_callback.call(self)
+              super
+            end
+          end
+        end
+      end
+    end
+  end
+end