fluentd 0.14.2 → 0.14.5

data/lib/fluent/plugin/out_forward.rb

@@ -53,18 +53,7 @@ module Fluent
     desc 'The timeout time when sending event logs.'
     config_param :send_timeout, :time, default: 60
     desc 'The transport protocol to use for heartbeats.(udp,tcp,none)'
-    config_param :heartbeat_type, default: :tcp do |val|
-      case val.downcase
-      when 'tcp'
-        :tcp
-      when 'udp'
-        :udp
-      when 'none'
-        :none
-      else
-        raise ConfigError, "forward output heartbeat type should be 'tcp', 'udp', or 'none'"
-      end
-    end
+    config_param :heartbeat_type, :enum, list: [:tcp, :udp, :none], default: :tcp
     desc 'The interval of the heartbeat packer.'
     config_param :heartbeat_interval, :time, default: 1
     desc 'The wait time before accepting a server fault recovery.'
@@ -81,32 +70,54 @@ module Fluent
     desc 'Change the protocol to at-least-once.'
     config_param :require_ack_response, :bool, default: false  # require in_forward to respond with ack
     desc 'This option is used when require_ack_response is true.'
-    config_param :ack_response_timeout, :time, default: 190  # 0 means do not wait for ack responses
+    config_param :ack_response_timeout, :time, default: 190
+    desc 'Reading data size from server'
+    config_param :read_length, :size, default: 512 # 512bytes
+    desc 'The interval while reading data from server'
+    config_param :read_interval_msec, :integer, default: 50 # 50ms
     # Linux default tcp_syn_retries is 5 (in many environment)
     # 3 + 6 + 12 + 24 + 48 + 96 -> 189 (sec)
     desc 'Enable client-side DNS round robin.'
     config_param :dns_round_robin, :bool, default: false # heartbeat_type 'udp' is not available for this
 
+    config_section :security, required: false, multi: false do
+      desc 'The hostname'
+      config_param :self_hostname, :string
+      desc 'Shared key for authentication'
+      config_param :shared_key, :string, secret: true
+    end
+
+    config_section :server, param_name: :servers do
+      desc "The IP address or host name of the server."
+      config_param :host, :string
+      desc "The name of the server. Used in log messages."
+      config_param :name, :string, default: nil
+      desc "The port number of the host."
+      config_param :port, :integer, default: LISTEN_PORT
+      desc "The shared key per server."
+      config_param :shared_key, :string, default: nil, secret: true
+      desc "The username for authentication."
+      config_param :username, :string, default: ''
+      desc "The password for authentication."
+      config_param :password, :string, default: '', secret: true
+      desc "Marks a node as the standby node for an Active-Standby model between Fluentd nodes."
+      config_param :standby, :bool, default: false
+      desc "The load balancing weight."
+      config_param :weight, :integer, default: 60
+    end
+
     attr_reader :nodes
 
-    # backward compatibility
-    config_param :port, :integer, default: LISTEN_PORT
-    config_param :host, :string, default: nil
+    config_param :port, :integer, default: LISTEN_PORT, obsoleted: "User <server> section instead."
+    config_param :host, :string, default: nil, obsoleted: "Use <server> section instead."
+
+    attr_reader :read_interval, :recover_sample_size
 
     def configure(conf)
       super
 
-      # backward compatibility
-      if host = conf['host']
-        log.warn "'host' option in forward output is obsoleted. Use '<server> host xxx </server>' instead."
-        port = conf['port']
-        port = port ? port.to_i : LISTEN_PORT
-        element = conf.add_element('server')
-        element['host'] = host
-        element['port'] = port.to_s
-      end
-
-      recover_sample_size = @recover_wait / @heartbeat_interval
+      @read_interval = @read_interval_msec / 1000.0
+      @recover_sample_size = @recover_wait / @heartbeat_interval
 
       if @dns_round_robin
         if @heartbeat_type == :udp
@@ -114,39 +125,23 @@ module Fluent
         end
       end
 
-      conf.elements.each {|e|
-        next if e.name != "server"
-
-        host = e['host']
-        port = e['port']
-        port = port ? port.to_i : LISTEN_PORT
-
-        weight = e['weight']
-        weight = weight ? weight.to_i : 60
-
-        standby = !!e['standby']
-
-        name = e['name']
-        unless name
-          name = "#{host}:#{port}"
-        end
-
+      @servers.each do |server|
         failure = FailureDetector.new(@heartbeat_interval, @hard_timeout, Time.now.to_i.to_f)
+        name = server.name || "#{server.host}:#{server.port}"
 
-        node_conf = NodeConfig.new(name, host, port, weight, standby, failure,
-          @phi_threshold, recover_sample_size, @expire_dns_cache, @phi_failure_detector, @dns_round_robin)
-
+        log.info "adding forwarding server '#{name}'", host: server.host, port: server.port, weight: server.weight, plugin_id: plugin_id
         if @heartbeat_type == :none
-          @nodes << NoneHeartbeatNode.new(log, node_conf)
+          @nodes << NoneHeartbeatNode.new(self, server, failure: failure)
         else
-          @nodes << Node.new(log, node_conf)
+          @nodes << Node.new(self, server, failure: failure)
         end
-        log.info "adding forwarding server '#{name}'", host: host, port: port, weight: weight, plugin_id: plugin_id
-      }
+      end
 
       if @nodes.empty?
         raise ConfigError, "forward output plugin requires at least one <server> is required"
       end
+
+      raise Fluent::ConfigError, "ack_response_timeout must be a positive integer" if @ack_response_timeout < 1
     end
 
     def start
@@ -155,6 +150,7 @@ module Fluent
       @rand_seed = Random.new.seed
       rebuild_weight_array
       @rr = 0
+      @usock = nil
 
       unless @heartbeat_type == :none
         @loop = Coolio::Loop.new
@@ -205,7 +201,7 @@ module Fluent
 
         if node.available?
           begin
-            send_data(node, tag, chunk)
+            node.send_data(tag, chunk)
             return
           rescue
             # for load balancing during detecting crashed servers
@@ -221,6 +217,12 @@ module Fluent
       end
     end
 
+    # MessagePack FixArray length is 3
+    FORWARD_HEADER = [0x93].pack('C').freeze
+    def forward_header
+      FORWARD_HEADER
+    end
+
     private
 
     def rebuild_weight_array
@@ -265,111 +267,6 @@ module Fluent
       @weight_array = weight_array
     end
 
-    # MessagePack FixArray length is 3
-    FORWARD_HEADER = [0x93].pack('C').freeze
-    def forward_header
-      FORWARD_HEADER
-    end
-
-    #FORWARD_TCP_HEARTBEAT_DATA = FORWARD_HEADER + ''.to_msgpack + [].to_msgpack
-    def send_heartbeat_tcp(node)
-      sock = connect(node)
-      begin
-        opt = [1, @send_timeout.to_i].pack('I!I!')  # { int l_onoff; int l_linger; }
-        sock.setsockopt(Socket::SOL_SOCKET, Socket::SO_LINGER, opt)
-        opt = [@send_timeout.to_i, 0].pack('L!L!')  # struct timeval
-        # don't send any data to not cause a compatibility problem
-        #sock.setsockopt(Socket::SOL_SOCKET, Socket::SO_SNDTIMEO, opt)
-        #sock.write FORWARD_TCP_HEARTBEAT_DATA
-        node.heartbeat(true)
-      ensure
-        sock.close_write
-        sock.close
-      end
-    end
-
-    def send_data(node, tag, chunk)
-      sock = connect(node)
-      begin
-        opt = [1, @send_timeout.to_i].pack('I!I!')  # { int l_onoff; int l_linger; }
-        sock.setsockopt(Socket::SOL_SOCKET, Socket::SO_LINGER, opt)
-
-        opt = [@send_timeout.to_i, 0].pack('L!L!')  # struct timeval
-        sock.setsockopt(Socket::SOL_SOCKET, Socket::SO_SNDTIMEO, opt)
-
-        # beginArray(3)
-        sock.write forward_header
-
-        # writeRaw(tag)
-        sock.write tag.to_msgpack  # tag
-
-        # beginRaw(size)
-        sz = chunk.size
-        #if sz < 32
-        #  # FixRaw
-        #  sock.write [0xa0 | sz].pack('C')
-        #elsif sz < 65536
-        #  # raw 16
-        #  sock.write [0xda, sz].pack('Cn')
-        #else
-        # raw 32
-        sock.write [0xdb, sz].pack('CN')
-        #end
-
-        # writeRawBody(packed_es)
-        chunk.write_to(sock)
-
-        option = { 'size' => chunk.size_of_events }
-        option['chunk'] = Base64.encode64(chunk.unique_id) if @require_ack_response
-        sock.write option.to_msgpack
-
-        if @require_ack_response && @ack_response_timeout > 0
-          # Waiting for a response here results in a decrease of throughput because a chunk queue is locked.
-          # To avoid a decrease of troughput, it is necessary to prepare a list of chunks that wait for responses
-          # and process them asynchronously.
-          if IO.select([sock], nil, nil, @ack_response_timeout)
-            raw_data = sock.recv(1024)
-
-            # When connection is closed by remote host, socket is ready to read and #recv returns an empty string that means EOF.
-            # If this happens we assume the data wasn't delivered and retry it.
-            if raw_data.empty?
-              @log.warn "node #{node.host}:#{node.port} closed the connection. regard it as unavailable."
-              node.disable!
-              raise ForwardOutputConnectionClosedError, "node #{node.host}:#{node.port} closed connection"
-            else
-              # Serialization type of the response is same as sent data.
-              res = MessagePack.unpack(raw_data)
-
-              if res['ack'] != option['chunk']
-                # Some errors may have occured when ack and chunk id is different, so send the chunk again.
-                raise ForwardOutputResponseError, "ack in response and chunk id in sent data are different"
-              end
-            end
-
-          else
-            # IO.select returns nil on timeout.
-            # There are 2 types of cases when no response has been received:
-            # (1) the node does not support sending responses
-            # (2) the node does support sending response but responses have not arrived for some reasons.
-            @log.warn "no response from #{node.host}:#{node.port}. regard it as unavailable."
-            node.disable!
-            raise ForwardOutputACKTimeoutError, "node #{node.host}:#{node.port} does not return ACK"
-          end
-        end
-
-        node.heartbeat(false)
-        res  # for test
-      ensure
-        sock.close_write
-        sock.close
-      end
-    end
-
-    def connect(node)
-      # TODO unix socket?
-      TCPSocket.new(node.resolved_host, node.port)
-    end
-
     class HeartbeatRequestTimer < Coolio::TimerWatcher
       def initialize(interval, callback)
         super(interval, true)
@@ -390,15 +287,11 @@ module Fluent
           rebuild_weight_array
         end
         begin
-          #log.trace "sending heartbeat #{n.host}:#{n.port} on #{@heartbeat_type}"
-          if @heartbeat_type == :tcp
-            send_heartbeat_tcp(n)
-          else
-            @usock.send "\0", 0, Socket.pack_sockaddr_in(n.port, n.resolved_host)
-          end
+          log.trace "sending heartbeat", host: n.host, port: n.port, heartbeat_type: @heartbeat_type
+          n.usock = @usock if @usock
+          n.send_heartbeat
         rescue Errno::EAGAIN, Errno::EWOULDBLOCK, Errno::EINTR, Errno::ECONNREFUSED
-          # TODO log
-          log.debug "failed to send heartbeat packet to #{n.host}:#{n.port}", error: $!.to_s
+          log.debug "failed to send heartbeat packet", host: n.host, port: n.port, heartbeat_type: @heartbeat_type, error: $!
         end
       }
     end
@@ -434,27 +327,38 @@ module Fluent
       end
     end
 
-    NodeConfig = Struct.new("NodeConfig", :name, :host, :port, :weight, :standby, :failure,
-      :phi_threshold, :recover_sample_size, :expire_dns_cache, :phi_failure_detector, :dns_round_robin)
-
     class Node
-      def initialize(log, conf)
-        @log = log
-        @conf = conf
-        @name = @conf.name
-        @host = @conf.host
-        @port = @conf.port
-        @weight = @conf.weight
-        @failure = @conf.failure
+      def initialize(sender, server, failure:)
+        @sender = sender
+        @log = sender.log
+
+        @name = server.name
+        @host = server.host
+        @port = server.port
+        @weight = server.weight
+        @standby = server.standby
+        @failure = failure
         @available = true
+        @state = nil
+
+        @usock = nil
+
+        @username = server.username
+        @password = server.password
+        @shared_key = server.shared_key || (sender.security && sender.security.shared_key) || ""
+        @shared_key_salt = generate_salt
+        @shared_key_nonce = ""
+
+        @unpacker = Fluent::Engine.msgpack_unpacker
 
         @resolved_host = nil
         @resolved_time = 0
         resolved_host  # check dns
       end
 
-      attr_reader :conf
-      attr_reader :name, :host, :port, :weight
+      attr_accessor :usock
+
+      attr_reader :name, :host, :port, :weight, :standby, :state
       attr_reader :sockaddr  # used by on_heartbeat
       attr_reader :failure, :available # for test
 
@@ -467,33 +371,177 @@ module Fluent
       end
 
       def standby?
-        @conf.standby
+        @standby
+      end
+
+      def connect
+        TCPSocket.new(resolved_host, port)
+      end
+
+      def set_socket_options(sock)
+        opt = [1, @sender.send_timeout.to_i].pack('I!I!')  # { int l_onoff; int l_linger; }
+        sock.setsockopt(Socket::SOL_SOCKET, Socket::SO_LINGER, opt)
+
+        opt = [@sender.send_timeout.to_i, 0].pack('L!L!')  # struct timeval
+        sock.setsockopt(Socket::SOL_SOCKET, Socket::SO_SNDTIMEO, opt)
+
+        sock
+      end
+
+      def establish_connection(sock)
+        while available? && @state != :established
+          begin
+            # TODO: On Ruby 2.2 or earlier, read_nonblock doesn't work expectedly.
+            # We need rewrite around here using new socket/server plugin helper.
+            buf = sock.read_nonblock(@sender.read_length)
+            if buf.empty?
+              sleep @sender.read_interval
+              next
+            end
+            @unpacker.feed_each(buf) do |data|
+              on_read(sock, data)
+            end
+          rescue IO::WaitReadable
+            # If the exception is Errno::EWOULDBLOCK or Errno::EAGAIN, it is extended by IO::WaitReadable.
+            # So IO::WaitReadable can be used to rescue the exceptions for retrying read_nonblock.
+            # http://docs.ruby-lang.org/en/2.3.0/IO.html#method-i-read_nonblock
+            sleep @sender.read_interval unless @state == :established
+          rescue SystemCallError => e
+            @log.warn "disconnected by error", host: @host, port: @port, error: e
+            disable!
+            break
+          rescue EOFError
+            @log.warn "disconnected", host: @host, port: @port
+            disable!
+            break
+          end
+        end
+      end
+
+      def send_data(tag, chunk)
+        sock = connect
+        @state = @sender.security ? :helo : :established
+        begin
+          set_socket_options(sock)
+
+          if @state != :established
+            establish_connection(sock)
+          end
+
+          unless available?
+            raise ForwardOutputConnectionClosedError, "failed to establish connection with node #{@name}"
+          end
+
+          option = { 'size' => chunk.size_of_events }
+          option['chunk'] = Base64.encode64(chunk.unique_id) if @sender.require_ack_response
+
+          # out_forward always uses Raw32 type for content.
+          # Raw16 can store only 64kbytes, and it should be much smaller than buffer chunk size.
+
+          sock.write @sender.forward_header        # beginArray(3)
+          sock.write tag.to_msgpack                # 1. writeRaw(tag)
+          sock.write [0xdb, chunk.size].pack('CN') # 2. beginRaw(size) raw32
+          chunk.write_to(sock)                     #    writeRawBody(packed_es)
+          sock.write option.to_msgpack             # 3. writeOption(option)
+
+          if @sender.require_ack_response
+            # Waiting for a response here results in a decrease of throughput because a chunk queue is locked.
+            # To avoid a decrease of troughput, it is necessary to prepare a list of chunks that wait for responses
+            # and process them asynchronously.
+            if IO.select([sock], nil, nil, @sender.ack_response_timeout)
+              raw_data = begin
+                           sock.recv(1024)
+                         rescue Errno::ECONNRESET
+                           ""
+                         end
+
+              # When connection is closed by remote host, socket is ready to read and #recv returns an empty string that means EOF.
+              # If this happens we assume the data wasn't delivered and retry it.
+              if raw_data.empty?
+                @log.warn "node closed the connection. regard it as unavailable.", host: @host, port: @port
+                disable!
+                raise ForwardOutputConnectionClosedError, "node #{@host}:#{@port} closed connection"
+              else
+                @unpacker.feed(raw_data)
+                res = @unpacker.read
+                if res['ack'] != option['chunk']
+                  # Some errors may have occured when ack and chunk id is different, so send the chunk again.
+                  raise ForwardOutputResponseError, "ack in response and chunk id in sent data are different"
+                end
+              end
+
+            else
+              # IO.select returns nil on timeout.
+              # There are 2 types of cases when no response has been received:
+              # (1) the node does not support sending responses
+              # (2) the node does support sending response but responses have not arrived for some reasons.
+              @log.warn "no response from node. regard it as unavailable.", host: @host, port: @port
+              disable!
+              raise ForwardOutputACKTimeoutError, "node #{host}:#{port} does not return ACK"
+            end
+          end
+
+          heartbeat(false)
+          res  # for test
+        ensure
+          sock.close_write
+          sock.close
+        end
+      end
+
+      # FORWARD_TCP_HEARTBEAT_DATA = FORWARD_HEADER + ''.to_msgpack + [].to_msgpack
+      def send_heartbeat
+        case @sender.heartbeat_type
+        when :tcp
+          sock = connect
+          begin
+            opt = [1, @sender.send_timeout.to_i].pack('I!I!')  # { int l_onoff; int l_linger; }
+            sock.setsockopt(Socket::SOL_SOCKET, Socket::SO_LINGER, opt)
+            # opt = [@sender.send_timeout.to_i, 0].pack('L!L!')  # struct timeval
+            # sock.setsockopt(Socket::SOL_SOCKET, Socket::SO_SNDTIMEO, opt)
+
+            ## don't send any data to not cause a compatibility problem
+            # sock.write FORWARD_TCP_HEARTBEAT_DATA
+
+            # successful tcp connection establishment is considered as valid heartbeat
+            heartbeat(true)
+          ensure
+            sock.close_write
+            sock.close
+          end
+        when :udp
+          @usock.send "\0", 0, Socket.pack_sockaddr_in(n.port, n.resolved_host)
+        when :none # :none doesn't use this class
+          raise "BUG: heartbeat_type none must not use Node"
+        else
+          raise "BUG: unknown heartbeat_type '#{@sender.heartbeat_type}'"
+        end
       end
 
       def resolved_host
-        case @conf.expire_dns_cache
+        case @sender.expire_dns_cache
         when 0
           # cache is disabled
-          return resolve_dns!
+          resolve_dns!
 
         when nil
           # persistent cache
-          return @resolved_host ||= resolve_dns!
+          @resolved_host ||= resolve_dns!
 
         else
           now = Engine.now
           rh = @resolved_host
-          if !rh || now - @resolved_time >= @conf.expire_dns_cache
+          if !rh || now - @resolved_time >= @sender.expire_dns_cache
             rh = @resolved_host = resolve_dns!
             @resolved_time = now
           end
-          return rh
+          rh
         end
       end
 
       def resolve_dns!
         addrinfo_list = Socket.getaddrinfo(@host, @port, nil, Socket::SOCK_STREAM)
-        addrinfo = @conf.dns_round_robin ? addrinfo_list.sample : addrinfo_list.first
+        addrinfo = @sender.dns_round_robin ? addrinfo_list.sample : addrinfo_list.first
         @sockaddr = Socket.pack_sockaddr_in(addrinfo[1], addrinfo[3]) # used by on_heartbeat
         addrinfo[3]
       end
@@ -516,36 +564,122 @@ module Fluent
           return true
         end
 
-        if @conf.phi_failure_detector
+        if @sender.phi_failure_detector
           phi = @failure.phi(now)
-          #$log.trace "phi '#{@name}'", :host=>@host, :port=>@port, :phi=>phi
-          if phi > @conf.phi_threshold
-            @log.warn "detached forwarding server '#{@name}'", host: @host, port: @port, phi: phi
+          if phi > @sender.phi_threshold
+            @log.warn "detached forwarding server '#{@name}'", host: @host, port: @port, phi: phi, phi_threshold: @sender.phi_threshold
             @available = false
             @resolved_host = nil  # expire cached host
             @failure.clear
             return true
           end
         end
-        return false
+        false
       end
 
       def heartbeat(detect=true)
         now = Time.now.to_f
         @failure.add(now)
-        #@log.trace "heartbeat from '#{@name}'", :host=>@host, :port=>@port, :available=>@available, :sample_size=>@failure.sample_size
-        if detect && !@available && @failure.sample_size > @conf.recover_sample_size
+        if detect && !@available && @failure.sample_size > @sender.recover_sample_size
           @available = true
           @log.warn "recovered forwarding server '#{@name}'", host: @host, port: @port
-          return true
+          true
         else
-          return nil
+          nil
         end
       end
 
+      # TODO: #to_msgpack(string) is deprecated
       def to_msgpack(out = '')
         [@host, @port, @weight, @available].to_msgpack(out)
       end
+
+      def generate_salt
+        SecureRandom.hex(16)
+      end
+
+      def check_helo(message)
+        @log.debug "checking helo"
+        # ['HELO', options(hash)]
+        unless message.size == 2 && message[0] == 'HELO'
+          return false
+        end
+        opts = message[1] || {}
+        # make shared_key_check failed (instead of error) if protocol version mismatch exist
+        @shared_key_nonce = opts['nonce'] || ''
+        @authentication = opts['auth'] || ''
+        true
+      end
+
+      def generate_ping
+        @log.debug "generating ping"
+        # ['PING', self_hostname, sharedkey\_salt, sha512\_hex(sharedkey\_salt + self_hostname + nonce + shared_key),
+        #  username || '', sha512\_hex(auth\_salt + username + password) || '']
+        shared_key_hexdigest = Digest::SHA512.new.update(@shared_key_salt)
+          .update(@sender.security.self_hostname)
+          .update(@shared_key_nonce)
+          .update(@shared_key)
+          .hexdigest
+        ping = ['PING', @sender.security.self_hostname, @shared_key_salt, shared_key_hexdigest]
+        if !@authentication.empty?
+          password_hexdigest = Digest::SHA512.new.update(@authentication).update(@username).update(@password).hexdigest
+          ping.push(@username, password_hexdigest)
+        else
+          ping.push('','')
+        end
+        ping
+      end
+
+      def check_pong(message)
+        @log.debug "checking pong"
+        # ['PONG', bool(authentication result), 'reason if authentication failed',
+        #  self_hostname, sha512\_hex(salt + self_hostname + nonce + sharedkey)]
+        unless message.size == 5 && message[0] == 'PONG'
+          return false, 'invalid format for PONG message'
+        end
+        _pong, auth_result, reason, hostname, shared_key_hexdigest = message
+
+        unless auth_result
+          return false, 'authentication failed: ' + reason
+        end
+
+        if hostname == @sender.security.self_hostname
+          return false, 'same hostname between input and output: invalid configuration'
+        end
+
+        clientside = Digest::SHA512.new.update(@shared_key_salt).update(hostname).update(@shared_key_nonce).update(@shared_key).hexdigest
+        unless shared_key_hexdigest == clientside
+          return false, 'shared key mismatch'
+        end
+
+        return true, nil
+      end
+
+      def on_read(sock, data)
+        @log.trace __callee__
+
+        case @state
+        when :helo
+          unless check_helo(data)
+            @log.warn "received invalid helo message from #{@name}"
+            disable! # shutdown
+            return
+          end
+          sock.write(generate_ping.to_msgpack)
+          @state = :pingpong
+        when :pingpong
+          succeeded, reason = check_pong(data)
+          unless succeeded
+            @log.warn "connection refused to #{@name}: #{reason}"
+            disable! # shutdown
+            return
+          end
+          @state = :established
+          @log.debug "connection established", host: @host, port: @port
+        else
+          raise "BUG: unknown session state: #{@state}"
+        end
+      end
     end
 
     # Override Node to disable heartbeat
@@ -625,33 +759,5 @@ module Fluent
         @last = 0
       end
     end
-
-    ## TODO
-    #class RPC
-    #  def initialize(this)
-    #    @this = this
-    #  end
-    #
-    #  def list_nodes
-    #    @this.nodes
-    #  end
-    #
-    #  def list_fault_nodes
-    #    list_nodes.select {|n| !n.available? }
-    #  end
-    #
-    #  def list_available_nodes
-    #    list_nodes.select {|n| n.available? }
-    #  end
-    #
-    #  def add_node(name, host, port, weight)
-    #  end
-    #
-    #  def recover_node(host, port)
-    #  end
-    #
-    #  def remove_node(host, port)
-    #  end
-    #end
   end
 end