roma 0.8.12 → 0.8.13p1

data/ruby/server/lib/roma/command/sys_command_receiver.rb

@@ -19,6 +19,7 @@ module Roma
         end
         @rttable.enabled_failover = false
         res = broadcast_cmd("rbalse\r\n")
+        res[@stats.ap_str] = "BYE"
         send_data("#{res.inspect}\r\n")
         close_connection_after_writing
         @stop_event_loop = true
@@ -84,6 +85,7 @@ module Roma
         send_stat_result(nil,$roma.wb_get_stat,regexp)
         send_stat_result(nil,@rttable.get_stat(@stats.ap_str),regexp)
         send_stat_result(nil,conn_get_stat,regexp)
+        send_stat_result(nil,DNSCache.instance.get_stat,regexp)
         send_data("END\r\n")
       end
 
@@ -366,6 +368,244 @@ module Roma
         end
       end
 
+      def ev_set_descriptor_table_size(s)
+        if s.length != 2
+          return send_data("CLIENT_ERROR number of arguments\r\n")
+        elsif s[1].to_i < 1024
+          return send_data("CLIENT_ERROR length must be greater than 1024\r\n")
+        end
+
+        res = broadcast_cmd("rset_descriptor_table_size #{s[1]}\r\n")
+
+        EM.set_descriptor_table_size(s[1].to_i)
+        res[@stats.ap_str] = "STORED"
+        send_data("#{res}\r\n")
+      end
+
+      def ev_rset_descriptor_table_size(s)
+        if s.length != 2
+          return send_data("CLIENT_ERROR number of arguments\r\n")
+        elsif s[1].to_i < 1024
+          return send_data("CLIENT_ERROR length must be greater than 1024\r\n")
+        end
+
+        EM.set_descriptor_table_size(s[1].to_i)
+        send_data("STORED\r\n")
+      end
+
+      # set_latency_avg_calc_rule <mode> <count> <command1> <command2>....
+      # <mode> is on/off
+      # <count> is denominator to calculate average. 
+      # <commandx> is target command
+      def ev_set_latency_avg_calc_rule(s)
+        #check argument
+        if /^on$|^off$/ !~ s[1]
+          return send_data("CLIENT_ERROR argument 1: please input \"on\" or \"off\"\r\n")
+        elsif s[1] == "on" && (s.length <= 3 || s[2].to_i < 1)
+          return send_data("CLIENT_ERROR number of arguments (0 for 3) and <count> must be greater than zero\r\n")
+        elsif s[1] == "off" && !(s.length == 2)
+          return send_data("CLIENT_ERROR number of arguments (0 for 1, or more 3)\r\n")
+        end
+
+        #check support commands
+        s.each_index {|idx|
+          if idx >= 3 && (!Event::Handler::ev_list.include?(s[idx]) || Event::Handler::system_commands.include?(s[idx]))
+             return send_data("NOT SUPPORT [#{s[idx]}] command\r\n")
+          end
+        }
+
+        arg ="rset_latency_avg_calc_rule"
+        s.each_index {|idx|
+          arg += " #{s[idx]}" if idx>=1
+        }
+        res = broadcast_cmd("#{arg}\r\n")
+
+        if s[1] =="on"
+          @stats.latency_check_cmd = [] #reset
+          s.each_index {|idx|
+            @stats.latency_check_cmd.push(s[idx]) if idx >= 3
+          }
+          @stats.latency_check_time_count = s[2].to_i
+          @stats.latency_log = true
+          res[@stats.ap_str] = "ACTIVATED"
+        elsif s[1] =="off"
+          @stats.latency_check_cmd = [] #reset
+          @stats.latency_check_time_count = nil
+          @stats.latency_log = false
+          res[@stats.ap_str] = "DEACTIVATED"
+        end
+        @stats.latency_data = Hash.new { |hash,key| hash[key] = {}}
+        send_data("#{res}\r\n")
+      end
+
+      def ev_rset_latency_avg_calc_rule(s)
+        if /^on$|^off$/ !~ s[1]
+          return send_data("CLIENT_ERROR argument 1: please input \"on\" or \"off\"\r\n")
+        elsif s[1] == "on" && (s.length <= 3 || s[2].to_i < 1)
+          return send_data("CLIENT_ERROR number of arguments (0 for 3) and <count> must be greater than zero\r\n")
+        elsif s[1] == "off" && !(s.length == 2)
+          return send_data("CLIENT_ERROR number of arguments (0 for 1, or more 3)\r\n")
+        end
+
+        s.each_index {|idx|
+          if idx >= 3 && (!Event::Handler::ev_list.include?(s[idx]) || Event::Handler::system_commands.include?(s[idx]))
+             return send_data("NOT SUPPORT [#{s[idx]}] command\r\n")
+          end
+        }
+
+        if s[1] =="on"
+          @latency_data = Hash.new { |hash,key| hash[key] = {}}
+          @stats.latency_check_cmd = []
+          s.each_index {|idx|
+            @stats.latency_check_cmd.push(s[idx]) if idx >= 3
+          }
+          @stats.latency_check_time_count = s[1].to_i
+          @stats.latency_log = true
+          send_data("ACTIVATED\r\n")
+        elsif s[1] =="off"
+          @latency_data = Hash.new { |hash,key| hash[key] = {}}
+          @stats.latency_check_cmd = []
+          @stats.latency_check_time_count = nil
+          @stats.latency_log = false
+          send_data("DEACTIVATED\r\n")
+        end
+      end
+
+      # add_calc_latency_average <command1> <command2>....
+      def ev_add_latency_avg_calc_cmd(s)
+        #check argument
+        if s.length < 2
+          return send_data("CLIENT_ERROR number of arguments (0 for 2)\r\n")
+        end
+        #check support commands
+        s.each_index {|idx|
+          if idx >= 1 && (!Event::Handler::ev_list.include?(s[idx]) || Event::Handler::system_commands.include?(s[idx]))
+             return send_data("NOT SUPPORT [#{s[idx]}] command\r\n")
+          end
+          if idx >= 1 && @stats.latency_check_cmd.include?(s[idx])
+            return send_data("ALREADY SET [#{s[idx]}] command\r\n")
+          end
+        }
+
+        arg ="radd_latency_avg_calc_cmd"
+        s.each_index {|idx|
+          arg += " #{s[idx]}" if idx>=1
+        }
+        res = broadcast_cmd("#{arg}\r\n")
+
+        s.each_index {|idx|
+          @stats.latency_check_cmd.push(s[idx]) if idx >= 1
+        }
+        res[@stats.ap_str] = "SET"
+        send_data("#{res}\r\n")
+      end
+
+      def ev_radd_latency_avg_calc_cmd(s)
+        if s.length < 2
+          return send_data("CLIENT_ERROR number of arguments (0 for 2)\r\n")
+        end
+        s.each_index {|idx|
+          if idx >= 2 && (!Event::Handler::ev_list.include?(s[idx]) || Event::Handler::system_commands.include?(s[idx]))
+             return send_data("NOT SUPPORT [#{s[idx]}] command\r\n")
+          end
+          if idx >= 1 && @stats.latency_check_cmd.include?(s[idx])
+            return send_data("ALREADY SET [#{s[idx]}] command\r\n")
+          end
+        }
+
+        s.each_index {|idx|
+          @stats.latency_check_cmd.push(s[idx]) if idx >= 1
+        }
+        send_data("SET\r\n")
+      end
+
+      # del_calc_latency_average <command1> <command2>....
+      def ev_del_latency_avg_calc_cmd(s)
+        #check argument
+        if s.length < 2
+          return send_data("CLIENT_ERROR number of arguments (0 for 2)\r\n")
+        end
+        
+        #check support commands
+        s.each_index {|idx|
+          if idx >= 1 && !@stats.latency_check_cmd.include?(s[idx])
+            return send_data("[#{s[idx]}] command is NOT set\r\n")
+          end
+        }
+
+        arg ="rdel_latency_avg_calc_cmd"
+        s.each_index {|idx|
+          arg += " #{s[idx]}" if idx>=1
+        }
+        res = broadcast_cmd("#{arg}\r\n")
+
+        s.each_index {|idx|
+          @stats.latency_check_cmd.delete(s[idx]) if idx >= 1
+          @stats.latency_data.delete(s[idx]) if idx >= 1
+        }
+        res[@stats.ap_str] = "DELETED"
+        send_data("#{res}\r\n")
+      end
+
+      def ev_rdel_latency_avg_calc_cmd(s)
+        if s.length < 2
+          return send_data("CLIENT_ERROR number of arguments (0 for 2)\r\n")
+        end
+
+        # reset
+        s.each_index {|idx|
+          if idx >= 1 && !@stats.latency_check_cmd.include?(s[idx])
+            return send_data("[#{s[idx]}] command is NOT set\r\n")
+          end
+        }
+        s.each_index {|idx|
+          @stats.latency_check_cmd.delete(s[idx]) if idx >= 1
+          @stats.latency_data.delete(s[idx]) if idx >= 1
+        }
+        send_data("DELETED\r\n")
+      end
+
+      # chg_calc_latency_average_denominator <count>
+      def ev_chg_latency_avg_calc_time_count(s)
+        #check argument
+        if s.length != 2
+          return send_data("CLIENT_ERROR number of arguments (0 for 2)\r\n")
+        elsif s[1] != "nil" && s[1].to_i < 1
+          return send_data("s[1].class = #{s[1].class}\r\n")
+          return send_data("<count> must be greater than zero or nil[DEACTIVATE]\r\n")
+        end
+
+        res = broadcast_cmd("rchg_latency_avg_calc_time_count #{s[1]}\r\n")
+
+        if s[1] != "nil"
+          @stats.latency_check_time_count = s[1].to_i
+          @stats.latency_log = true
+        elsif s[1] == "nil"
+          @stats.latency_check_time_count = nil
+          @stats.latency_log = false
+        end
+        res[@stats.ap_str] = "CHANGED"
+        send_data("#{res}\r\n")
+      end
+
+      def ev_rchg_latency_avg_calc_time_count(s)
+        if s.length != 2
+          return send_data("CLIENT_ERROR number of arguments (0 for 2)\r\n")
+        elsif s[1] != "nil" && s[1].to_i < 1
+          return send_data("<count> must be greater than zero\r\n")
+        end
+
+        if s[1] != "nil"
+          @stats.latency_check_time_count = s[1].to_i
+          @stats.latency_log = true
+        elsif s[1] == "nil"
+          @stats.latency_check_time_count = nil
+          @stats.latency_log = false
+        end
+        @stats.latency_check_time_count = s[1].to_i
+        send_data("CHANGED\r\n")
+      end
+
       def ev_set_continuous_limit(s)
         if s.length < 2
           return send_data("CLIENT_ERROR number of arguments (0 for 1)\r\n")
@@ -472,6 +712,94 @@ module Roma
         send_data("STORED\r\n")
       end
 
+      # set_connection_pool_expire_time <sec>
+      # set to expired time(sec) for connection_pool expire time
+      def ev_set_connection_pool_expire_time(s)
+        if s.length != 2
+          return send_data("CLIENT_ERROR number of arguments\r\n")
+        end
+
+        res = broadcast_cmd("rset_connection_pool_expire_time #{s[1]}\r\n")
+        Messaging::ConPool.instance.expire_time = s[1].to_i
+        res[@stats.ap_str] = "STORED"
+        send_data("#{res}\r\n")
+      end
+
+      # rset_connection_pool_expire_time <sec>
+      def ev_rset_connection_pool_expire_time(s)
+        if s.length != 2
+          return send_data("CLIENT_ERROR number of arguments\r\n")
+        end
+        Messaging::ConPool.instance.expire_time = s[1].to_i
+        send_data("STORED\r\n")
+      end
+
+      # set_emconnection_pool_expire_time <sec>
+      def ev_set_emconnection_pool_expire_time(s)
+        # chcking s incude command and value (NOT check digit)
+        if s.length != 2
+          return send_data("CLIENT_ERROR number of arguments\r\n")
+        end
+
+        #if ARGV is 0, expire time become infinity(NOT happen expire)
+        if s[1].to_i == 0
+          s[1] = "2147483647"
+        end
+        res = broadcast_cmd("rset_emconnection_pool_expire_time #{s[1]}\r\n")
+        Event::EMConPool::instance.expire_time = s[1].to_i
+        res[@stats.ap_str] = "STORED"
+        send_data("#{res}\r\n")
+      end
+
+      # rset_emconnection_pool_expire_time <sec>
+      def ev_rset_emconnection_pool_expire_time(s)
+        if s.length != 2
+          return send_data("CLIENT_ERROR number of arguments\r\n")
+        end
+        Event::EMConPool::instance.expire_time = s[1].to_i
+        send_data("STORED\r\n")
+      end
+
+      # switch_dns_caching <on|off|true|false>
+      def ev_switch_dns_caching(s)
+        if s.length != 2
+          return send_data("CLIENT_ERROR number of arguments\r\n")
+        end
+
+        res = broadcast_cmd("rswitch_dns_caching #{s[1]}\r\n")
+        if s[1] == 'on' || s[1] == 'true'
+          DNSCache.instance.enable_dns_cache
+          @log.info("DNS caching enabled")
+          res[@stats.ap_str] = "ENABLED"
+        elsif s[1] == 'off' || s[1] == 'false'
+          DNSCache.instance.disable_dns_cache
+          @log.info("DNS caching disabled")
+          res[@stats.ap_str] = "DISABLED"
+        else
+          res[@stats.ap_str] = "NOTSWITCHED"
+        end
+        send_data("#{res}\r\n")
+      end
+
+      # rswitch_dns_caching <on|off|true|false>
+      def ev_rswitch_dns_caching(s)
+        if s.length != 2
+          return send_data("CLIENT_ERROR number of arguments\r\n")
+        end
+
+        if s[1] == 'on' || s[1] == 'true'
+          DNSCache.instance.enable_dns_cache
+          @log.info("DNS caching enabled")
+          return send_data("ENABLED\r\n")
+        elsif s[1] == 'off' || s[1] == 'false'
+          DNSCache.instance.disable_dns_cache
+          @log.info("DNS caching disabled")
+          return send_data("DISABLED\r\n")
+        else
+          send_data("NOTSWITCHED\r\n")
+        end
+      end
+
       # set_hilatency_warn_time <sec>
       # set to threshold of warn message into a log when hilatency occured in a command.
       def ev_set_hilatency_warn_time(s)
@@ -542,6 +870,31 @@ module Roma
         end        
       end
 
+      # set_wb_shift_size <size>
+      def ev_set_wb_shift_size(s)
+        if s.length != 2
+          return send_data("CLIENT_ERROR number of arguments\r\n")
+        elsif s[1].to_i < 1
+          return send_data("CLIENT_ERROR length must be greater than zero\r\n")
+        end
+
+        res = broadcast_cmd("rset_wb_shift_size #{s[1]}\r\n")
+        $roma.wb_writer.shift_size = s[1].to_i
+        res[@stats.ap_str] = "STORED"
+        send_data("#{res}\r\n")
+      end
+
+      def ev_rset_wb_shift_size(s)
+        if s.length != 2
+          return send_data("CLIENT_ERROR number of arguments\r\n")
+        elsif s[1].to_i < 1
+          return send_data("CLIENT_ERROR length must be greater than zero\r\n")
+        end
+
+        $roma.wb_writer.shift_size = s[1].to_i
+        send_data("STORED\r\n")
+      end
+
       private 
 
       def dcnice(p)

data/ruby/server/lib/roma/command/vn_command_receiver.rb

@@ -45,14 +45,20 @@ module Roma
         count = rcount = 0
         @log.debug("#{__method__}:#{s.inspect} received.")
         loop {
-          context_bin = read_bytes(20, 100)
+          context_bin = read_bytes(20, @stats.spushv_read_timeout)
           vn, last, clk, expt, klen = context_bin.unpack('NNNNN')
           break if klen == 0 # end of dump ?
           k = read_bytes(klen)
-          vlen_bin = read_bytes(4, 100)
+          vlen_bin = read_bytes(4, @stats.spushv_read_timeout)
           vlen, =  vlen_bin.unpack('N')
           if vlen != 0
-            v = read_bytes(vlen, 100)
+            if klen > @stats.spushv_klength_warn
+              @log.warn("#{__method__}:Too long key: key = #{k}")
+            end
+            if vlen > @stats.spushv_vlength_warn
+              @log.warn("#{__method__}:Too long value: key = #{k} vlen = #{vlen}")
+            end
+            v = read_bytes(vlen, @stats.spushv_read_timeout)
 
             createhash(s[1]) unless @storages[s[1]]
             if @storages[s[1]].load_stream_dump(vn, last, clk, expt, k, v)

data/ruby/server/lib/roma/config.rb

@@ -8,6 +8,8 @@ module Roma
 
     # :no_action | :auto_assign | :shutdown
     DEFAULT_LOST_ACTION = :auto_assign
+    # to use a auto recover function, AUTO_RECOVER is to set true
+    AUTO_RECOVER = false
 
     # failover setting
     # threshold of failover occurrence
@@ -50,6 +52,10 @@ module Roma
     # max length of the eventmachine's connection pool
     CONNECTION_EMPOOL_MAX = 15
 
+    # lookup hostname to the nameserver everytime or only first time.
+    # if set true, the result of lookup will keep.
+    DNS_CACHING = false
+
     # storage setting
     STORAGE_CLASS = Roma::Storage::RubyHashStorage
     STORAGE_DIVNUM = 10

data/ruby/server/lib/roma/dns_cache.rb

@@ -0,0 +1,40 @@
+require 'singleton'
+
+module Roma
+  class DNSCache
+    include Singleton
+
+    def initialize
+      @@addrs = {}
+      @@enabled_caching = false
+      if Config.const_defined?(:DNS_CACHING)
+        @@enabled_caching = Config::DNS_CACHING
+      end
+    end
+
+    def resolve_name(host)
+      return host unless @@enabled_caching
+
+      unless @@addrs.include?(host)
+        res = TCPSocket.gethostbyname(host)
+        @@addrs[host] = res[3]
+      end
+      @@addrs[host]
+    end
+
+    def disable_dns_cache
+      @@enabled_caching = false
+      @@addrs.clear
+    end
+
+    def enable_dns_cache
+      @@enabled_caching = true
+    end
+
+    def get_stat
+      ret = {}
+      ret["dns_caching"] = @@enabled_caching
+      ret
+    end
+  end
+end

data/ruby/server/lib/roma/event/con_pool.rb

@@ -4,6 +4,7 @@
 require 'singleton'
 require 'eventmachine'
 require 'roma/logging/rlogger'
+require 'roma/dns_cache'
 
 module Roma
   module Event
@@ -114,7 +115,8 @@ module Roma
       end
 
       def create_connection(ap)
-        addr,port = ap.split(/[:_]/)
+        host,port = ap.split(/[:_]/)
+        addr = DNSCache.instance.resolve_name(host)
         con = EventMachine::connect(addr, port, Roma::Event::EMConnection)
         con.ap = ap
         con

data/ruby/server/lib/roma/event/handler.rb

@@ -50,7 +50,15 @@ module Roma
         @@connection_expire_time
       end
 
-      attr_accessor :timeout
+      @@timeout = 10
+      def self.timeout=(n)
+        @@timeout = n
+      end
+
+      def self.timeout
+        @@timeout
+      end
+
       attr_reader :connected
       attr_reader :lastcmd
       attr_reader :last_access
@@ -71,7 +79,6 @@ module Roma
 
         @storages = storages
         @rttable = rttable
-        @timeout = 10
         @log = Roma::Logging::RLogger.instance
         @last_access = Time.now
       end
@@ -143,11 +150,13 @@ module Roma
 
       def dispatcher
         @stats = Roma::Stats.instance
+        #@log.debug("Roma::Event::Handler.dipatcher called")
         while(@connected) do
           @enter_time = nil
           next unless s=gets
           @enter_time = Time.now
           s=s.chomp.split(/ /)
+          # check whether comand was send or not? and check this command listed on ROMA?
           if s[0] && @@ev_list.key?(s[0].downcase)
             send(@@ev_list[s[0].downcase],s)
             @lastcmd=s
@@ -161,6 +170,7 @@ module Roma
             @log.warn("command error:#{s}")
             send_data("ERROR\r\n")
             close_connection_after_writing
+            next
           end
 
           # hilatency check
@@ -168,6 +178,10 @@ module Roma
           if ps > @stats.hilatency_warn_time
             @log.warn("hilatency occurred in #{@lastcmd} put in a #{ps} seconds")
           end
+          # check latency average
+          if @stats.latency_check_cmd.include?(@lastcmd[0])
+            Roma::AsyncProcess::queue_latency.push(Roma::AsyncMessage.new('calc_latency_average', [ps, @lastcmd[0]]))
+          end
 
           d = EM.connection_count - @@ccl_start
           if d > 0 &&
@@ -216,7 +230,7 @@ module Roma
           else
             remain = size - @rbuf.size
             Fiber.yield(remain)
-            if Time.now.to_i - t > @timeout * mult
+            if Time.now.to_i - t > @@timeout * mult
               @log.warn("#{__FILE__}:#{__LINE__}:#{@addr}:#{@port} read_bytes time out");
               close_connection
               return nil
@@ -245,6 +259,7 @@ module Roma
       def conn_get_stat
         ret = {}
         ret["connection.count"] = EM.connection_count
+        ret["connection.descriptor_table_size"] = EM.set_descriptor_table_size
         ret["connection.continuous_limit"] = Handler.get_ccl
         ret["connection.accepted_connection_expire_time"] = Handler.connection_expire_time
         ret["connection.handler_instance_count"] = Handler.connections.length

data/ruby/server/lib/roma/messaging/con_pool.rb

@@ -1,6 +1,7 @@
 require 'thread'
 require 'socket'
 require 'singleton'
+require 'roma/dns_cache'
 
 module Roma
   module Messaging
@@ -53,7 +54,8 @@ module Roma
       end
 
       def create_connection(ap)
-        addr, port = ap.split(/[:_]/)
+        host, port = ap.split(/[:_]/)
+        addr = DNSCache.instance.resolve_name(host)
         TCPSocket.new(addr, port)
       end
 

data/ruby/server/lib/roma/romad.rb

@@ -21,6 +21,7 @@ module Roma
     attr :storages
     attr :rttable
     attr :stats
+    attr :wb_writer
 
     attr_accessor :eventloop
     attr_accessor :startup
@@ -36,7 +37,7 @@ module Roma
       initialize_storages
       initialize_handler
       initialize_plugin
-      initialize_wb_witer
+      initialize_wb_writer
     end
 
     def start
@@ -191,7 +192,7 @@ module Roma
       end
     end
 
-    def initialize_wb_witer
+    def initialize_wb_writer
       @wb_writer = Roma::WriteBehind::FileWriter.new(
                                                      Roma::Config::WRITEBEHIND_PATH, 
                                                      Roma::Config::WRITEBEHIND_SHIFT_SIZE,
@@ -379,6 +380,10 @@ module Roma
         end
       end
       
+      if Roma::Config.const_defined?(:RTTABLE_SUB_NID)
+        @rttable.sub_nid = Roma::Config::RTTABLE_SUB_NID
+      end
+      
       if Roma::Config.const_defined?(:ROUTING_FAIL_CNT_THRESHOLD)
         @rttable.fail_cnt_threshold = Roma::Config::ROUTING_FAIL_CNT_THRESHOLD
       end
@@ -386,6 +391,8 @@ module Roma
         @rttable.fail_cnt_gap = Roma::Config::ROUTING_FAIL_CNT_GAP
       end
       @rttable.lost_action = Roma::Config::DEFAULT_LOST_ACTION
+      @rttable.auto_recover = Roma::Config::AUTO_RECOVER if defined?(Roma::Config::AUTO_RECOVER)
+
       @rttable.enabled_failover = false
       @rttable.set_leave_proc{|nid|
         Roma::Messaging::ConPool.instance.close_same_host(nid)
@@ -399,6 +406,13 @@ module Roma
           @log.error("Romad has stopped, so that lose data.")
         end
       }
+      @rttable.set_recover_proc{|action|
+        if (@rttable.lost_action == :shutdown || @rttable.lost_action == :auto_assign) && @rttable.auto_recover == true
+          Roma::AsyncProcess::queue.push(Roma::AsyncMessage.new("#{action}"))
+        elsif
+          @log.error("AUTO_RECOVER is off or Unavailable value is set to [DEFAULT_LOST_ACTION] => #{@rttable.lost_action}")
+        end
+      }
     end
 
     def initialize_rttable_join
@@ -507,6 +521,8 @@ module Roma
           @stats.run_storage_clean_up
         stop_clean_up
       end
+    rescue Exception =>e
+      @log.error("#{e}\n#{$@}")
     end
 
     def timer_event_10sec
@@ -525,7 +541,7 @@ module Roma
         @log.warn("failover disable now!!")
       else
         version_check
-        @rttable.delete_old_trans
+        @rttable.delete_old_trans(@stats.routing_trans_timeout)
         start_sync_routing_process
       end