roma 0.8.2 → 0.8.10

data/{lib → ruby/server/lib}/roma/command/util_command_receiver.rb

@@ -9,14 +9,14 @@ module Roma
         con = get_connection(nid)
         con.send(cmd)
         res = con.gets
-        if res
-          res.chomp!
+        if res == nil
+          @rttable.proc_failed(nid)
+          return nil
+        elsif res.start_with?("ERROR") == false
           @rttable.proc_succeed(nid)
           return_connection(nid, con)
-        else
-          @rttable.proc_failed(nid)
         end
-        res
+        res.chomp
       rescue => e
         @rttable.proc_failed(nid)
         @log.error("#{e}\n#{$@}")
@@ -61,6 +61,16 @@ module Roma
         nil
       end
 
+      # change to actual time for a memcached's expire time value
+      def chg_time_expt(expt)
+        if expt == 0
+          expt = 0x7fffffff
+        elsif expt < 2592000
+          expt += Time.now.to_i
+        end
+        expt
+      end
+
     end # module UtilCommandReceiver
 
   end # module Command

data/{lib → ruby/server/lib}/roma/command/vn_command_receiver.rb

@@ -50,7 +50,8 @@ module Roma
           vlen, =  vlen_bin.unpack('N')
           if vlen != 0
             v = read_bytes(vlen, 100)
-            
+
+            createhash(s[1]) unless @storages[s[1]]
             if @storages[s[1]].load_stream_dump(vn, last, clk, expt, k, v)
               count += 1
 #              @log.debug("#{__method__}:[#{vn} #{last} #{clk} #{expt} #{k}] was stored.")
@@ -59,6 +60,7 @@ module Roma
 #              @log.warn("#{__method__}:[#{vn} #{last} #{clk} #{expt} #{k}] was rejected.")
             end
           else
+            createhash(s[1]) unless @storages[s[1]]
             if @storages[s[1]].load_stream_dump(vn, last, clk, expt, k, nil)
 #              @log.debug("#{__method__}:[#{vn} #{last} #{clk} #{expt} #{k}] was stored.")
               count += 1
@@ -70,8 +72,16 @@ module Roma
         }
         send_data("STORED\r\n")
         @log.debug("#{__method__}:#{s[2]} #{count} keys loaded. #{rcount} keys rejected.")
+      rescue Storage::StorageException => e
+        @log.error("#{e.inspect} #{$@}")
+        close_connection
+        if Config.const_defined?(:STORAGE_EXCEPTION_ACTION) &&
+            Config::STORAGE_EXCEPTION_ACTION == :shutdown
+          @log.error("Romad will stop")
+          @stop_event_loop = true
+        end
       rescue => e
-        @log.error("#{e}\n#{$@}")
+        @log.error("#{e} #{$@}")
       ensure
         @stats.run_receive_a_vnode = false
       end      
@@ -85,13 +95,11 @@ module Roma
           send_data("CLIENT_ERROR usage:reqpushv vnode-id node-id primary-flag(true/false)\r\n")
           return
         end
-
         if @stats.run_iterate_storage == true
           @log.warn("reqpushv rejected:#{s}")
           send_data("REJECTED\r\n")
           return
         end
-
         Roma::AsyncProcess::queue.push(Roma::AsyncMessage.new('reqpushv',[s[1],s[2],s[3]]))
         send_data("PUSHED\r\n")
       rescue =>e

data/ruby/server/lib/roma/config.rb

@@ -0,0 +1,84 @@
+require 'roma/storage/rh_storage'
+
+module Roma
+  
+  module Config
+    DEFAULT_PORT = 12000
+    DEFAULT_NAME = 'ROMA'
+
+    # :no_action | :auto_assign | :shutdown
+    DEFAULT_LOST_ACTION = :auto_assign
+
+    # failover setting
+    # threshold of failover occurrence
+    ROUTING_FAIL_CNT_THRESHOLD = 15
+    # ROUTING_FAIL_CNT_GAP(sec) doesn't increase the failover counter.
+    ROUTING_FAIL_CNT_GAP = 0
+
+    # log setting
+    LOG_SHIFT_AGE = 10
+    LOG_SHIFT_SIZE = 1024 * 1024 * 10
+    LOG_PATH = '.'
+    # :debug | :info | :warn | :error
+    LOG_LEVEL = :debug
+
+    # routing setting
+    RTTABLE_PATH = '.'
+
+    # connection setting
+
+    # to use a system call of epoll, CONNECTION_USE_EPOLL is to set true
+    CONNECTION_USE_EPOLL = true
+    # to use a system call of epoll, CONNECTION_DESCRIPTOR_TABLE_SIZE can be setting
+    CONNECTION_DESCRIPTOR_TABLE_SIZE = 4096
+
+    # like a MaxStartups spec in the sshd_config
+    # 'start:rate:full'
+    CONNECTION_CONTINUOUS_LIMIT = '200:30:300'
+    # expired time(sec) for accepted connections
+    CONNECTION_EXPTIME = 0
+
+    # expired time(sec) for an async connection in the connection pool
+    # CONNECTION_POOL_EXPTIME should be less than CONNECTION_EXPTIME
+    CONNECTION_POOL_EXPTIME = 30
+    # max length of the connection pool
+    CONNECTION_POOL_MAX = 5
+
+    # expired time(sec) for an eventmachine's connection in the connection pool
+    # CONNECTION_EMPOOL_EXPTIME should be less than CONNECTION_EXPTIME
+    CONNECTION_EMPOOL_EXPTIME = 30
+    # max length of the eventmachine's connection pool
+    CONNECTION_EMPOOL_MAX = 15
+
+    # storage setting
+    STORAGE_CLASS = Roma::Storage::RubyHashStorage
+    STORAGE_DIVNUM = 10
+    STORAGE_PATH = '.'
+    STORAGE_DUMP_PATH = '/tmp'
+    STORAGE_OPTION = ''
+    # :no_action | :shutdown
+    STORAGE_EXCEPTION_ACTION = :no_action
+
+    # expired time(sec) for deleted keys, expired keys and invalid vnode keys
+    # typical value is 5 days
+    STORAGE_DELMARK_EXPTIME = 60 * 60 * 24 * 5
+
+    # data copy setting
+    DATACOPY_STREAM_COPY_WAIT_PARAM = 0.001
+
+    # plugin setting
+    PLUGIN_FILES = ['plugin_storage.rb']
+
+    # write-behind setting
+    WRITEBEHIND_PATH = './wb'
+    WRITEBEHIND_SHIFT_SIZE = 1024 * 1024 * 10
+
+    # redundant setting
+    # REDUNDANT_ZREDUNDANT_SIZE is a option for a redundancy of compressed data.
+    # when the data size is more then REDUNDANT_ZREDUNDANT_SIZE, data compression is done.
+    # however, it dose't in case of REDUNDANT_ZREDUNDANT_SIZE is zero.
+    REDUNDANT_ZREDUNDANT_SIZE = 0
+
+  end
+
+end

data/{lib → ruby/server/lib}/roma/event/con_pool.rb

@@ -11,10 +11,12 @@ module Roma
       attr_writer :fiber
       attr_reader :connected
       attr_accessor :ap
+      attr_accessor :last_access
 
       def post_init
         @rbuf = ''
         @connected = true
+        @last_access = Time.now
       end
       
       def receive_data(data)
@@ -76,21 +78,30 @@ module Roma
       include Singleton
       attr :pool
       attr_accessor :maxlength
+      attr_accessor :expire_time
 
       def initialize
         @pool = {}
-        @maxlength = 10
+        @maxlength = 30
+        @expire_time = 30
         @lock = Mutex.new
       end
 
       def get_connection(ap)
         ret = @pool[ap].shift if @pool.key?(ap) && @pool[ap].length > 0
+        if ret && ret.last_access < Time.now - @expire_time
+          ret.close_connection if ret.connected
+          ret = nil
+          Logging::RLogger.instance.info("EM connection expired at #{ap},remains #{@pool[ap].length}")
+        end
         ret = create_connection(ap) if ret == nil || ret.connected != true
         ret
       end
 
       def return_connection(ap, con)
         return if con.connected == false
+
+        con.last_access = Time.now
         if @pool.key?(ap) && @pool[ap].length > 0
           if @pool[ap].length > @maxlength
             con.close_connection

data/ruby/server/lib/roma/event/handler.rb

@@ -0,0 +1,256 @@
+# 
+# File: handler.rb
+#
+require 'eventmachine'
+require 'roma/event/con_pool'
+require 'roma/logging/rlogger'
+require 'roma/stats'
+require 'roma/storage/basic_storage'
+require 'socket'
+
+module Roma
+  module Event
+
+    class Handler < EventMachine::Connection
+      @@ev_list={}
+      def self.ev_list; @@ev_list; end
+      @@system_commands={}
+      def self.system_commands; @@system_commands; end
+
+      @@ccl_start = 200
+      @@ccl_rate = 30
+      @@ccl_full = 300
+
+      def self.get_ccl
+        "#{@@ccl_start}:#{@@ccl_rate}:#{@@ccl_full}"
+      end
+
+      def self.set_ccl(ccl)
+        if ccl =~ /^(\d+):(\d+):(\d+)$/
+          s,r,f = $1.to_i,$2.to_i,$3.to_i
+          return false if(s < 0 || f < 0 || r < 0 || r > 100 || s > f)
+          @@ccl_start = s
+          @@ccl_rate = r
+          @@ccl_full = f
+          return true
+        else
+          return false
+        end
+      end
+
+      @@connections = {}
+      def self.connections; @@connections; end
+
+      @@connection_expire_time = 60
+      def self.connection_expire_time=(t)
+        @@connection_expire_time = t
+      end
+
+      def self.connection_expire_time
+        @@connection_expire_time
+      end
+
+      attr_accessor :timeout
+      attr_reader :connected
+      attr_reader :lastcmd
+      attr_reader :last_access
+      attr_reader :addr, :port
+
+      def initialize(storages, rttable)
+        @rbuf=''
+        unless has_event?
+          public_methods.each{|m|
+            if m.to_s.start_with?('ev_')
+              add_event(m.to_s[3..-1],m)
+            end
+          }
+        end
+        @th1 = 100
+        @close_rate = 70
+        @th2 = 200
+
+        @storages = storages
+        @rttable = rttable
+        @timeout = 10
+        @log = Roma::Logging::RLogger.instance
+        @last_access = Time.now
+      end
+
+      def post_init
+        @port, @addr = Socket.unpack_sockaddr_in(get_peername)
+        @log.info("Connected from #{@addr}:#{@port}. I have #{EM.connection_count} connections.")
+        @connected = true
+        @last_access = Time.now
+        @@connections[self] = @last_access
+        @fiber = Fiber.new { dispatcher }
+      rescue Exception =>e
+        @log.error("#{__FILE__}:#{__LINE__}:#{e.inspect} #{$@}")
+      end
+
+      def receive_data(data)
+        @rbuf << data
+        @last_access = Time.now
+        @fiber.resume
+      rescue Exception =>e
+        @log.error("#{__FILE__}:#{__LINE__}:#{@addr}:#{@port} #{e.inspect} #{$@}")
+      end
+
+      def unbind
+        @connected=false
+        begin
+          @fiber.resume
+        rescue FiberError
+        end
+        EventMachine::stop_event_loop if @stop_event_loop
+        @@connections.delete(self)
+        if @enter_time
+          # hilatency check
+          ps = Time.now - @enter_time
+          if ps > @stats.hilatency_warn_time
+            @log.warn("#{@lastcmd} has incompleted, passage of #{ps} seconds")
+          end
+        end
+        @log.info("Disconnected from #{@addr}:#{@port}")
+      rescue Exception =>e
+        @log.warn("#{__FILE__}:#{__LINE__}:#{@addr}:#{@port} #{e.inspect} #{$@}")
+      end
+
+      protected
+
+      def has_event?
+        @@ev_list.length!=0
+      end
+
+      def add_event(c,m)
+        @@ev_list[c]=m
+      end
+
+      def exit
+        EventMachine::stop_event_loop
+      end
+
+      private
+
+      def get_connection(ap)
+        con=Roma::Event::EMConPool::instance.get_connection(ap)
+        con.fiber=@fiber
+        con
+      end
+
+      def return_connection(ap,con)
+        Roma::Event::EMConPool.instance.return_connection(ap,con)
+      end
+
+      def dispatcher
+        @stats = Roma::Stats.instance
+        while(@connected) do
+          @enter_time = nil
+          next unless s=gets
+          @enter_time = Time.now
+          s=s.chomp.split(/ /)
+          if s[0] && @@ev_list.key?(s[0].downcase)
+            send(@@ev_list[s[0].downcase],s)
+            @lastcmd=s
+            next if @@system_commands.key?(s[0].downcase)
+          elsif s.length==0
+            next
+          elsif s[0]=='!!'
+            send(@@ev_list[@lastcmd[0].downcase],@lastcmd)
+            next if @@system_commands.key?(@lastcmd[0].downcase)
+          else
+            @log.warn("command error:#{s}")
+            send_data("ERROR\r\n")
+            close_connection_after_writing
+          end
+
+          # hilatency check
+          ps = Time.now - @enter_time
+          if ps > @stats.hilatency_warn_time
+            @log.warn("hilatency occurred in #{@lastcmd} put in a #{ps} seconds")
+          end
+
+          d = EM.connection_count - @@ccl_start
+          if d > 0 &&
+              rand(100) < @@ccl_rate + (100 - @@ccl_rate) * d / (@@ccl_full - @@ccl_start)
+            send_data("ERROR\r\n")
+            close_connection_after_writing
+            @log.warn("Connection count > #{@@ccl_start}:closed")
+          end
+        end
+      rescue Storage::StorageException => e
+        @log.error("#{e.inspect} #{s} #{$@}")
+        send_data("SERVER_ERROR #{e} in storage engine\r\n")
+        close_connection_after_writing
+        if Config.const_defined?(:STORAGE_EXCEPTION_ACTION) &&
+            Config::STORAGE_EXCEPTION_ACTION == :shutdown
+          @log.error("Romad will stop")
+          @stop_event_loop = true
+        end
+      rescue Exception =>e
+        @log.warn("#{__FILE__}:#{__LINE__}:#{@addr}:#{@port} #{e} #{$@}")
+        close_connection
+      end
+
+      def pop(size)
+        return '' if size == 0
+        if @rbuf.size >= size
+          r = @rbuf[0..size-1]
+          @rbuf = @rbuf[size..-1]
+          r
+        else
+          nil
+        end
+      end
+
+      def read_bytes(size, mult = 1)
+        t=Time.now.to_i
+        while(@connected) do
+          d = pop(size)
+          if d
+            return d
+          else
+            remain = size - @rbuf.size
+            Fiber.yield(remain)
+            if Time.now.to_i - t > @timeout * mult
+              @log.warn("#{__FILE__}:#{__LINE__}:#{@addr}:#{@port} read_bytes time out");
+              close_connection
+              return nil
+            end
+          end
+        end
+        nil
+      end
+
+      def gets
+        while(@connected) do
+          if idx=@rbuf.index("\n")
+            return pop(idx+1)
+          else
+            Fiber.yield(@rbuf.size)
+          end
+        end
+        nil
+      end
+
+      def detach_socket
+        @connected = false
+        Socket::for_fd(detach)
+      end
+
+      def conn_get_stat
+        ret = {}
+        ret["connection.count"] = EM.connection_count
+        ret["connection.continuous_limit"] = Handler.get_ccl
+        ret["connection.accepted_connection_expire_time"] = Handler.connection_expire_time
+        ret["connection.handler_instance_count"] = Handler.connections.length
+        ret["connection.pool_maxlength"] = Messaging::ConPool.instance.maxlength
+        ret["connection.pool_expire_time"] = Messaging::ConPool.instance.expire_time
+        ret["connection.EMpool_maxlength"] = Event::EMConPool::instance.maxlength
+        ret["connection.EMpool_expire_time"] = Event::EMConPool.instance.expire_time
+        ret
+      end
+
+    end # class Handler < EventMachine::Connection
+
+  end # module Event
+end # module Roma