roma 0.8.2 → 0.8.10

data/ruby/server/lib/roma/plugin/plugin_debug.rb

@@ -0,0 +1,31 @@
+
+module Roma
+  module CommandPlugin
+
+    module PluginOperation
+      include ::Roma::CommandPlugin
+
+      # DANGER!!
+      def ev_eval(s)
+        cmd = s[1..-1].join(' ')
+        @log.debug("eval(#{cmd})")
+        send_data("#{eval(cmd)}\r\n")
+      rescue Exception =>e
+        send_data("#{e}\r\n")
+      end
+
+      def get_key_info(key)
+        d = Digest::SHA1.hexdigest(key).hex % @rttable.hbits
+        vn = @rttable.get_vnode_id(d)
+        nodes = @rttable.search_nodes_for_write(vn)
+        s = sprintf("d = %s 0x%x\r\n",d,d)
+        send_data(s)
+        s = sprintf("vn = %s 0x%x\r\n",vn,vn)
+        send_data(s)
+        send_data("nodes = #{nodes.inspect}\r\n")
+        "END"
+      end
+
+    end
+  end
+end

data/ruby/server/lib/roma/plugin/plugin_map.rb

@@ -0,0 +1,177 @@
+require 'roma/messaging/con_pool'
+require 'roma/command/command_definition'
+
+module Roma
+  module CommandPlugin
+
+    module PluginMap
+      include Roma::CommandPlugin
+      include Roma::Command::Definition
+      
+      # map_set <key> <mapkey> <flags> <expt> <bytes> [forward]\r\n
+      # <data block>\r\n
+      #
+      # (STORED|NOT_STORED|SERVER_ERROR <error message>)\r\n
+      def_write_command_with_key_value :map_set, 5 do |ctx|
+        v = {}
+        v = Marshal.load(ctx.stored.value) if ctx.stored
+          
+        v[ctx.argv[2]] = ctx.params.value
+        expt = chg_time_expt(ctx.argv[4].to_i)
+        
+        # [flags, expire time, value, kind of counter(:write/:delete), result message]
+        [0, expt, Marshal.dump(v), :write, 'STORED']
+      end
+
+      # map_get <key> <mapkey> [forward]\r\n
+      #
+      # (
+      # [VALUE <key> 0 <value length>\r\n
+      # <value>\r\n]
+      # END\r\n
+      # |SERVER_ERROR <error message>\r\n)
+      def_read_command_with_key :map_get, :multi_line do |ctx|
+        if ctx.stored
+          v = Marshal.load(ctx.stored.value)[ctx.argv[2]]
+          send_data("VALUE #{ctx.params.key} 0 #{v.length}\r\n#{v}\r\n") if v
+        end
+        send_data("END\r\n")
+      end
+
+      # map_delete <key> <mapkey> [forward]\r\n
+      #
+      # (DELETED|NOT_DELETED|NOT_FOUND|SERVER_ERROR <error message>)\r\n
+      def_write_command_with_key :map_delete do |ctx|
+        next send_data("NOT_FOUND\r\n") unless ctx.stored
+
+        v = Marshal.load(ctx.stored.value)
+        next send_data("NOT_DELETED\r\n") unless v.key?(ctx.argv[2])
+        
+        v.delete(ctx.argv[2])
+        
+        [0, ctx.stored.expt, Marshal.dump(v), :delete, 'DELETED']
+      end
+
+      # map_clear <key> [forward]\r\n
+      #
+      # (CLEARED|NOT_CLEARED|NOT_FOUND|SERVER_ERROR <error message>)\r\n
+      def_write_command_with_key :map_clear do |ctx|
+        next send_data("NOT_FOUND\r\n") unless ctx.stored
+
+        [0, ctx.stored.expt, Marshal.dump({}), :delete, 'CLEARED']
+      end
+
+
+      # map_size <key> [forward]\r\n
+      #
+      # (<length>|NOT_FOUND|SERVER_ERROR <error message>)\r\n
+      def_read_command_with_key :map_size do |ctx|
+        if ctx.stored
+          ret = Marshal.load(ctx.stored.value).size
+          send_data("#{ret}\r\n")
+        else
+          send_data("NOT_FOUND\r\n")
+        end
+      end
+
+      # map_key? <key> <mapkey> [forward]\r\n
+      #
+      # (true|false|NOT_FOUND|SERVER_ERROR <error message>)\r\n
+      def_read_command_with_key :map_key? do |ctx|
+        if ctx.stored
+          ret = Marshal.load(ctx.stored.value).key? ctx.argv[2]
+          send_data("#{ret}\r\n")
+        else
+          send_data("NOT_FOUND\r\n")
+        end
+      end
+      
+      # map_value? <key> <bytes> [forward]\r\n
+      # <data block>\r\n
+      #
+      # (true|false|NOT_FOUND|SERVER_ERROR <error message>)\r\n
+      def_read_command_with_key_value :map_value?, 2 do |ctx|
+        if ctx.stored
+          ret = Marshal.load(ctx.stored.value).value? ctx.params.value
+          send_data("#{ret}\r\n")
+        else
+          send_data("NOT_FOUND\r\n")
+        end
+      end
+      
+      # map_empty? <key> [forward]\r\n
+      #
+      # (true|false|NOT_FOUND|SERVER_ERROR <error message>)\r\n
+      def_read_command_with_key :map_empty? do |ctx|
+        if ctx.stored
+          v = Marshal.load(ctx.stored.value)
+          send_data("#{v.empty?}\r\n")
+        else
+          send_data("NOT_FOUND\r\n")
+        end
+      end
+
+      # map_keys <key> [forward]\r\n
+      #
+      # (
+      # [VALUE <key> 0 <length of length string>\r\n
+      # <length string>\r\n
+      # (VALUE <key> 0 <value length>\r\n
+      # <value>\r\n)*
+      # ]
+      # END\r\n
+      # |SERVER_ERROR <error message>\r\n)
+      def_read_command_with_key :map_keys, :multi_line do |ctx|
+        if ctx.stored
+          v = Marshal.load(ctx.stored.value).keys
+          len = v.length
+          send_data("VALUE #{ctx.params.key} 0 #{len.to_s.length}\r\n#{len.to_s}\r\n")
+          v.each{|val|
+            send_data("VALUE #{ctx.params.key} 0 #{val.length}\r\n#{val}\r\n")
+          }
+        end
+        send_data("END\r\n")
+      end
+
+      # map_values <key> [forward]\r\n
+      #
+      # (
+      # [VALUE <key> 0 <length of length string>\r\n
+      # <length string>\r\n
+      # (VALUE <key> 0 <value length>\r\n
+      # <value>\r\n)*
+      # ]
+      # END\r\n
+      # |SERVER_ERROR <error message>\r\n)
+      def_read_command_with_key :map_values, :multi_line do |ctx|
+        if ctx.stored
+          v = Marshal.load(ctx.stored.value).values
+          len = v.length
+          send_data("VALUE #{ctx.params.key} 0 #{len.to_s.length}\r\n#{len.to_s}\r\n")
+          v.each{|val|
+            send_data("VALUE #{ctx.params.key} 0 #{val.length}\r\n#{val}\r\n")
+          }
+        end
+        send_data("END\r\n")
+      end
+
+      # map_to_s <key> [forward]\r\n
+      #
+      # (
+      # [VALUE <key> 0 <value length>\r\n
+      # <value>\r\n]
+      # END\r\n
+      # |SERVER_ERROR <error message>\r\n)
+      def_read_command_with_key :map_to_s, :multi_line do |ctx|
+        if ctx.stored
+          v = Marshal.load(ctx.stored.value).inspect
+          send_data("VALUE #{ctx.params.key} 0 #{v.length}\r\n#{v}\r\n")
+        end
+        send_data("END\r\n")
+      end
+
+    end # module PluginMap
+  end # module CommandPlugin
+end # module Roma
+ 
+

data/ruby/server/lib/roma/plugin/plugin_mapcount.rb

@@ -0,0 +1,185 @@
+require 'json'
+require 'roma/command/command_definition'
+
+module Roma
+  module CommandPlugin
+
+    module PluginMapCount
+      include ::Roma::CommandPlugin
+      include ::Roma::Command::Definition
+
+      # mapcount_countup <key> <expt> <sub_keys_length>\r\n
+      # <sub_keys> \r\n
+      #
+      # (
+      # VALUE <key> 0 <length of json string>\r\n
+      # <json string>\r\n
+      # END\r\n
+      # |CLIENT_ERROR invalid sub_keys format: <sub_keys>\r\n
+      # |SERVER_ERROR <error message>\r\n)
+      def_write_command_with_key_value :mapcount_countup, 3, :multi_line do |ctx|
+        countup(ctx, :json)
+      end
+
+      # mapcount_countup_ms <key> <expt> <sub_keys_length>\r\n
+      # <sub_keys> \r\n
+      #
+      # (
+      # VALUE <key> 0 <length of Marshal string>\r\n
+      # <Marshal string>\r\n
+      # END\r\n
+      # |CLIENT_ERROR invalid sub_keys format: <sub_keys>\r\n
+      # |SERVER_ERROR <error message>\r\n)
+      def_write_command_with_key_value :mapcount_countup_ms, 3, :multi_line do |ctx|
+        countup(ctx, :marshal)
+      end
+
+      # mapcount_update <key> <expt> <sub_keys_length>\r\n
+      # <sub_keys>\r\n
+      #
+      # (
+      # [VALUE <key> 0 <length of json string>\r\n
+      # <json string>\r\n]
+      # END\r\n
+      # |SERVER_ERROR <error message>\r\n)
+      def_write_command_with_key_value :mapcount_update, 3, :multi_line do |ctx|
+        update(ctx, :json)
+      end
+
+      # mapcount_update_ms <key> <expt> <sub_keys_length>\r\n
+      # <sub_keys>\r\n
+      #
+      # (
+      # [VALUE <key> 0 <length of Marshal string>\r\n
+      # <Marshal string>\r\n]
+      # END\r\n
+      # |SERVER_ERROR <error message>\r\n)
+      def_write_command_with_key_value :mapcount_update_ms, 3, :multi_line do |ctx|
+        update(ctx, :marshal)
+      end
+
+      # mapcount_get <key> 0 <sub_keys_str_len>\r\n
+      # <sub_keys>\r\n
+      #
+      # (
+      # [VALUE <key> 0 <length of json string>\r\n
+      # <json string>\r\n]
+      # END\r\n
+      # |SERVER_ERROR <error message>\r\n)
+      def_read_command_with_key_value :mapcount_get, 3, :multi_line do |ctx|
+        get(ctx, :json)
+      end
+
+      # mapcount_get_ms <key> 0 <sub_keys_str_len>\r\n
+      # <sub_keys>\r\n
+      #
+      # (
+      # [VALUE <key> 0 <length of Marshal string>\r\n
+      # <Marshal string>\r\n]
+      # END\r\n
+      # |SERVER_ERROR <error message>\r\n)
+      def_read_command_with_key_value :mapcount_get_ms, 3, :multi_line do |ctx|
+        get(ctx, :marshal)
+      end
+
+      private
+      def countup(ctx, stype)
+       v = {}
+        v = data_load(ctx.stored.value) if ctx.stored
+
+        args = ctx.params.value.split(/\s*,\s*/)
+        args.each do |arg|
+          if arg =~ /^([A-Za-z0-9]+)(:(\-?[\d]+))?$/
+            key = $1
+            count = 1
+            count = $3.to_i if $3
+            v[key] ||= 0
+            v[key] += count
+          else
+            raise ClientErrorException, "invalid sub_keys format: #{ctx.params.value}"
+          end
+        end
+
+        v["last_updated_date"] = Time.now.gmtime.strftime(DATE_FORMAT)
+        expt = chg_time_expt(ctx.argv[2].to_i)
+
+        ret_str = return_str(v, stype)
+        ret_msg = "VALUE #{ctx.params.key} 0 #{ret_str.length}\r\n#{ret_str}\r\nEND"
+        [0, expt, Marshal.dump(v), :write, ret_msg]
+      end
+
+      def update(ctx, stype)
+        if !ctx.stored
+          send_data("END\r\n")
+          return
+        end
+
+        v = {}
+        v = data_load(ctx.stored.value)
+        v["last_updated_date"] = Time.now.gmtime.strftime(DATE_FORMAT)
+
+        if v.is_a?(Hash)
+          args = ctx.params.value.split(/\s*,\s*/)
+          if args.count == 0
+            ret = return_str(v, stype)
+          else
+            ret = {}
+            ret["last_updated_date"] = v["last_updated_date"]
+            args.each do |arg|
+              ret[arg] = v[arg] if v[arg] != nil
+            end
+            ret = return_str(ret, stype)
+          end
+        end
+
+        expt = chg_time_expt(ctx.argv[2].to_i)
+
+        ret_msg = "VALUE #{ctx.params.key} 0 #{ret.length}\r\n#{ret}\r\nEND"
+        [0, expt, Marshal.dump(v), :write, ret_msg]
+      end
+
+      def get(ctx, stype)
+        ret = nil
+        if ctx.stored
+          ret_val = data_load(ctx.stored.value)
+
+          if ret_val.is_a?(Hash)
+            args = ctx.params.value.split(/\s*,\s*/)
+            if args.count == 0
+              ret = return_str(ret_val, stype)
+            else
+              ret = {}
+              ret["last_updated_date"] = ret_val["last_updated_date"]
+              args.each do |arg|
+                ret[arg] = ret_val[arg] if ret_val[arg] != nil
+              end
+              ret = return_str(ret, stype)
+            end
+          end
+        end
+
+        send_data("VALUE #{ctx.params.key} 0 #{ret.length}\r\n#{ret}\r\n") if ret
+        send_data("END\r\n")
+      end
+
+      def return_str(data, stype)
+        return Marshal.dump(data) if stype == :marshal
+        data.to_json
+      end
+
+      def data_load(data)
+        begin
+          Marshal.load(data)
+        rescue => e
+          msg = "SERVER_ERROR #{e} #{$@}".tr("\r\n"," ")
+          send_data("#{msg}\r\n")
+          @log.error("#{e} #{$@}")
+        end
+      end
+
+      DATE_FORMAT = "%Y-%m-%dT%H:%M:%S +00"
+
+      PLUGIN_MAPCOUNT_VERSION = "1.0.1"
+    end # PluginMapCount
+  end # CommandPlugin
+end # Roma

data/{lib/roma/command/st_command_receiver.rb → ruby/server/lib/roma/plugin/plugin_storage.rb}

@@ -1,12 +1,12 @@
 require 'zlib'
 require 'digest/sha1'
-require "roma/config"
 require 'roma/async_process'
 
 module Roma
-  module Command
+  module CommandPlugin
 
-    module StorageCommandReceiver
+    module PluginStorage
+      include ::Roma::CommandPlugin
 
       # "set" means "store this data".
       # <command name> <key> <flags> <exptime> <bytes> [noreply]\r\n
@@ -14,91 +14,58 @@ module Roma
       def ev_set(s); set(:set,s); end
       def ev_fset(s); fset(:set,s); end
 
-      # rset <key> <hash value> <timelimit> <length>
-      # "set" means "store this data".
-      # <command name> <key> <digest> <exptime> <bytes> [noreply]\r\n
-      # <data block>\r\n
-      def ev_rset(s)
+      # get <key>*\r\n
+      def ev_get(s)
+        return ev_gets(s) if s.length > 2
+
         key,hname = s[1].split("\e")
         hname ||= @defhash
-        d = s[2].to_i
-        d = Digest::SHA1.hexdigest(key).hex % @rttable.hbits if d == 0
-        data = read_bytes(s[5].to_i)
-        read_bytes(2)
+        d = Digest::SHA1.hexdigest(key).hex % @rttable.hbits
         vn = @rttable.get_vnode_id(d)
-        unless @storages.key?(hname)
-          send_data("SERVER_ERROR #{hname} dose not exists.\r\n")
+        nodes = @rttable.search_nodes(vn)
+
+        unless nodes.include?(@nid)
+          @log.warn("forward get #{s[1]}")
+          res = forward_get(nodes[0], s[1], d)
+          if res
+            send_data(res)
+          else
+            send_data("SERVER_ERROR Message forward failed.\r\n")
+          end
           return
         end
-        if @storages[hname].rset(vn, key, d, s[3].to_i, s[4].to_i, data)
-          send_data("STORED\r\n")
-        else
-          @log.error("rset NOT_STORED:#{@storages[hname].error_message} #{vn} #{s[1]} #{d} #{s[3]} #{s[4]}")
-          send_data("NOT_STORED\r\n")
-        end
-        @stats.redundant_count += 1
-      end
 
-      # <command name> <key> <digest> <exptime> <bytes> [noreply]\r\n
-      # <compressed data block>\r\n
-      def ev_rzset(s)
-        key,hname = s[1].split("\e")
-        hname ||= @defhash
-        d = s[2].to_i
-        d = Digest::SHA1.hexdigest(key).hex % @rttable.hbits if d == 0
-        zdata = read_bytes(s[5].to_i)
-        read_bytes(2)
-        vn = @rttable.get_vnode_id(d)
         unless @storages.key?(hname)
           send_data("SERVER_ERROR #{hname} dose not exists.\r\n")
           return
         end
-
-        data = Zlib::Inflate.inflate(zdata)
-        if @storages[hname].rset(vn, key, d, s[3].to_i, s[4].to_i, data)
-          send_data("STORED\r\n")
-        else
-          @log.error("rzset NOT_STORED:#{@storages[hname].error_message} #{vn} #{s[1]} #{d} #{s[3]} #{s[4]}")
-          send_data("NOT_STORED\r\n")
-        end
-        @stats.redundant_count += 1
-      rescue Zlib::DataError => e
-        @log.error("rzset NOT_STORED:#{e} #{vn} #{s[1]} #{d} #{s[3]} #{s[4]}")
-        send_data("NOT_STORED\r\n")
+        data = @storages[hname].get(vn, key, 0)
+        @stats.read_count += 1
+        send_data("VALUE #{s[1]} 0 #{data.length}\r\n#{data}\r\n") if data
+        send_data("END\r\n")
       end
 
-      # get <key>*\r\n
-      def ev_get(s)
-        return ev_gets(s) if s.length > 2
-
+      # fget <key>
+      def ev_fget(s)
         key,hname = s[1].split("\e")
         hname ||= @defhash
         d = Digest::SHA1.hexdigest(key).hex % @rttable.hbits
         vn = @rttable.get_vnode_id(d)
+        nodes = @rttable.search_nodes(vn)
+
+        unless nodes.include?(@nid)
+          @log.error("fget failed key=#{s[1]} vn=#{vn}")
+          return send_data("SERVER_ERROR Routing table is inconsistent.\r\n")
+        end
+
         unless @storages.key?(hname)
           send_data("SERVER_ERROR #{hname} dose not exists.\r\n")
           return
         end
         data = @storages[hname].get(vn, key, 0)
         @stats.read_count += 1
-        if data
-          return send_data("VALUE #{s[1]} 0 #{data.length}\r\n#{data}\r\nEND\r\n")
-        end
-
-        nodes = @rttable.search_nodes(vn)
-        if nodes.include?(@nid)
-          return send_data("END\r\n")
-        end
-
-        nodes.delete(@nid)
-        if nodes.length != 0
-          @log.warn("forward get #{s[1]}")
-          res = forward_get(nodes[0], s[1], d)
-          if res
-            return send_data(res)
-          end
-        end
-        send_data("SERVER_ERROR Message forward failed.\r\n")
+        send_data("VALUE #{s[1]} 0 #{data.length}\r\n#{data}\r\n") if data
+        send_data("END\r\n")
       end
 
       # gets <key>*\r\n
@@ -150,15 +117,15 @@ module Roma
         vn = @rttable.get_vnode_id(d)
         nodes = @rttable.search_nodes_for_write(vn)
         if nodes[0] != @nid
-          cmd = "fdelete #{s[1]}"
+          cmd = "fdelete #{key}\e#{hname}"
           s[2..-1].each{|c| cmd << " #{c}"}
           cmd << "\r\n"
           @log.warn("forward delete #{s[1]}")
           res = send_cmd(nodes[0], cmd)
-          if res
-            return send_data("#{res}\r\n")
+          if res == nil || res.start_with?("ERROR")
+            return send_data("SERVER_ERROR Message forward failed.\r\n")
           end
-          return send_data("SERVER_ERROR Message forward failed.\r\n")
+          return send_data("#{res}\r\n")
         end
         unless @storages.key?(hname)
           send_data("SERVER_ERROR #{hname} dose not exists.\r\n")
@@ -166,11 +133,20 @@ module Roma
         end
         res = @storages[hname].delete(vn, key, d)
         @stats.delete_count += 1
+
         return send_data("NOT_DELETED\r\n") unless res
         return send_data("NOT_FOUND\r\n") if res == :deletemark
 
+        if @stats.wb_command_map.key?(:delete)
+          Roma::WriteBehindProcess::push(hname, @stats.wb_command_map[:delete], key, res[4])
+        end
+
         nodes[1..-1].each{ |nid|
-          send_cmd(nid,"rdelete #{s[1]} #{res[2]}\r\n")
+          res2 = send_cmd(nid,"rdelete #{key}\e#{hname} #{res[2]}\r\n")
+          unless res2
+            Roma::AsyncProcess::queue.push(Roma::AsyncMessage.new('rdelete',[nid,hname,s[1],res[2]]))
+            @log.warn("rdelete failed:#{s[1]}\e#{hname} #{d} #{res[2]} -> #{nid}")
+          end
         }
         return send_data("NOT_FOUND\r\n") unless res[4]
         send_data("DELETED\r\n")
@@ -193,12 +169,21 @@ module Roma
         end
         res = @storages[hname].delete(vn, key, d)
         @stats.delete_count += 1
+
         return send_data("NOT_DELETED\r\n") unless res
         return send_data("NOT_FOUND\r\n") if res == :deletemark
 
+        if @stats.wb_command_map.key?(:delete)
+          Roma::WriteBehindProcess::push(hname, @stats.wb_command_map[:delete], key, res[4])
+        end
+
         nodes.delete(@nid)
         nodes.each{ |nid|
-          send_cmd(nid,"rdelete #{s[1]} #{res[2]}\r\n")
+          res2 = send_cmd(nid,"rdelete #{key}\e#{hname} #{res[2]}\r\n")
+          unless res2
+            Roma::AsyncProcess::queue.push(Roma::AsyncMessage.new('rdelete',[nid,hname,s[1],res[2]]))
+            @log.warn("rdelete failed:#{s[1]}\e#{hname} #{d} #{res[2]} -> #{nid}")
+          end
         }
         return send_data("NOT_FOUND\r\n") unless res[4]
         send_data("DELETED\r\n")
@@ -221,24 +206,6 @@ module Roma
         end
       end
 
-      # out <key> <vn>
-      def ev_out(s)
-        key,hname = s[1].split("\e")
-        hname ||= @defhash
-        if s.length >= 3
-          vn = s[2].to_i
-        else
-          d = Digest::SHA1.hexdigest(key).hex % @rttable.hbits
-          vn = @rttable.get_vnode_id(d)
-        end
-        res = @storages[hname].out(vn, key, 0)
-        @stats.out_message_count += 1
-        unless res
-          return send_data("NOT_DELETED\r\n")
-        end
-        send_data("DELETED\r\n")
-      end
-
       # "add" means that "add a new data to a store"
       # <command name> <key> <flags> <exptime> <bytes> [noreply]\r\n
       # <data block>\r\n
@@ -277,11 +244,11 @@ module Roma
         nodes = @rttable.search_nodes_for_write(vn)
         if nodes[0] != @nid
           @log.warn("forward cas key=#{key} vn=#{vn} to #{nodes[0]}")
-          res = send_cmd(nodes[0],"fcas #{s[1]} #{d} #{s[3]} #{v.length} #{s[5]}\r\n#{v}\r\n")
-          if res
-            return send_data("#{res}\r\n")
+          res = send_cmd(nodes[0],"fcas #{key}\e#{hname} #{d} #{s[3]} #{v.length} #{s[5]}\r\n#{v}\r\n")
+          if res == nil || res.start_with?("ERROR")
+            return send_data("SERVER_ERROR Message forward failed.\r\n")
           end
-          return send_data("SERVER_ERROR Message forward failed.\r\n")
+          return send_data("#{res}\r\n")
         end
 
         store_cas(hname, vn, key, d, s[5].to_i, s[3].to_i, v, nodes[1..-1])
@@ -313,6 +280,72 @@ module Roma
       def ev_decr(s); incr_decr(:decr,s); end
       def ev_fdecr(s); fincr_fdecr(:decr,s); end
 
+      # set_expt <key> <expt>
+      def ev_set_expt(s)
+        key,hname = s[1].split("\e")
+        hname ||= @defhash
+        d = Digest::SHA1.hexdigest(key).hex % @rttable.hbits
+        vn = @rttable.get_vnode_id(d)
+        nodes = @rttable.search_nodes_for_write(vn)
+        if nodes[0] != @nid
+          @log.warn("forward set_expt key=#{key} vn=#{vn} to #{nodes[0]}")
+          res = send_cmd(nodes[0],"fset_expt #{s[1]} #{s[2]}\r\n")
+          if res
+            return send_data("#{res}\r\n")
+          end
+          return send_data("SERVER_ERROR Message forward failed.\r\n")
+        end
+        
+        unless @storages.key?(hname)
+          send_data("SERVER_ERROR #{hname} dose not exists.\r\n")
+          return
+        end
+
+        expt = chg_time_expt(s[2].to_i)
+        ret = @storages[hname].set_expt(vn, key, d, expt)
+
+        if ret
+          if @stats.wb_command_map.key?(:set_export)
+            Roma::WriteBehindProcess::push(hname, @stats.wb_command_map[:set_expt], k, expt.to_s)
+          end
+          redundant(nodes[1..-1], hname, key, d, ret[2], ret[3], ret[4])
+          send_data("STORED\r\n")
+        else
+          return send_data("NOT_STORED\r\n")
+        end
+      end
+
+      # fset_expt <key> <expt>
+      def ev_fset_expt(s)
+        key,hname = s[1].split("\e")
+        hname ||= @defhash
+        d = Digest::SHA1.hexdigest(key).hex % @rttable.hbits
+        vn = @rttable.get_vnode_id(d)
+        nodes = @rttable.search_nodes_for_write(vn)
+        if nodes.include?(@nid) == false
+          @log.error("fset_expt failed key = #{s[1]} vn = #{vn}")
+          return send_data("SERVER_ERROR Routing table is inconsistent.\r\n")
+        end
+        
+        unless @storages.key?(hname)
+          send_data("SERVER_ERROR #{hname} dose not exists.\r\n")
+          return
+        end
+
+        expt = chg_time_expt(s[2].to_i)
+        ret = @storages[hname].set_expt(vn, key, d, expt)
+
+        if ret
+          if @stats.wb_command_map.key?(:set_export)
+            Roma::WriteBehindProcess::push(hname, @stats.wb_command_map[:set_expt], k, expt.to_s)
+          end
+          redundant(nodes[1..-1], hname, key, d, ret[2], ret[3], ret[4])
+          send_data("STORED\r\n")
+        else
+          return send_data("NOT_STORED\r\n")
+        end
+      end
+
       # set_size_of_zredundant <n>
       def ev_set_size_of_zredundant(s)
         if s.length != 2 || s[1].to_i == 0
@@ -337,18 +370,30 @@ module Roma
 
       def forward_get(nid, k, d)
         con = get_connection(nid)
-        con.send("get #{k}\r\n")
+        con.send("fget #{k}\r\n")
         res = con.gets
-        return res if res == "END\r\n"
-        s = res.split(/ /)
-        res << con.read_bytes(s[3].to_i + 2)
-        res << con.gets
+        if res == nil
+          @rttable.proc_failed(nid)
+          @log.error("forward get failed:nid=#{nid} key=#{k}")
+          return nil
+        elsif res == "END\r\n"
+          # value dose not found
+        elsif res.start_with?("ERROR")
+          @rttable.proc_succeed(nid)
+          con.close_connection
+          return nil
+        else
+          s = res.split(/ /)
+          res << con.read_bytes(s[3].to_i + 2)
+          res << con.gets
+        end
         return_connection(nid, con)
         @rttable.proc_succeed(nid)
         res
       rescue => e
-        @rttable.proc_failed(nid)
-        @log.error("forward get failed:nid=#{nid} key=#{key}")
+        @rttable.proc_failed(nid) if e.message != "no connection"
+        @log.error("#{e.inspect}/#{$@}")
+        @log.error("forward get failed:nid=#{nid} key=#{k}")
         nil
       end
 
@@ -371,18 +416,18 @@ module Roma
       end
 
       def store(fnc, hname, vn, k, d, expt, v, nodes)
-        if expt == 0
-          expt = 0x7fffffff
-        elsif expt < 2592000
-          expt += Time.now.to_i
-        end
+        expt = chg_time_expt(expt)
         unless @storages.key?(hname)
           send_data("SERVER_ERROR #{hname} dose not exists.\r\n")
           return
         end
         ret = @storages[hname].send(fnc, vn, k, d, expt ,v)
         @stats.write_count += 1
+
         if ret
+          if @stats.wb_command_map.key?(fnc)
+            Roma::WriteBehindProcess::push(hname, @stats.wb_command_map[fnc], k, ret[4])
+          end
           redundant(nodes, hname, k, d, ret[2], expt, ret[4])
           send_data("STORED\r\n")
         else
@@ -392,11 +437,7 @@ module Roma
       end
 
       def store_cas(hname, vn, k, d, clk, expt, v, nodes)
-        if expt == 0
-          expt = 0x7fffffff
-        elsif expt < 2592000
-          expt += Time.now.to_i
-        end
+        expt = chg_time_expt(expt)
         unless @storages.key?(hname)
           send_data("SERVER_ERROR #{hname} dose not exists.\r\n")
           return
@@ -404,6 +445,7 @@ module Roma
 
         ret = @storages[hname].cas(vn, k, d, clk, expt ,v)
         @stats.write_count += 1
+
         case ret
         when nil
           @log.error("cas NOT_STORED:#{hname} #{vn} #{k} #{d} #{expt} #{clk}")
@@ -413,56 +455,34 @@ module Roma
         when :exists
           send_data("EXISTS\r\n")
         else
+          if @stats.wb_command_map.key?(:cas)
+            Roma::WriteBehindProcess::push(hname, @stats.wb_command_map[:cas], k, ret[4])
+          end
           redundant(nodes, hname, k, d, ret[2], expt, ret[4])
           send_data("STORED\r\n")          
         end
       end
 
       def redundant(nodes, hname, k, d, clk, expt, v)
-        if @rttable.min_version == nil || @rttable.min_version < 0x000306 # ver.0.3.6
-          return redundant_older_than_000306(nodes, hname, k, d, clk, expt, v)
-        end
-
         if @stats.size_of_zredundant > 0 && @stats.size_of_zredundant < v.length 
           return zredundant(nodes, hname, k, d, clk, expt, v)
         end
 
         nodes.each{ |nid|
           res = send_cmd(nid,"rset #{k}\e#{hname} #{d} #{clk} #{expt} #{v.length}\r\n#{v}\r\n")
-          unless res
+          if res == nil || res.start_with?("ERROR")
             Roma::AsyncProcess::queue.push(Roma::AsyncMessage.new('redundant',[nid,hname,k,d,clk,expt,v]))
             @log.warn("redundant failed:#{k}\e#{hname} #{d} #{clk} #{expt} #{v.length} -> #{nid}")
           end
         }
       end
 
-      def redundant_older_than_000306(nodes, hname, k, d, clk, expt, v)
-        nodes.each{ |nid|
-          if @rttable.version_of_nodes[nid] >= 0x000306 &&
-              @stats.size_of_zredundant > 0 && @stats.size_of_zredundant < v.length
-
-            zv = Zlib::Deflate.deflate(v) unless zv
-            res = send_cmd(nid,"rzset #{k}\e#{hname} #{d} #{clk} #{expt} #{zv.length}\r\n#{zv}\r\n")
-            unless res
-              Roma::AsyncProcess::queue.push(Roma::AsyncMessage.new('zredundant',[nid,hname,k,d,clk,expt,zv]))
-              @log.warn("redundant_older_than_000306 failed:#{k}\e#{hname} #{d} #{clk} #{expt} #{zv.length} -> #{nid}")
-            end
-          else
-            res = send_cmd(nid,"rset #{k}\e#{hname} #{d} #{clk} #{expt} #{v.length}\r\n#{v}\r\n")
-            unless res
-              Roma::AsyncProcess::queue.push(Roma::AsyncMessage.new('redundant',[nid,hname,k,d,clk,expt,v]))
-              @log.warn("redundant_older_than_000306 failed:#{k}\e#{hname} #{d} #{clk} #{expt} #{v.length} -> #{nid}")
-            end
-          end
-        }
-      end
-
       def zredundant(nodes, hname, k, d, clk, expt, v)
         zv = Zlib::Deflate.deflate(v)
 
         nodes.each{ |nid|
           res = send_cmd(nid,"rzset #{k}\e#{hname} #{d} #{clk} #{expt} #{zv.length}\r\n#{zv}\r\n")
-          unless res
+          if res == nil || res.start_with?("ERROR")
             Roma::AsyncProcess::queue.push(Roma::AsyncMessage.new('zredundant',[nid,hname,k,d,clk,expt,zv]))
             @log.warn("zredundant failed:#{k}\e#{hname} #{d} #{clk} #{expt} #{zv.length} -> #{nid}")
           end
@@ -480,10 +500,10 @@ module Roma
         if nodes[0] != @nid
           @log.warn("forward #{fnc} key=#{key} vn=#{vn} to #{nodes[0]}")
           res = send_cmd(nodes[0],"f#{fnc} #{s[1]} #{d} #{s[3]} #{v.length}\r\n#{v}\r\n")
-          if res
-            return send_data("#{res}\r\n")
+          if res == nil || res.start_with?("ERROR")
+            return send_data("SERVER_ERROR Message forward failed.\r\n")
           end
-          return send_data("SERVER_ERROR Message forward failed.\r\n")
+          return send_data("#{res}\r\n")
         end
 
         store(fnc, hname, vn, key, d, s[3].to_i, v, nodes[1..-1])
@@ -514,7 +534,11 @@ module Roma
         end
         res = @storages[hname].send(fnc, vn, k, d, v)
         @stats.write_count += 1
+
         if res
+          if @stats.wb_command_map.key?(fnc)
+            Roma::WriteBehindProcess::push(hname, @stats.wb_command_map[fnc], k, res[4])
+          end
           redundant(nodes, hname, k, d, res[2], res[3], res[4])
           send_data("#{res[4]}\r\n")
         else
@@ -532,10 +556,10 @@ module Roma
         if nodes[0] != @nid
           @log.debug("forward #{fnc} key=#{s[1]} vn=#{vn} to #{nodes[0]}")
           res = send_cmd(nodes[0],"f#{fnc} #{s[1]} #{d} #{s[2]}\r\n")
-          if res
-            return send_data("#{res}\r\n")
+          if res == nil || res.start_with?("ERROR")
+            return send_data("SERVER_ERROR Message forward failed.\r\n")
           end
-          return send_data("SERVER_ERROR Message forward failed.\r\n")
+          return send_data("#{res}\r\n")
         end
 
         store_incr_decr(fnc, hname, vn, key, d, v, nodes[1..-1])
@@ -558,7 +582,7 @@ module Roma
         store_incr_decr(fnc, hname, vn, key, d, v, nodes)
       end
 
-    end # module StorageCommandReceiver
+    end # module PluginStorage
 
-  end # module Command
+  end # module CommandPlugin
 end # module Roma