roma 0.8.13 → 0.8.14

data/ruby/server/lib/roma/event/con_pool.rb

@@ -90,7 +90,7 @@ module Roma
 
       def get_connection(ap)
         ret = @pool[ap].shift if @pool.key?(ap) && @pool[ap].length > 0
-        if ret && ret.last_access < Time.now - @expire_time
+        if ret && @expire_time != 0 && ret.last_access < Time.now - @expire_time
           ret.close_connection if ret.connected
           ret = nil
           Logging::RLogger.instance.info("EM connection expired at #{ap},remains #{@pool[ap].length}")

data/ruby/server/lib/roma/event/handler.rb

@@ -50,7 +50,15 @@ module Roma
         @@connection_expire_time
       end
 
-      attr_accessor :timeout
+      @@timeout = 10
+      def self.timeout=(n)
+        @@timeout = n
+      end
+
+      def self.timeout
+        @@timeout
+      end
+
       attr_reader :connected
       attr_reader :lastcmd
       attr_reader :last_access
@@ -71,7 +79,6 @@ module Roma
 
         @storages = storages
         @rttable = rttable
-        @timeout = 10
         @log = Roma::Logging::RLogger.instance
         @last_access = Time.now
       end
@@ -96,7 +103,6 @@ module Roma
       end
 
       def unbind
-        @log.debug("Roma::Event::Handler.unbind called")
         @connected=false
         begin
           @fiber.resume
@@ -144,7 +150,7 @@ module Roma
 
       def dispatcher
         @stats = Roma::Stats.instance
-        @log.debug("Roma::Event::Handler.dipatcher called")
+        #@log.debug("Roma::Event::Handler.dipatcher called")
         while(@connected) do
           @enter_time = nil
           next unless s=gets
@@ -224,7 +230,7 @@ module Roma
           else
             remain = size - @rbuf.size
             Fiber.yield(remain)
-            if Time.now.to_i - t > @timeout * mult
+            if Time.now.to_i - t > @@timeout * mult
               @log.warn("#{__FILE__}:#{__LINE__}:#{@addr}:#{@port} read_bytes time out");
               close_connection
               return nil

data/ruby/server/lib/roma/messaging/con_pool.rb

@@ -21,7 +21,7 @@ module Roma
 
       def get_connection(ap)
         ret,last = @pool[ap].shift if @pool.key?(ap) && @pool[ap].length > 0
-        if ret && last < Time.now - @expire_time
+        if ret && @expire_time != 0 && last < Time.now - @expire_time
           ret.close
           ret = nil
           Logging::RLogger.instance.info("connection expired at #{ap},remains #{@pool[ap].length}")

data/ruby/server/lib/roma/romad.rb

@@ -521,6 +521,8 @@ module Roma
           @stats.run_storage_clean_up
         stop_clean_up
       end
+    rescue Exception =>e
+      @log.error("#{e}\n#{$@}")
     end
 
     def timer_event_10sec
@@ -539,7 +541,7 @@ module Roma
         @log.warn("failover disable now!!")
       else
         version_check
-        @rttable.delete_old_trans
+        @rttable.delete_old_trans(@stats.routing_trans_timeout)
         start_sync_routing_process
       end
 

data/ruby/server/lib/roma/stats.rb

@@ -63,6 +63,14 @@ module Roma
     attr_accessor :latency_data
     #attr_accessor :latency_denominator
 
+    # for vnode copy parameter
+    attr_accessor :spushv_klength_warn
+    attr_accessor :spushv_vlength_warn
+    attr_accessor :spushv_read_timeout
+    attr_accessor :reqpushv_timeout_count
+
+    attr_accessor :routing_trans_timeout
+
     def initialize
       @config_path = nil
       @run_recover = false
@@ -90,8 +98,13 @@ module Roma
       @wb_command_map = {}
       @latency_log = false
       @latency_check_cmd =["get", "set", "delete"]
-      @latency_check_time_count = nil
+      @latency_check_time_count = false
       @latency_data = Hash.new { |hash,key| hash[key] = {} } #double hash
+      @spushv_klength_warn = 1024 # 1kB
+      @spushv_vlength_warn = 1024 * 1024 # 1MB
+      @spushv_read_timeout = 100
+      @reqpushv_timeout_count = 300 # 0.1 * 300 sec
+      @routing_trans_timeout = 3600 * 3 # 3hr
     end
 
     def ap_str
@@ -132,6 +145,11 @@ module Roma
       ret['stats.latency_log']  = @latency_log
       ret['stats.latency_check_cmd']  = @latency_check_cmd
       ret['stats.latency_check_time_count']  = @latency_check_time_count
+      ret['stats.spushv_klength_warn'] = @spushv_klength_warn
+      ret['stats.spushv_vlength_warn'] = @spushv_vlength_warn
+      ret['stats.spushv_read_timeout'] = @spushv_read_timeout
+      ret['stats.reqpushv_timeout_count'] = @reqpushv_timeout_count
+      ret['stats.routing_trans_timeout'] = @routing_trans_timeout
       ret
     end
 

data/ruby/server/lib/roma/storage/basic_storage.rb

@@ -1,4 +1,5 @@
 require 'digest/sha1'
+require 'thread'
 
 module Roma
   module Storage
@@ -7,11 +8,11 @@ module Roma
 
     class BasicStorage
 
-      attr :hdb
-      attr :hdiv
-      attr :ext_name
-      
+      attr_reader :hdb
+      attr_reader :hdiv
+      attr_reader :ext_name
       attr_reader :error_message
+      attr_reader :dbs
 
       attr_writer :vn_list
       attr_writer :storage_path
@@ -23,8 +24,17 @@ module Roma
       attr_accessor :each_clean_up_sleep
       attr_accessor :logic_clock_expire
 
+      attr_accessor :do_each_vn_dump
+
       def initialize
+        # database handler
         @hdb = []
+        # database cache handler
+        @hdbc = []
+        # status of a database
+        @dbs = []
+        @log_fd = nil
+
         @hdiv = Hash.new(0)
 
         @ext_name = 'db'
@@ -35,6 +45,10 @@ module Roma
         @each_vn_dump_sleep_count = 100
         @each_clean_up_sleep = 0.01
         @logic_clock_expire = 300
+
+        @each_cache_lock = Mutex::new
+        @each_clean_up_lock = Mutex::new
+        @stat_lock = Mutex::new
       end
 
       def get_stat
@@ -46,6 +60,7 @@ module Roma
         ret['storage.each_vn_dump_sleep_count'] = @each_vn_dump_sleep_count
         ret['storage.each_clean_up_sleep'] = @each_clean_up_sleep
         ret['storage.logic_clock_expire'] = @logic_clock_expire
+        ret['storage.safecopy_stats'] = @dbs.inspect
         ret
       end
 
@@ -71,27 +86,45 @@ module Roma
       end
       protected :create_div_hash
 
-      def opendb
-        create_div_hash
+      def mkdir_p(md_path)
         path = ''
-        @storage_path.split('/').each{|p|
-          if p.length==0
+        md_path.split('/').each do |p|
+          if p.length == 0
             path = '/'
             next
           end
           path << p
           Dir::mkdir(path) unless File.exist?(path)
           path << '/'
-        }
-        @divnum.times{ |i|
+        end
+      end
+      protected :mkdir_p
+
+      def opendb
+        create_div_hash
+        mkdir_p(@storage_path)
+        @divnum.times do |i|
+          # open database file
           @hdb[i] = open_db("#{@storage_path}/#{i}.#{@ext_name}")
-        }
+          # check cache file
+          if File.exist?(cache_file_name(i))
+            @hdbc[i] = open_db(cache_file_name(i))
+            stop_clean_up { @dbs[i] = :safecopy_flushed }
+          else
+            @dbs[i] = :normal
+            @hdbc[i] = nil
+          end
+        end
+        open_log
       end
 
       def closedb
         stop_clean_up
         buf = @hdb; @hdb = []
-        buf.each{ |hdb| close_db(hdb) }
+        buf.each{ |h| close_db(h) if h }
+        buf = @hdbc; @hdbc = []
+        buf.each{ |h| close_db(h) if h }
+        close_log
       end
 
 
@@ -117,15 +150,41 @@ module Roma
       end
       private :pack_header, :unpack_header, :pack_data, :unpack_data
 
+      def db_get(vn, k)
+        n = @hdiv[vn]
+        d = @hdb[n].get(k)
+        return d if @dbs[n] == :normal
+
+        c = @hdbc[n].get(k)
+        return d unless c # in case of out of :normal status
+
+        if @dbs[n] == :cachecleaning && d
+          # in case of existing value is both @hdb and @hdbc
+          vn, lat, clk, expt = unpack_header(d)
+          cvn, clat, cclk, cexpt = unpack_header(c)
+          return d if cmp_clk(clk, cclk) > 0 # if @hdb newer than @hdbc
+        end
+        c
+      end
+
+      def db_put(vn, k, v)
+        n = @hdiv[vn]
+        if @dbs[n] == :safecopy_flushing || @dbs[n] == :safecopy_flushed
+          ret = @hdbc[n].put(k, v)
+        else
+          ret = @hdb[n].put(k, v)
+        end
+        ret
+      end
 
       def get_context(vn, k, d)
-        buf = @hdb[@hdiv[vn]].get(k)
+        buf = db_get(vn, k)
         return nil unless buf
         unpack_header(buf)
       end
 
       def cas(vn, k, d, clk, expt, v)
-        buf = @hdb[@hdiv[vn]].get(k)
+        buf = db_get(vn ,k)
         return :not_found unless buf
         t = Time.now.to_i
         data = unpack_data(buf)
@@ -133,12 +192,12 @@ module Roma
         return :exists if clk != data[2]
         clk = (data[2] + 1) & 0xffffffff
         ret = [vn, t, clk, expt, v]
-        return ret if @hdb[@hdiv[vn]].put(k, pack_data(*ret))
+        return ret if db_put(vn, k, pack_data(*ret))
         nil
       end
 
       def rset(vn, k, d, clk, expt, v)
-        buf = @hdb[@hdiv[vn]].get(k)
+        buf = db_get(vn, k)
         t = Time.now.to_i
         if buf
           data = unpack_data(buf)
@@ -149,13 +208,13 @@ module Roma
         end
 
         ret = [vn, t, clk, expt, v]
-        return ret if @hdb[@hdiv[vn]].put(k, pack_data(*ret))
+        return ret if db_put(vn, k, pack_data(*ret))
         @error_message = "error:put"
         nil
       end
 
       def set(vn, k, d, expt, v)
-        buf = @hdb[@hdiv[vn]].get(k)
+        buf = db_get(vn, k)
         clk = 0
         if buf
           data = unpack_data(buf)
@@ -163,12 +222,12 @@ module Roma
         end
 
         ret = [vn, Time.now.to_i, clk, expt, v]
-        return ret if @hdb[@hdiv[vn]].put(k, pack_data(*ret))
+        return ret if db_put(vn , k, pack_data(*ret))
         nil
       end
 
       def add(vn, k, d, expt, v)
-        buf = @hdb[@hdiv[vn]].get(k)
+        buf = db_get(vn, k)
         clk = 0
         if buf
           vn, t, clk, expt2, v2 = unpack_data(buf)
@@ -178,12 +237,12 @@ module Roma
         
         # not exist
         ret = [vn, Time.now.to_i, clk, expt, v]
-        return ret if @hdb[@hdiv[vn]].put(k, pack_data(*ret))
+        return ret if db_put(vn, k, pack_data(*ret))
         nil
       end
 
       def replace(vn, k, d, expt, v)
-        buf = @hdb[@hdiv[vn]].get(k)
+        buf = db_get(vn, k)
         return nil unless buf
 
         # buf != nil
@@ -192,12 +251,12 @@ module Roma
         clk = (clk + 1) & 0xffffffff
 
         ret = [vn, Time.now.to_i, clk, expt, v]
-        return ret if @hdb[@hdiv[vn]].put(k, pack_data(*ret))
+        return ret if db_put(vn, k, pack_data(*ret))
         nil
       end
 
       def append(vn, k, d, expt, v)
-        buf = @hdb[@hdiv[vn]].get(k)
+        buf = db_get(vn, k)
         return nil unless buf
 
         # buf != nil
@@ -206,12 +265,12 @@ module Roma
         clk = (clk + 1) & 0xffffffff
 
         ret = [vn, Time.now.to_i, clk, expt, v2 + v]
-        return ret if @hdb[@hdiv[vn]].put(k, pack_data(*ret))
+        return ret if db_put(vn, k, pack_data(*ret))
         nil
       end
 
       def prepend(vn, k, d, expt, v)
-        buf = @hdb[@hdiv[vn]].get(k)
+        buf = db_get(vn, k)
         return nil unless buf
 
         # buf != nil
@@ -220,12 +279,12 @@ module Roma
         clk = (clk + 1) & 0xffffffff
 
         ret = [vn, Time.now.to_i, clk, expt, v + v2]
-        return ret if @hdb[@hdiv[vn]].put(k, pack_data(*ret))
+        return ret if db_put(vn, k, pack_data(*ret))
         nil
       end
 
       def get(vn, k, d)
-        buf = @hdb[@hdiv[vn]].get(k)
+        buf = db_get(vn, k)
         return nil unless buf
         vn, t, clk, expt, v = unpack_data(buf)
 
@@ -234,7 +293,7 @@ module Roma
       end
 
       def get_raw(vn, k, d)
-        buf = @hdb[@hdiv[vn]].get(k)
+        buf = db_get(vn, k)
         return nil unless buf
 
         unpack_data(buf)
@@ -249,7 +308,7 @@ module Roma
       end
 
       def rdelete(vn, k, d, clk)
-        buf = @hdb[@hdiv[vn]].get(k)
+        buf = db_get(vn, k)
         t = Time.now.to_i
         if buf
           data = unpack_header(buf)
@@ -264,7 +323,7 @@ module Roma
         # [ 8..11] logical clock
         # [12..15] exptime(unix time) => 0
         ret = [vn, t, clk, 0]
-        if @hdb[@hdiv[vn]].put(k,pack_header(*ret))
+        if db_put(vn, k, pack_header(*ret))
           return ret
         else
           return nil
@@ -272,7 +331,7 @@ module Roma
       end
 
       def delete(vn, k, d)
-        buf = @hdb[@hdiv[vn]].get(k)
+        buf = db_get(vn, k)
         v = ret = nil
         clk = 0
         if buf
@@ -287,7 +346,7 @@ module Roma
         # [ 8..11] logical clock
         # [12..15] exptime(unix time) => 0
         ret = [vn, Time.now.to_i, clk, 0, v]
-        if @hdb[@hdiv[vn]].put(k,pack_header(*ret[0..-2]))
+        if db_put(vn, k, pack_header(*ret[0..-2]))
           return ret
         else
           return nil
@@ -299,7 +358,7 @@ module Roma
       end 
 
       def incr(vn, k, d, v)
-        buf = @hdb[@hdiv[vn]].get(k)
+        buf = db_get(vn, k)
         return nil unless buf
 
         # buf != nil
@@ -312,12 +371,12 @@ module Roma
         v = v & 0xffffffffffffffff
 
         ret = [vn, Time.now.to_i, clk, expt2, v.to_s]
-        return ret if @hdb[@hdiv[vn]].put(k, pack_data(*ret))
+        return ret if db_put(vn, k, pack_data(*ret))
         nil
       end
 
       def decr(vn, k, d, v)
-        buf = @hdb[@hdiv[vn]].get(k)
+        buf = db_get(vn, k)
         return nil unless buf
 
         # buf != nil
@@ -330,19 +389,19 @@ module Roma
         v = v & 0xffffffffffffffff
 
         ret = [vn, Time.now.to_i, clk, expt2, v.to_s]
-        return ret if @hdb[@hdiv[vn]].put(k, pack_data(*ret))
+        return ret if db_put(vn, k, pack_data(*ret))
         nil
       end
 
       # set expire time
       def set_expt(vn, k, d, expt)
-        buf = @hdb[@hdiv[vn]].get(k)
+        buf = db_get(vn, k)
         if buf
           vn, t, clk, expt2, v = unpack_data(buf)
           return nil if Time.now.to_i > expt2
           clk = (clk + 1) & 0xffffffff
           ret = [vn, Time.now.to_i, clk, expt, v]
-          return ret if @hdb[@hdiv[vn]].put(k, pack_data(*ret))
+          return ret if db_put(vn, k, pack_data(*ret))
         end
         nil
       end
@@ -353,19 +412,14 @@ module Roma
         res
       end
 
-      def add_vnode(vn)
-      end
-
-      def del_vnode(vn)
-        buf = get_vnode_hash(vn)
-        buf.each_key{ |k| @hdb[@hdiv[vn]].out(k) }
-      end
-
       def each_clean_up(t, vnhash)
         @do_clean_up = true
+        return unless @each_clean_up_lock.try_lock
         nt = Time.now.to_i
-        @hdb.each{ |hdb|
-          hdb.each{ |k, v|
+        @divnum.times do |i|
+          next if @dbs[i] != :normal
+          hdb = @hdb[i]
+          hdb.each do |k, v|
             return unless @do_clean_up # 1st check
             vn, last, clk, expt = unpack_header(v)
             vn_stat = vnhash[vn]
@@ -380,12 +434,22 @@ module Roma
             end
             return unless @do_clean_up # 2nd ckeck 
             sleep @each_clean_up_sleep
-          }
-        }
+          end
+        end
+      ensure
+        @each_clean_up_lock.unlock if @each_clean_up_lock.locked?  
       end
 
-      def stop_clean_up
-         @do_clean_up = false
+      def stop_clean_up(&block)
+        @do_clean_up = false
+        if block
+          @each_clean_up_lock.lock
+          begin
+            block.call
+          ensure
+            @each_clean_up_lock.unlock
+          end
+        end
       end
 
       def load(dmp)
@@ -414,19 +478,40 @@ module Roma
       end
 
       def load_stream_dump(vn, last, clk, expt, k, v)
-        buf = @hdb[@hdiv[vn]].get(k)
+        buf = db_get(vn, k)
         if buf
           data = unpack_header(buf)
           if last - data[1] < @logic_clock_expire && cmp_clk(clk,data[2]) <= 0
             return nil
           end
         end
-        
+
+        ret = [vn, last, clk, expt, v]
+        if expt == 0
+          # for the deleted mark
+          return ret if db_put(vn, k, pack_header(*ret[0..3]))
+        else
+          return ret if db_put(vn, k, pack_data(*ret))
+        end
+        nil
+      end
+
+      def load_stream_dump_for_cachecleaning(vn, last, clk, expt, k, v)
+        n = @hdiv[vn]
+        buf = @hdb[n].get(k)
+        if buf
+          data = unpack_header(buf)
+          if last - data[1] < @logic_clock_expire && cmp_clk(clk,data[2]) <= 0
+            return nil
+          end
+        end
+
         ret = [vn, last, clk, expt, v]
         if expt == 0
-          return ret if @hdb[@hdiv[vn]].put(k, pack_header(*ret[0..3]))
+          # for the deleted mark
+          return ret if @hdb[n].put(k, pack_header(*ret[0..3]))
         else
-          return ret if @hdb[@hdiv[vn]].put(k, pack_data(*ret))
+          return ret if @hdb[n].put(k, pack_data(*ret))
         end
         nil
       end
@@ -438,43 +523,68 @@ module Roma
         Marshal.dump(buf)
       end
 
-      def dump_file(path,except_vnh = nil)
-        pbuf = ''
-        path.split('/').each{|p|
-          pbuf << p
-          begin
-            Dir::mkdir(pbuf) unless File.exist?(pbuf)
-          rescue
+      def each_vn_dump(target_vn)
+        @do_each_vn_dump = true
+        n = @hdiv[target_vn]
+        if @dbs[n] == :normal
+          # in case of :normal status
+          each_unpacked_db(target_vn, @hdb) do |vn, last, clk, expt, k, val|
+            return unless @do_each_vn_dump
+            yield vn_dump_pack(vn, last, clk, expt, k, val)
           end
-          pbuf << '/'
-        }
-        @divnum.times{|i|
-          f = open("#{path}/#{i}.dump","wb")
-          each_hdb_dump(i,except_vnh){|data| f.write(data) }
-          f.close
-        }
-        open("#{path}/eod","w"){|f|
-          f.puts Time.now
-        }
+        else
+          # in case of out of :normal status
+          @each_cache_lock.synchronize do
+            each_unpacked_db(target_vn, @hdbc) do |cvn, clast, cclk, cexpt, k, cval|
+              return unless @do_each_vn_dump
+              data = @hdb[n].get(k)
+              if data
+                vn, last, clk, expt, val = unpack_data(data)
+                #puts "#{k} #{clk} #{cclk} #{cmp_clk(clk, cclk)} #{val}"
+                if cmp_clk(clk, cclk) > 0
+                  yield vn_dump_pack(vn, last, clk, expt, k, val)
+                else
+                  yield vn_dump_pack(cvn, clast, cclk, cexpt, k, cval)
+                end
+              else
+                yield vn_dump_pack(cvn, clast, cclk, cexpt, k, cval)
+              end
+            end
+          end
+          each_unpacked_db(target_vn, @hdb) do |vn, last, clk, expt, k, val|
+            return unless @do_each_vn_dump
+            unless @hdbc[n].get(k)
+              yield vn_dump_pack(vn, last, clk, expt, k, val)
+            end
+          end
+        end
+      ensure
+        @do_each_vn_dump = false
       end
 
-      def each_vn_dump(target_vn)
+      def vn_dump_pack(vn, last, clk, expt, k, val)
+          if val
+            return [vn, last, clk, expt, k.length, k, val.length, val].pack("NNNNNa#{k.length}Na#{val.length}")
+          else
+            return [vn, last, clk, expt, k.length, k, 0].pack("NNNNNa#{k.length}N")
+          end          
+      end
+      private :vn_dump_pack
+
+      def each_unpacked_db(target_vn, db)
         count = 0
         tn =  Time.now.to_i
-        @hdb[@hdiv[target_vn]].each{|k,v|
+        db[@hdiv[target_vn]].each do |k,v|
           vn, last, clk, expt, val = unpack_data(v)
           if vn != target_vn || (expt != 0 && tn > expt)
             count += 1
             sleep @each_vn_dump_sleep if count % @each_vn_dump_sleep_count == 0
             next
           end
-          if val
-            yield [vn, last, clk, expt, k.length, k, val.length, val].pack("NNNNNa#{k.length}Na#{val.length}")
-          else
-            yield [vn, last, clk, expt, k.length, k, 0].pack("NNNNNa#{k.length}N")
-          end
-        }
+          yield vn, last, clk, expt, k, val
+        end
       end
+      private :each_unpacked_db
 
       def each_hdb_dump(i,except_vnh = nil)
         count = 0
@@ -490,6 +600,54 @@ module Roma
         }
       end
 
+      # Remove a key for the cache(@hdbc).
+      # +dn+:: number of database
+      # +key+:: key
+      def out_cache(dn, key)
+        @hdbc[dn].out(key)
+      end
+
+      # Calls the geven block, 
+      # passes the cache(@hdbc) element.
+      # +dn+:: number of database
+      # +keys+:: key list
+      def each_cache_by_keys(dn, keys)
+        keys.each do |k|
+          v = @hdbc[dn].get(k)
+          vn, last, clk, expt, val = unpack_data(v)
+          yield [vn, last, clk, expt, k, val]
+        end
+      end
+
+      # Calls the geven block, 
+      # passes the cache(@hdbc) element as the spushv command data format.
+      # +dn+:: number of database
+      # +keys+:: key list
+      def each_cache_dump_pack(dn, keys)
+        keys.each do |k|
+          v = @hdbc[dn].get(k)
+          vn, last, clk, expt, val = unpack_data(v)
+          yield vn_dump_pack(vn, last, clk, expt, k, val)
+        end
+      end
+
+      # Returns a key array in a cache(@hdbc).
+      # +dn+:: number of database
+      # +kn+:: number of keys which is return value
+      def get_keys_in_cache(dn, kn=100)
+        return nil if @do_each_vn_dump
+        ret = []
+        return ret unless @hdbc[dn]
+        count = 0
+        @each_cache_lock.synchronize do
+          @hdbc[dn].each do |k, v|
+            ret << k
+            break if (count+=1) >= kn
+          end
+        end
+        ret
+      end
+
       # Create vnode dump.
       def get_vnode_hash(vn)
         buf = {}
@@ -504,6 +662,108 @@ module Roma
       end
       private :get_vnode_hash
 
+      def flush_db(dn)
+        @hdb[dn].sync
+      end
+
+      def cache_file_name(dn)
+        "#{@storage_path}/#{dn}.cache.#{@ext_name}"
+      end
+
+      def set_db_stat(dn, stat)
+        @stat_lock.synchronize do
+          case @dbs[dn]
+          when :normal
+            if stat == :safecopy_flushing
+              # open cache
+              @hdbc[dn] = open_db(cache_file_name(dn))
+              stop_clean_up { @dbs[dn] = stat }
+              write_log("#{dn} #{stat.to_s}")
+              stat
+            else
+              false
+            end
+          when :safecopy_flushing
+            if stat == :safecopy_flushed
+              write_log("#{dn} #{stat.to_s}")
+              @dbs[dn] = stat
+            else
+              false
+            end
+          when :safecopy_flushed
+            if stat == :cachecleaning
+              write_log("#{dn} #{stat.to_s}")
+              @dbs[dn] = stat
+            else
+              false
+            end
+          when :cachecleaning
+            if stat == :normal
+              write_log("#{dn} #{stat.to_s}")
+              @dbs[dn] = stat
+              # remove cache
+              close_db(@hdbc[dn])
+              @hdbc[dn] = nil
+              if File.exist?("#{@storage_path}/#{dn}.cache.#{@ext_name}")
+                File.unlink("#{@storage_path}/#{dn}.cache.#{@ext_name}")
+              end
+              stat
+            elsif stat == :safecopy_flushing
+              write_log("#{dn} #{stat.to_s}")
+              @dbs[dn] = stat
+            else
+              false
+            end
+          else
+            false
+          end
+        end
+      end
+
+      def get_logfile_list
+        l={}
+        files=Dir.glob("#{@storage_path}/status.log.*")
+        files.each{ |file|
+          if /$.+status\.log\.(\d+)$/=~file
+            l[$1.to_i]=$&
+          end
+        }
+        # sorted by old order
+        l.to_a.sort{|a,b| a[0]<=>b[0]}
+      end
+
+      def open_log
+        logs = get_logfile_list
+        if logs.length == 0
+          @log_name="#{@storage_path}/status.log.1"
+        else
+          if File::stat("#{@fname}.#{logs.last[0]}").size == 0
+            @log_name="#{@fname}.#{logs.last[0]}"
+          else
+            @log_name="#{@fname}.#{logs.last[0]+1}"
+          end
+        end
+        @log_fd=File.open(@log_name,"a")
+      end
+
+      def write_log(line)
+        return unless @log_name
+        # log rotation
+        if File::stat(@log_name).size > 1000 * 1024
+          close_log
+          open_log
+        end
+        t = Time.now
+        tstr = "#{t.strftime('%Y-%m-%dT%H:%M:%S')}.#{t.usec}"
+        @log_fd.write("#{tstr} #{line}\n")
+        @log_fd.flush
+      end
+
+      def close_log
+        @log_fd.close if @log_fd
+        @log_fd = nil
+      end
+
     end # class BasicStorage
 
   end # module Storage