roma 0.8.13 → 0.8.14

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: eaf529b1a3ec8c250e906ce4b360cc694455159d
+  data.tar.gz: f566d3a0c29055ac384683ec5c7859723b9fc091
+SHA512:
+  metadata.gz: 0d259866eb0fc71effe35310966a706bc2517959ed8d37fe793bc3bbf77ccb55b24815df765c1f90945ce397f4e8f63df8be28342290c97fe69aa9190db63af4
+  data.tar.gz: e7ac00f6f5a1db6e4ebfdc17e1170ae431d0a15211ec92c3b978fd1003d33b479822f0dcb0e0aff38e949f2aab8ca1e4155d4c75f074b38934d62a7908dbca1f

data/CHANGELOG

@@ -1,3 +1,23 @@
+*0.8.14 (Mar 26 2014)*
+
+* add timeout parameter for a vnode copy [junji torii] 0522750
+* add command to change routing_trans_timeout [Rui Bando] 8247698
+* Added command to change klength and vlength limit when vnode copy [Paras Patel] fd14206
+* Add spushv_read_timeout and reqpushv_timeout_count set Method [Paras Patel] 264d3ec
+* fix : add rescue in timer_event_1sec [junji torii] 11cc4ea
+* fix : warning of class variable access from toplevel(for adjust Ruby2.1.1) [junji torii] 9df1a5c
+* fix : modify default value(nil -> false) of autorecover func [hiroaki-iwase] 30b18c5
+* add snapshot function [junji torii] 2ab928f
+* add safecopy_integration_test for snapshot [hiroaki-iwase] 8e799c1
+
+*0.8.13-p1 (Jan 15 2014)*
+
+* add timeout parameter for a vnode copy [junji torii] 0522750
+* remove some debug log [hiroaki-iwase] 94d96dc
+* Add Gemfile. [Hiroki Matsue] 9029bf8
+* Ignore test related files. [Hiroki Matsue] 556a931
+* fix:add rescue in timer_event_1sec [junji torii] 11cc4ea
+
 *0.8.13 (Sep 12 2013)*
 
 * change target ruby version about encoding problem (upper 1.9.2 => upper 1.9.1) [hiroaki-iwase] e56d2f4

data/Gemfile

@@ -0,0 +1,17 @@
+source "https://rubygems.org"
+
+gem 'eventmachine'
+
+# Back-end storages
+group :tokyocabinet do
+  gem 'tokyocabinet', :git => 'https://github.com/roma/tokyocabinet-ruby.git'
+end
+
+group :gdbm do
+  gem 'ffi'
+  gem 'gdbm'
+end
+
+group :sqlite3 do
+  gem 'sqlite3'
+end

data/Rakefile

@@ -39,8 +39,8 @@ EXEC_TABLE = Dir.entries(base + 'bin').reject{ |d| d =~ /^\.+$/ || d =~ /^sample
 require File.expand_path(File.join('ruby', 'server', 'lib', 'roma', 'version'), File.dirname(__FILE__))
 VER_NUM = Roma::VERSION
 
-if VER_NUM =~ /([0-9.]+)$/
-  CURRENT_VERSION = $1
+if VER_NUM =~ /([0-9.p-]+)$/
+  CURRENT_VERSION = $1.delete("-")
 else
   CURRENT_VERSION = "0.0.0"
 end

data/ruby/server/bin/cpdb

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+
+path =  File.dirname(File.expand_path($PROGRAM_NAME))
+$LOAD_PATH << path + "/../lib"
+
+require 'roma/tools/cpdb'

data/ruby/server/bin/safecopy_integration_test

@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby
+require 'pathname'
+
+base_path = Pathname(__FILE__).dirname.parent.parent.expand_path
+$LOAD_PATH.unshift("#{base_path}/server/lib")
+
+client_base_path = Pathname(__FILE__).dirname.parent.parent.parent.parent.expand_path
+$LOAD_PATH.unshift("#{client_base_path}/roma-ruby-client/lib")
+
+require 'roma/tools/safecopy_integration_test'

data/ruby/server/bin/safecopy_test

@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby
+require 'pathname'
+
+base_path = Pathname(__FILE__).dirname.parent.parent.expand_path
+$LOAD_PATH.unshift("#{base_path}/server/lib")
+
+client_base_path = Pathname(__FILE__).dirname.parent.parent.parent.parent.expand_path
+$LOAD_PATH.unshift("#{client_base_path}/roma-ruby-client/lib")
+
+require 'roma/tools/safecopy_test'

data/ruby/server/lib/roma/async_process.rb

@@ -462,11 +462,11 @@ module Roma
       Roma::Messaging::ConPool.instance.return_connection(src_nid,con)
       # waiting for pushv
       count = 0
-      while @rttable.search_nodes(vn).include?(@stats.ap_str)==false && count < 300
+      while @rttable.search_nodes(vn).include?(@stats.ap_str)==false && count < @stats.reqpushv_timeout_count
         sleep 0.1
         count += 1
       end
-      if count >= 300
+      if count >= @stats.reqpushv_timeout_count
         @log.warn("#{__method__}:request has been time-out.vn=#{vn} nid=#{src_nid}")
         return :timeout
       end
@@ -748,7 +748,7 @@ module Roma
         @log.error("#{__method__}:#{e.inspect} #{$@}")
 
       ensure
-        if @stats.latency_check_time_count != nil && Time.now.to_i - @stats.latency_data[cmd]["time"] > @stats.latency_check_time_count
+        if @stats.latency_check_time_count && Time.now.to_i - @stats.latency_data[cmd]["time"] > @stats.latency_check_time_count
           average = @stats.latency_data[cmd]["latency"].inject(0.0){|r,i| r+=i }/@stats.latency_data[cmd]["latency"].size
           max = @stats.latency_data[cmd]["latency_max"]["current"]
           min = @stats.latency_data[cmd]["latency_min"]["current"]
@@ -770,6 +770,90 @@ module Roma
       true
     end
 
+    def asyncev_start_storage_flush_process(args)
+      hname, dn = args
+      @log.debug("#{__method__} #{args.inspect}")
+      
+      st = @storages[hname]
+      if st.dbs[dn] != :safecopy_flushing
+        @log.error("Can not flush storage. stat = #{st.dbs[dn]}")
+        return true
+      end
+      t = Thread::new do
+        begin
+          st.flush_db(dn)
+          st.set_db_stat(dn,:safecopy_flushed)
+          @log.info("#{__method__}:storage has flushed. (#{hname}, #{dn})")
+        rescue =>e
+          @log.error("#{__method__}:#{e.inspect} #{$@}")
+        ensure
+        end
+      end
+      t[:name] = __method__
+      true
+    end
+
+    def asyncev_start_storage_cachecleaning_process(args)
+      hname, dn = args
+      @log.debug("#{__method__} #{args.inspect}")
+      
+      st = @storages[hname]
+      if st.dbs[dn] != :cachecleaning
+        @log.error("Can not start cachecleaning process. stat = #{st.dbs[dn]}")
+        return true
+      end
+      t = Thread::new do
+        begin
+          storage_cachecleaning_process(hname, dn)
+        rescue =>e
+          @log.error("#{__method__}:#{e.inspect} #{$@}")
+        ensure
+        end
+      end
+      t[:name] = __method__
+      true
+    end
+
+    def storage_cachecleaning_process(hname, dn)
+      count = 0
+      rcount = 0
+      st = @storages[hname]
+      
+      @do_storage_cachecleaning_process = true
+      loop do
+        # get keys in a cache up to 100 kyes
+        keys = st.get_keys_in_cache(dn)
+        break if keys == nil || keys.length == 0
+        break unless @do_storage_cachecleaning_process
+        
+        # @log.debug("#{__method__}:#{keys.length} keys found")
+        
+        # copy cache -> db
+        st.each_cache_by_keys(dn, keys) do |vn, last, clk, expt, k, v|
+          break unless @do_storage_cachecleaning_process
+          if st.load_stream_dump_for_cachecleaning(vn, last, clk, expt, k, v)
+            count += 1
+            # @log.debug("#{__method__}:[#{vn} #{last} #{clk} #{expt} #{k}] was stored.")
+          else
+            rcount += 1
+            # @log.debug("#{__method__}:[#{vn} #{last} #{clk} #{expt} #{k}] was rejected.")
+          end
+        end
+
+        # remove keys in a cache
+        keys.each { |key| st.out_cache(dn, key) }
+      end
+      if @do_storage_cachecleaning_process == false
+        @log.warn("#{__method__}:uncompleted")
+      else
+        st.set_db_stat(dn,:normal)
+      end
+      @log.debug("#{__method__}:#{count} keys loaded.")
+      @log.debug("#{__method__}:#{rcount} keys rejected.") if rcount > 0
+    ensure
+      @do_storage_cachecleaning_process = false
+    end
+
   end # module AsyncProcess
 
 end # module Roma

data/ruby/server/lib/roma/command/sys_command_receiver.rb

@@ -1,3 +1,4 @@
+require 'roma/async_process'
 
 module Roma
   module Command
@@ -430,7 +431,7 @@ module Roma
           res[@stats.ap_str] = "ACTIVATED"
         elsif s[1] =="off"
           @stats.latency_check_cmd = [] #reset
-          @stats.latency_check_time_count = nil
+          @stats.latency_check_time_count = false
           @stats.latency_log = false
           res[@stats.ap_str] = "DEACTIVATED"
         end
@@ -459,13 +460,13 @@ module Roma
           s.each_index {|idx|
             @stats.latency_check_cmd.push(s[idx]) if idx >= 3
           }
-          @stats.latency_check_time_count = s[1].to_i
+          @stats.latency_check_time_count = s[2].to_i
           @stats.latency_log = true
           send_data("ACTIVATED\r\n")
         elsif s[1] =="off"
           @latency_data = Hash.new { |hash,key| hash[key] = {}}
           @stats.latency_check_cmd = []
-          @stats.latency_check_time_count = nil
+          @stats.latency_check_time_count = false
           @stats.latency_log = false
           send_data("DEACTIVATED\r\n")
         end
@@ -581,7 +582,7 @@ module Roma
           @stats.latency_check_time_count = s[1].to_i
           @stats.latency_log = true
         elsif s[1] == "nil"
-          @stats.latency_check_time_count = nil
+          @stats.latency_check_time_count = false
           @stats.latency_log = false
         end
         res[@stats.ap_str] = "CHANGED"
@@ -599,7 +600,7 @@ module Roma
           @stats.latency_check_time_count = s[1].to_i
           @stats.latency_log = true
         elsif s[1] == "nil"
-          @stats.latency_check_time_count = nil
+          @stats.latency_check_time_count = false
           @stats.latency_log = false
         end
         @stats.latency_check_time_count = s[1].to_i
@@ -828,6 +829,138 @@ module Roma
         send_data("STORED\r\n")
       end
 
+      # set_routing_trans_timeout <sec>
+      def ev_set_routing_trans_timeout(s)
+        if s.length != 2
+          return send_data("CLIENT_ERROR number of arguments\n\r")
+        end
+        if s[1].to_f <= 0
+          return send_data("CLIENT_ERROR time value must be lager than 0\r\n")
+        end
+        res = broadcast_cmd("rset_routing_trans_timeout #{s[1]}\r\n")
+        @stats.routing_trans_timeout = s[1].to_f
+        res[@stats.ap_str] = "STORED"
+
+        send_data("#{res}\r\n")
+      end
+
+      # rset_set_routing_trans_timeout <sec>
+      def ev_rset_routing_trans_timeout(s)
+        if s.length != 2
+          return send_data("CLIENT_ERROR number of arguments\n\r")
+        end
+        if s[1].to_f <= 0
+          return send_data("CLIENT_ERROR time value must be lager than 0\r\n")
+        end
+        @stats.routing_trans_timeout = s[1].to_f
+
+        send_data("STORED\r\n")
+      end
+      
+      # set_spushv_read_timeout <sec>
+      def ev_set_spushv_read_timeout(s)
+        if s.length != 2
+          return send_data("CLIENT_ERROR number of arguments\n\r")
+        end
+        if s[1].to_i <= 0
+          return send_data("CLIENT_ERROR time value must be lager than 0\r\n")
+        end
+        res = broadcast_cmd("rset_spushv_read_timeout #{s[1]}\r\n")
+        @stats.spushv_read_timeout = s[1].to_i
+        res[@stats.ap_str] = "STORED"
+        send_data("#{res}\r\n")
+      end
+
+      # rset_spushv_read_timeout <sec>
+      def ev_rset_spushv_read_timeout(s)
+        if s.length != 2
+          return send_data("CLIENT_ERROR number of arguments\n\r")
+        end
+        if s[1].to_i <= 0
+          return send_data("CLIENT_ERROR time value must be lager than 0\r\n")
+        end
+        @stats.spushv_read_timeout = s[1].to_i
+        send_data("STORED\r\n")
+      end
+
+      # set_reqpushv_timeout_count <sec>
+      def ev_set_reqpushv_timeout_count(s)
+        if s.length != 2
+          return send_data("CLIENT_ERROR number of arguments\n\r")
+        end
+        if s[1].to_i <= 0
+          return send_data("CLIENT_ERROR time value must be lager than 0\r\n")
+        end
+        res = broadcast_cmd("rset_reqpushv_timeout_count #{s[1]}\r\n")
+        @stats.reqpushv_timeout_count = s[1].to_i
+        res[@stats.ap_str] = "STORED"
+        send_data("#{res}\r\n")
+      end
+
+      # ev_rset_reqpushv_timeout_count <sec>
+      def ev_rset_reqpushv_timeout_count(s)
+        if s.length != 2
+          return send_data("CLIENT_ERROR number of arguments\n\r")
+        end
+        if s[1].to_i <= 0
+          return send_data("CLIENT_ERROR time value must be lager than 0\r\n")
+        end
+        @stats.reqpushv_timeout_count = s[1].to_i
+        send_data("STORED\r\n")
+      end
+
+      # set_spushv_klength_warn <byte>
+      def ev_set_spushv_klength_warn(s)
+        if s.length != 2
+          return send_data("CLIENT_ERROR number of arguments\n\r")        
+        end
+        if s[1].to_i <= 0
+          return send_data("CLIENT_ERROR size value must be larger than 0 \r\n")
+        end
+        res = broadcast_cmd("rset_spushv_klength_warn #{s[1]}\r\n")
+        @stats.spushv_klength_warn = s[1].to_i 
+        res[@stats.ap_str] = "STORED"
+        send_data("#{res}\r\n")
+      end
+ 
+      # rset_set_spushv_klength_warn <byte>
+      def ev_rset_spushv_klength_warn(s)
+        if s.length != 2   
+          return send_data("CLIENT_ERROR number of arguments\n\r")
+        end
+        if s[1].to_i <= 0
+          return send_data("CLIENT_ERROR size value must be larger than 0 \r\n")
+        end 
+        @stats.spushv_klength_warn = s[1].to_i 
+        send_data("STORED\r\n") 
+      end
+
+      # set_spushv_vlength_warn <byte>
+      def ev_set_spushv_vlength_warn(s)
+        if s.length != 2
+          return send_data("CLIENT_ERROR number of arguments\n\r")       
+        end
+        if s[1].to_i <= 0
+          return send_data("CLIENT_ERROR size value must be larger than 0 \r\n")
+        end
+        res = broadcast_cmd("rset_spushv_vlength_warn #{s[1]}\r\n")
+        @stats.spushv_vlength_warn = s[1].to_i
+        res[@stats.ap_str] = "STORED"
+        send_data("#{res}\r\n")
+      end
+ 
+      # rset_set_spushv_vlength_warn <byte>
+      def ev_rset_spushv_vlength_warn(s)
+        if s.length != 2   
+          return send_data("CLIENT_ERROR number of arguments\n\r")
+        end
+        if s[1].to_i <= 0
+          return send_data("CLIENT_ERROR size value must be larger than 0\r\n")
+        end 
+        @stats.spushv_vlength_warn = s[1].to_i 
+        send_data("STORED\r\n") 
+      end
+
       # wb_command_map <hash string>
       # ex.
       # {:set=>1,:append=>2,:delete=>3}
@@ -895,6 +1028,48 @@ module Roma
         send_data("STORED\r\n")
       end
 
+      # set_storage_status [number of file][safecopy|normal]{hash_name}
+      def ev_set_storage_status(s)
+        if s.length < 3
+          return send_data("CLIENT_ERROR number of arguments (#{s.length - 1} for 2)\r\n")
+        end
+
+        if s.length >= 4
+          hname = s[3]
+        else
+          hname = 'roma'
+        end
+        st = @storages[hname]
+        unless st
+          return send_data("CLIENT_ERROR hash_name = #{hanme} dose not found\r\n")
+        end
+        dn = s[1].to_i
+        if st.divnum <= dn
+          return send_data("CLIENT_ERROR divnum <= #{dn}\r\n")
+        end
+        if s[2] == 'safecopy'
+          if st.dbs[dn] != :normal
+            return send_data("CLIENT_ERROR storage[#{dn}] != :normal status\r\n")
+          end
+          if st.set_db_stat(dn, :safecopy_flushing) == false
+            return send_data("CLIENT_ERROR storage[#{dn}] != :normal status\r\n")
+          end
+          Roma::AsyncProcess::queue.push(Roma::AsyncMessage.new('start_storage_flush_process',[hname, dn]))
+        elsif s[2] ==  'normal'
+          if st.dbs[dn] != :safecopy_flushed
+            return send_data("CLIENT_ERROR storage[#{dn}] != :safecopy_flushed status\r\n")
+          end
+          if st.set_db_stat(dn, :cachecleaning) == false
+            return send_data("CLIENT_ERROR storage[#{dn}] != :safecopy_flushed status\r\n")
+          end
+          Roma::AsyncProcess::queue.push(Roma::AsyncMessage.new('start_storage_cachecleaning_process',[hname, dn]))
+        else
+          return send_data("CLIENT_ERROR status parse error\r\n")
+        end
+        
+        send_data("PUSHED\r\n")
+      end
+
       private 
 
       def dcnice(p)
@@ -960,3 +1135,4 @@ module Roma
     end # module SystemCommandReceiver
   end # module Command
 end # module Roma
+

data/ruby/server/lib/roma/command/vn_command_receiver.rb

@@ -45,14 +45,20 @@ module Roma
         count = rcount = 0
         @log.debug("#{__method__}:#{s.inspect} received.")
         loop {
-          context_bin = read_bytes(20, 100)
+          context_bin = read_bytes(20, @stats.spushv_read_timeout)
           vn, last, clk, expt, klen = context_bin.unpack('NNNNN')
           break if klen == 0 # end of dump ?
           k = read_bytes(klen)
-          vlen_bin = read_bytes(4, 100)
+          vlen_bin = read_bytes(4, @stats.spushv_read_timeout)
           vlen, =  vlen_bin.unpack('N')
           if vlen != 0
-            v = read_bytes(vlen, 100)
+            if klen > @stats.spushv_klength_warn
+              @log.warn("#{__method__}:Too long key: key = #{k}")
+            end
+            if vlen > @stats.spushv_vlength_warn
+              @log.warn("#{__method__}:Too long value: key = #{k} vlen = #{vlen}")
+            end
+            v = read_bytes(vlen, @stats.spushv_read_timeout)
 
             createhash(s[1]) unless @storages[s[1]]
             if @storages[s[1]].load_stream_dump(vn, last, clk, expt, k, v)