roma 1.2.0 → 1.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a17bad1bbf7dabab9d9833e7458902691dae3117
-  data.tar.gz: e97e8fb05d535d000d276dcea350317629ef9a00
+  metadata.gz: 5ad632c67885c4aaf13b8fdec745d45bbcd94feb
+  data.tar.gz: 9c5967e28f0ae8f250223b1e4d81852ad0723c38
 SHA512:
-  metadata.gz: f5dc132fd3475bcd2a4edceaa579bbf9b18319a84dda1ce7c443b1e40ab1cdb538e7bb791cb693d497b98867990ad31c294ee4f7969f1b20616b31a6bcb05de3
-  data.tar.gz: 3d2854df8f09147235a49a5891037fa65e5be84a59c73bacbfb3739decd8f0a3c728bdac2d44fb69e7aa2026c53ffef1abbe898695e62e5a098d7fcd9e76df59
+  metadata.gz: 343240d35ac31586f4325ac5f2b78de0e69b19a96684b76fff5ac0df0aa032eef53287283a35701b77d8ba6f06a8d58ed0116216fd5e5210d60851631b1ac6ba
+  data.tar.gz: 99bd24be9e021ae43d7330b69f0b6935422d790165c04a934118d76b3672d8c3d35627ac0b888ade65ebc56291409c2dc0c53303c9dae7f23ca55341285cfcb1

data/CHANGELOG

@@ -1,4 +1,47 @@
-*1.1.0 (Aug 19 2015)*
+*1.3.0 (Jan 21 2016)*
+
+* change unit test to pass in the poor env [hiroaki-iwase] f0d9cc8
+
+*1.3.0RC1 (Nov 13 2015)*
+
+* Update README.md [Paras Patel] 3350844
+* automation some manual tests [hiroaki-iwase] dea72fa
+  * auto_recover
+  * set_routing_trans_timeout
+  * st_class
+  * adm_tool
+  * check_enable_repeathost
+  * check_replication _in_host
+* add below func's unit test [hiroaki-iwase] f9a1721
+ * jaro-winkler 
+ * stat log level 
+ * stat secondary 
+ * shutdown_self 
+ * stat failover 
+ * get_key_info 
+ * enabled_repetition_in_routing? 
+ * switch_dns_caching 
+ * add_rttable_sub_nid 
+ * delete_rttable_sub_nid 
+ * clear_rttable_sub_nid 
+ * del_latency_avg_calc_cmd 
+ * add_latency_avg_calc_cmd 
+ * chg_latency_avg_calc_time_count 
+ * set_latency_avg_calc_rule
+* add test of below func [hiroaki-iwase] 613c0d0
+  * stat log level 
+  * stat secondary 
+  * shutdown_self
+* add unit test of jaro winkler [hiroaki-iwase] 22fd7b2
+* remove puts method in testfile [hiroaki-iwase] cec8a6d
+* implement cluster replication function [hiroaki-iwase] a1fa5e8
+* implement get_expt command [hiroaki-iwase] 04f1a14
+* change log level of push_a_vnode_stream from info to debug [hiroaki-iwase] e668fc0
+* add how to contributing [hiroaki-iwase] 6d9945a
+* add progress rate to data accumulation tool [hiroaki-iwase] 2f9d8aa
+* implement consistency check tool [Hiroaki Iwase] 3211bd9
+
+*1.2.0 (Aug 19 2015)*
 
 * Change gemspec [Hiroaki Iwase] db2bb42
 * Modify unit-test [Hiroaki Iwase] 7479034

data/Gemfile.lock

@@ -1,37 +1,37 @@
 PATH
   remote: .
   specs:
-    roma (1.2.0)
+    roma (1.3.0)
       eventmachine (~> 1.0.0)
       jaro_winkler (~> 1.3.5)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    archive-zip (0.7.0)
+    archive-zip (0.8.0)
       io-like (~> 0.3.0)
-    eventmachine (1.0.3)
-    ffi (1.9.6)
-    gdbm (1.2)
+    eventmachine (1.0.9.1)
+    ffi (1.9.10)
+    gdbm (1.3.0)
     gqtp (1.0.6)
-    groonga-client (0.1.0)
+    groonga-client (0.1.9)
       gqtp (>= 1.0.4)
       groonga-command (>= 1.0.8)
-    groonga-command (1.0.9)
+    groonga-command (1.1.5)
       json
     io-like (0.3.0)
-    jaro_winkler (1.3.5)
-    json (1.8.1)
-    pkg-config (1.1.6)
-    power_assert (0.2.2)
-    rake (10.2.2)
-    rroonga (4.0.8)
+    jaro_winkler (1.3.7)
+    json (1.8.3)
+    pkg-config (1.1.7)
+    power_assert (0.2.7)
+    rake (10.5.0)
+    rroonga (5.1.1)
       archive-zip
       groonga-client (>= 0.0.3)
       json
       pkg-config
-    sqlite3 (1.3.9)
-    test-unit (3.1.2)
+    sqlite3 (1.3.11)
+    test-unit (3.1.7)
       power_assert
 
 PLATFORMS
@@ -45,3 +45,6 @@ DEPENDENCIES
   rroonga
   sqlite3
   test-unit
+
+BUNDLED WITH
+   1.11.2

data/README.md

@@ -1,4 +1,4 @@
-# ROMA - A Distributed Key-Value Store in Ruby
+# ROMA - A Distributed Key-Value Store in Ruby 
 
 ROMA is one of the data storing systems for distributed key-value stores.  
 It is a completely decentralized distributed system that consists of multiple
@@ -74,5 +74,19 @@ END
 
 Refer to [Commands](http://roma-kvs.org/commands.html "Commands") for more detail information about ROMA Commands.
 
+
+## Contributing
+
+If you would like to contribute, please...
+
+1. Fork.
+2. Download [Ruby Client](https://github.com/roma/roma-ruby-client) to same directory.
+3. Make changes in a branch & add unit tests.
+4. Run Unit Test
+  * `ruby test/run_test.rb` (if unit test fails, run it again - it's fickle).
+5. Create a pull request.
+
+Contributions, improvements, comments and suggestions are welcome!
+
 ## Promoters
 Roma is promoted by [Rakuten, Inc.](http://global.rakuten.com/corp/) and [Rakuten Institute of Technology](http://rit.rakuten.co.jp/).

data/bin/consistency_test

@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby
+require 'pathname'
+
+base_path = Pathname(__FILE__).dirname.parent.expand_path
+$LOAD_PATH.unshift("#{base_path}/lib")
+
+client_base_path = Pathname(__FILE__).dirname.parent.parent.expand_path
+$LOAD_PATH.unshift("#{client_base_path}/roma-ruby-client/lib")
+
+require 'roma/tools/consistency_test'

data/bin/data_accumulation

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+require 'pathname'
+
+base_path = Pathname(__FILE__).dirname.parent.expand_path
+$LOAD_PATH.unshift("#{base_path}/lib")
+
+client_base_path = Pathname(__FILE__).dirname.parent.parent.expand_path
+$LOAD_PATH.unshift("#{client_base_path}/roma-ruby-client/lib")
+
+require 'roma/tools/data_accumulation'
+

data/lib/roma/async_process.rb

@@ -643,7 +643,7 @@ module Roma
     end
 
     def push_a_vnode_stream(hname, vn, nid)
-      @log.info("#{__method__}:hname=#{hname} vn=#{vn} nid=#{nid}")
+      @log.debug("#{__method__}:hname=#{hname} vn=#{vn} nid=#{nid}")
 
       stop_clean_up
 
@@ -670,7 +670,7 @@ module Roma
         con.write(data)
         sleep @stats.stream_copy_wait_param
       end
-      con.write("\0" * 20) # end of steram
+      con.write("\0" * 20) # end of stream
 
       res = con.gets # STORED\r\n or error string
       Roma::Messaging::ConPool.instance.return_connection(nid, con)
@@ -1043,5 +1043,42 @@ module Roma
 
       get_point(f, target_time, type, latency_time, new_pos, target_pos)
     end
+
+    def asyncev_start_replicate_existing_data_process(args)
+      # args is [$roma.cr_writer.replica_rttable])
+      t = Thread.new do
+        begin
+          $roma.cr_writer.run_existing_data_replication = true
+          replicate_existing_data_process(args)
+        rescue => e
+          @log.error("#{__method__}:#{e.inspect} #{$ERROR_POSITION}")
+        ensure
+          $roma.cr_writer.run_existing_data_replication = false
+        end
+      end
+      t[:name] = __method__
+    end
+
+    def replicate_existing_data_process(args)
+      @log.info("#{__method__} :start.")
+
+      @storages.each_key do |hname|
+        @rttable.v_idx.each_key do |vn|
+          raise unless $roma.cr_writer.run_existing_data_replication
+          args[0].v_idx[vn].each do |replica_nid|
+            res = push_a_vnode_stream(hname, vn, replica_nid)
+            if res != 'STORED'
+              @log.error("#{__method__}:push_a_vnode was failed:hname=#{hname} vn=#{vn}:#{res}")
+              return false
+            end
+          end
+        end
+      end
+
+      @log.info("#{__method__} has done.")
+    rescue => e
+      @log.error("#{e}\n#{$ERROR_POSITION}")
+    end
+
   end # module AsyncProcess
 end # module Roma

data/lib/roma/command/sys_command_receiver.rb

@@ -224,6 +224,86 @@ module Roma
         end
       end
 
+      # switch_replication command is change status of cluster replication
+      # if you want to activate, assign 1 nid(addr_port) of replication cluster as argument.
+      # if you want to copy existing data, add the 'all' after nid as argument
+      # switch_replication <true|false> [nid] [copy target]
+      def ev_switch_replication(s)
+        unless s.length.between?(2, 4)
+          return send_data("CLIENT_ERROR number of arguments\r\n")
+        end
+        unless s[1] =~ /^(true|false)$/
+          return send_data("CLIENT_ERROR value must be true or false\r\n")
+        end
+        if s[3] && s[3] != 'all'
+          return send_data("CLIENT_ERROR [copy target] must be all or nil\r\n")
+        end
+
+        res = broadcast_cmd("rswitch_replication #{s[1]} #{s[2]} #{s[3]}\r\n")
+
+        timeout(1){
+          case s[1]
+          when 'true'
+            $roma.cr_writer.update_mklhash(s[2])
+            $roma.cr_writer.update_nodelist(s[2])
+            $roma.cr_writer.update_rttable(s[2])
+            $roma.cr_writer.run_replication = true
+            if s[3] == 'all'
+              $roma.cr_writer.run_existing_data_replication = true
+              Roma::AsyncProcess::queue.push(Roma::AsyncMessage.new('start_replicate_existing_data_process', [$roma.cr_writer.replica_rttable]))
+            end
+            res[@stats.ap_str] = "ACTIVATED"
+          when 'false'
+            $roma.cr_writer.replica_mklhash = nil
+            $roma.cr_writer.replica_nodelist = []
+            $roma.cr_writer.replica_rttable = nil
+            $roma.cr_writer.run_replication = false
+            $roma.cr_writer.run_existing_data_replication = false
+            res[@stats.ap_str] = "DEACTIVATED"
+          end
+        }
+        send_data("#{res}\r\n")
+      rescue => e
+        send_data("#{e.class}: #{e}\r\n")
+      end
+
+      # rswitch_replication <true|false> [nid] [copy target]
+      def ev_rswitch_replication(s)
+        unless s.length.between?(2, 4)
+          return send_data("CLIENT_ERROR number of arguments\n\r")
+        end
+        unless s[1] =~ /^(true|false)$/
+          return send_data("CLIENT_ERROR value must be true or false\n\r")
+        end
+        if s[3] && s[3] != 'all'
+          return send_data("CLIENT_ERROR [copy target] must be all or nil\r\n")
+        end
+
+        timeout(1){
+          case s[1]
+          when 'true'
+            $roma.cr_writer.update_mklhash(s[2])
+            $roma.cr_writer.update_nodelist(s[2])
+            $roma.cr_writer.update_rttable(s[2])
+            $roma.cr_writer.run_replication = true
+            if s[3] == 'all'
+              $roma.cr_writer.run_existing_data_replication = true
+              Roma::AsyncProcess::queue.push(Roma::AsyncMessage.new('start_replicate_existing_data_process', [$roma.cr_writer.replica_rttable]))
+            end
+            send_data("ACTIVATED\r\n")
+          when 'false'
+            $roma.cr_writer.replica_mklhash = nil
+            $roma.cr_writer.replica_nodelist = []
+            $roma.cr_writer.replica_rttable = nil
+            $roma.cr_writer.run_replication = false
+            $roma.cr_writer.run_existing_data_replication = false
+            send_data("DEACTIVATED\r\n")
+          end
+        }
+      rescue => e
+        send_data("#{e.class}: #{e}\r\n")
+      end
+
       # dcnice command is setting priority for a data-copy thread.
       # a niceness of 1 is the highest priority and 5 is the lowest priority.
       # dcnice <priority:1 to 5>

data/lib/roma/command/vn_command_receiver.rb

@@ -6,7 +6,7 @@ module Roma
 
     module VnodeCommandReceiver
 
-      # spushv <true/false>
+      # spushv_protection <true/false>
       def ev_spushv_protection(s)
         if s.length == 1
           send_data("#{@stats.spushv_protection}\r\n")

data/lib/roma/config.rb

@@ -4,7 +4,7 @@ module Roma
   
   module Config
     # check when the booting
-    VERSION = "1.2.0"
+    VERSION = "1.3.0"
 
     DEFAULT_PORT = 12000
     DEFAULT_NAME = 'ROMA'

data/lib/roma/plugin/plugin_storage.rb

@@ -126,6 +126,7 @@ module Roma
         d = Digest::SHA1.hexdigest(key).hex % @rttable.hbits
         vn = @rttable.get_vnode_id(d)
         nodes = @rttable.search_nodes_for_write(vn)
+
         if nodes[0] != @nid
           cmd = "fdelete #{key}\e#{hname}"
           s[2..-1].each{|c| cmd << " #{c}"}
@@ -137,6 +138,7 @@ module Roma
           end
           return send_data("#{res}\r\n")
         end
+
         unless @storages.key?(hname)
           send_data("SERVER_ERROR #{hname} does not exists.\r\n")
           return
@@ -165,6 +167,12 @@ module Roma
           end
         }
         return send_data("NOT_FOUND\r\n") unless res[4]
+
+        if $roma.cr_writer.run_replication
+          fnc = 'delete'
+          Roma::WriteBehindProcess::push(nil, "#{fnc} #{s[1]}\r\n", s[1], nil)
+        end
+
         send_data("DELETED\r\n")
       end
 
@@ -208,6 +216,12 @@ module Roma
           end
         }
         return send_data("NOT_FOUND\r\n") unless res[4]
+
+        if $roma.cr_writer.run_replication
+          fnc = 'delete'
+          Roma::WriteBehindProcess::push(nil, "#{fnc} #{s[1]}\r\n", s[1], nil)
+        end
+
         send_data("DELETED\r\n")
       end
 
@@ -339,6 +353,10 @@ module Roma
             Roma::WriteBehindProcess::push(hname, @stats.wb_command_map[:set_expt], key, expt.to_s)
           end
           redundant(nodes[1..-1], hname, key, d, ret[2], ret[3], ret[4])
+          if $roma.cr_writer.run_replication
+            fnc = 'set_expt'
+            Roma::WriteBehindProcess::push(nil, "#{fnc} #{s[1]} #{expt}\r\n", s[1], expt)
+          end
           send_data("STORED\r\n")
         else
           return send_data("NOT_STORED\r\n")
@@ -376,12 +394,69 @@ module Roma
             Roma::WriteBehindProcess::push(hname, @stats.wb_command_map[:set_expt], key, expt.to_s)
           end
           redundant(nodes[1..-1], hname, key, d, ret[2], ret[3], ret[4])
+          if $roma.cr_writer.run_replication
+            fnc = 'set_expt'
+            Roma::WriteBehindProcess::push(nil, "#{fnc} #{s[1]} #{expt}\r\n", s[1], expt)
+          end
           send_data("STORED\r\n")
         else
           return send_data("NOT_STORED\r\n")
         end
       end
 
+      # If you want to get expired time as UNIXTIME, set the 'unix' in last argument
+      # Unless set this, expired time will be sent back as date format.
+      # get_expt <key> [unix]
+      def ev_get_expt(s)
+        unless s.length.between?(2, 3)
+          @log.error("get_expt: wrong number of arguments(#{s.length-1} to 2-3)")
+          return send_data("CLIENT_ERROR Wrong number of arguments.\r\n")
+        end
+        case s[2]
+        when 'unix'
+          is_unix = true
+        when nil
+          is_unix = false
+        else
+          @log.error("get_expt: wrong format of arguments.")
+          return send_data("CLIENT_ERROR Wrong format of arguments.\r\n")
+        end
+
+        key, hname = s[1].split("\e")
+        hname ||= @defhash
+        unless @storages.key?(hname)
+          send_data("SERVER_ERROR #{hname} does not exists.\r\n")
+          return
+        end
+
+        d = Digest::SHA1.hexdigest(key).hex % @rttable.hbits
+        vn = @rttable.get_vnode_id(d)
+
+        nodes = @rttable.search_nodes(vn)
+        unless nodes.include?(@nid)
+          @log.warn("forward get_expt #{s[1]} #{s[2]}")
+          res = forward_get_expt(nodes[0], vn, s[1], s[2])
+          if res
+            send_data(res)
+          else
+            send_data("SERVER_ERROR Message forward failed.\r\n")
+          end
+          return
+        end
+
+        data = @storages[hname].db_get(vn, key)
+        if data
+          if is_unix
+            expt = data.unpack('NNNNa*')[3]
+          else
+            expt = Time.at(data.unpack('NNNNa*')[3])
+          end
+          send_data("#{expt}\r\n")
+        end
+        send_data("END\r\n")
+      end
+
+
       # set_size_of_zredundant <n>
       def ev_set_size_of_zredundant(s)
         if s.length != 2 || s[1].to_i == 0
@@ -451,6 +526,22 @@ module Roma
         nil
       end
 
+      def forward_get_expt(nid, vn, key, is_unix=nil)
+        con = get_connection(nid)
+        con.send("get_expt #{key} #{is_unix}\r\n")
+        res = ''
+        while((line = con.gets)!="END\r\n")
+          res = line.chomp
+        end
+        return_connection(nid, con)
+        @rttable.proc_succeed(nid)
+        res
+      rescue => e
+        @rttable.proc_failed(nid)
+        @log.error("forward get_expt failed:nid=#{nid} key=#{key}")
+        nil
+      end
+
       def store(fnc, hname, vn, k, d, expt, v, nodes)
         expt = chg_time_expt(expt)
         unless @storages.key?(hname)
@@ -471,6 +562,10 @@ module Roma
             Roma::WriteBehindProcess::push(hname, @stats.wb_command_map[fnc], k, ret[4])
           end
           redundant(nodes, hname, k, d, ret[2], expt, ret[4])
+          if $roma.cr_writer.run_replication
+            k = "#{k}\e#{hname}" if hname != @defhash
+            Roma::WriteBehindProcess::push(nil, "#{fnc} #{k} 1 #{expt} #{v.length} \r\n#{v}\r\n", k, v)
+          end
           send_data("STORED\r\n")
         else
           @log.error("#{fnc} NOT_STORED:#{hname} #{vn} #{k} #{d} #{expt}")
@@ -505,6 +600,11 @@ module Roma
           if @stats.wb_command_map.key?(:cas)
             Roma::WriteBehindProcess::push(hname, @stats.wb_command_map[:cas], k, ret[4])
           end
+          if $roma.cr_writer.run_replication
+            k = "#{k}\e#{hname}" if hname != @defhash
+            fnc = 'set' # To restrain a defference between main and replica cluster due to clk 
+            Roma::WriteBehindProcess::push(nil, "#{fnc} #{k} 0 #{expt} #{v.length} \r\n#{v}\r\n", k, v)
+          end
           redundant(nodes, hname, k, d, ret[2], expt, ret[4])
           send_data("STORED\r\n")          
         end
@@ -585,7 +685,7 @@ module Roma
         store(fnc, hname, vn, key, d, s[3].to_i, v, nodes)
       end
 
-      def store_incr_decr(fnc, hname, vn, k, d,  v, nodes)
+      def store_incr_decr(fnc, hname, vn, k, d, v, nodes)
         unless @storages.key?(hname)
           send_data("SERVER_ERROR #{hname} does not exists.\r\n")
           return
@@ -603,6 +703,10 @@ module Roma
           if @stats.wb_command_map.key?(fnc)
             Roma::WriteBehindProcess::push(hname, @stats.wb_command_map[fnc], k, res[4])
           end
+          if $roma.cr_writer.run_replication
+            k = "#{k}\e#{hname}" if hname != @defhash
+            Roma::WriteBehindProcess::push(nil, "#{fnc} #{k} #{v}\r\n", k, v)
+          end
           redundant(nodes, hname, k, d, res[2], res[3], res[4])
           send_data("#{res[4]}\r\n")
         else