roma 0.8.14 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: eaf529b1a3ec8c250e906ce4b360cc694455159d
4
- data.tar.gz: f566d3a0c29055ac384683ec5c7859723b9fc091
3
+ metadata.gz: 463ee73649102c0de8e1b6a7fa8e42f5130f2332
4
+ data.tar.gz: d8f360a9690da7729b3b939486ac9c12a3eb32bf
5
5
  SHA512:
6
- metadata.gz: 0d259866eb0fc71effe35310966a706bc2517959ed8d37fe793bc3bbf77ccb55b24815df765c1f90945ce397f4e8f63df8be28342290c97fe69aa9190db63af4
7
- data.tar.gz: e7ac00f6f5a1db6e4ebfdc17e1170ae431d0a15211ec92c3b978fd1003d33b479822f0dcb0e0aff38e949f2aab8ca1e4155d4c75f074b38934d62a7908dbca1f
6
+ metadata.gz: ec235f905b6bf987aaa157b2c360dfe0b8adf683597cff10fb1149a27aff25a1e9d976a842b7c91907d78e4a2b0dafa281f5af97ce21d408571bbbad99911178
7
+ data.tar.gz: 6e9eda8b8885efc140708c2adc549b633263ed29e9bac65ccd5492f3d002c3bf0a8cc3556fc72220bca1ccbeda3111c0dc1c67365136a910a974b51530486742
data/CHANGELOG CHANGED
@@ -1,3 +1,22 @@
1
+ *1.0.0 (Sep 30 2014)*
2
+
3
+ * Add new function to get vnodes information of each data [hiroaki-iwase] ad25c6e
4
+ * Add new function to get key list without stopping ROMA [hiroaki-iwase] f36e907
5
+ * Add new GUI control system(Gladiator) [hiroaki-iwase] 66dfd58
6
+ * Add new function to get log data via ROMA console [hiroaki-iwase] b1995e4
7
+ * change some snapshot and storage logic [hiroaki-iwase] 257b0df
8
+ * Add new function to check routing consistecy [hiroaki-iwase] dcd1ace
9
+ * Add new function of getting routing event [hiroaki-iwase] e297215
10
+ * fix : no data copy in the join process when 2 hosts, 2 processes [junji torii] 10eae74
11
+ * Merge branch 'master' of git@github.com:roma/roma into fix_join [junji torii] 41e0269
12
+ * refacort:error case for the release command. [junji torii] 0144b63
13
+ * change release logic [hiroaki-iwase] f950fa1
14
+ * bugfix:end condition of recover process in no_action [junji torii] 9b41a3c
15
+ * {func}__prev support for DSL [junji torii] d772c56
16
+ * Add new function to get first instance information [hiroaki-iwase] e4b465f
17
+ * {func}__prev support in write-behind [junji torii] 96af508
18
+ * each_vn_dump supported in normal status [junji torii] 546ee7d
19
+
1
20
  *0.8.14 (Mar 26 2014)*
2
21
 
3
22
  * add timeout parameter for a vnode copy [junji torii] 0522750
@@ -305,7 +305,7 @@ module Roma
305
305
  t = Thread::new do
306
306
  begin
307
307
  timeout(@rttable.auto_recover_time){
308
- loop{
308
+ loop{
309
309
  sleep 1
310
310
  break if @rttable.auto_recover_status != "preparing"
311
311
  #break if @stats.run_join #run_join don't have possibility to be true in this case.
@@ -331,7 +331,7 @@ module Roma
331
331
  when :no_action
332
332
  @log.debug("auto recover NOT start. Because lost action is [no_action]")
333
333
  end
334
- end
334
+ end
335
335
  end
336
336
  t[:name] = __method__
337
337
  end
@@ -366,6 +366,7 @@ module Roma
366
366
  @do_acquired_recover_process = true
367
367
  loop do
368
368
  break unless @do_acquired_recover_process
369
+ break if @rttable.num_of_vn(@stats.ap_str)[2] == 0 # short vnodes
369
370
 
370
371
  vn, nodes, is_primary = @rttable.select_vn_for_recover(exclude_nodes)
371
372
  break unless vn
@@ -487,12 +488,18 @@ module Roma
487
488
 
488
489
  @do_release_process = true
489
490
  while(@rttable.has_node?(@stats.ap_str)) do
491
+ break unless @do_release_process
490
492
  @rttable.each_vnode do |vn, nids|
491
493
  break unless @do_release_process
492
494
  if nids.include?(@stats.ap_str)
493
495
 
494
496
  to_nid, new_nids = @rttable.select_node_for_release(@stats.ap_str, @stats.rep_host, nids)
495
- unless sync_a_vnode_for_release(vn, to_nid, new_nids)
497
+ res = sync_a_vnode_for_release(vn, to_nid, new_nids)
498
+ if res == :abort
499
+ @log.error("#{__method__}:release_process aborted due to SERVER_ERROR received.")
500
+ @do_release_process = false
501
+ end
502
+ if res == false
496
503
  @log.warn("#{__method__}:error at vn=#{vn} to_nid=#{to_nid} new_nid=#{new_nids}")
497
504
  redo
498
505
  end
@@ -510,7 +517,7 @@ module Roma
510
517
  def sync_a_vnode_for_release(vn, to_nid, new_nids)
511
518
  nids = @rttable.search_nodes(vn)
512
519
 
513
- if nids.include?(to_nid)==false || (is_primary && nids[0]!=to_nid)
520
+ if nids.include?(to_nid)==false
514
521
  @log.debug("#{__method__}:#{vn} #{to_nid}")
515
522
  # change routing data at the vnode and synchronize a data
516
523
  nids << to_nid
@@ -523,6 +530,7 @@ module Roma
523
530
  if res != "STORED"
524
531
  @rttable.rollback(vn)
525
532
  @log.error("#{__method__}:push_a_vnode was failed:hname=#{hname} vn=#{vn}:#{res}")
533
+ return :abort if res.start_with?("SERVER_ERROR")
526
534
  return false
527
535
  end
528
536
  }
@@ -607,14 +615,31 @@ module Roma
607
615
  if @rttable.rn == 1
608
616
  return [new_nid]
609
617
  end
610
- if nodes.length > @rttable.rn
611
- nodes.delete(new_nid)
612
- nodes.delete(nodes.last)
613
- nodes << new_nid
618
+ # [node_a, node_b, new_nid]
619
+ nodes.delete(new_nid)
620
+ # [node_a, node_b]
621
+
622
+ if nodes.length >= @rttable.rn
623
+ host = new_nid.split(/[:_]/)[0]
624
+ buf = [] # list of a same host
625
+ nodes.each do |nid|
626
+ buf << nid if nid.split(/[:_]/)[0] == host
627
+ end
628
+ if buf.length > 0
629
+ # include same host
630
+ # delete a last one, due to save a primary node
631
+ nodes.delete(buf.last)
632
+ else
633
+ nodes.delete(nodes.last)
634
+ end
614
635
  end
636
+
615
637
  if is_primary
616
- nodes.delete(new_nid)
638
+ # [new_nid, node_a]
617
639
  nodes.insert(0,new_nid)
640
+ else
641
+ # [node_a, new_nid]
642
+ nodes << new_nid
618
643
  end
619
644
  nodes
620
645
  end
@@ -636,7 +661,7 @@ module Roma
636
661
  return res.chomp
637
662
  end
638
663
 
639
- @storages[hname].each_vn_dump(vn){|data|
664
+ res_dump = @storages[hname].each_vn_dump(vn) do |data|
640
665
 
641
666
  unless @do_push_a_vnode_stream
642
667
  con.close
@@ -646,12 +671,16 @@ module Roma
646
671
 
647
672
  con.write(data)
648
673
  sleep @stats.stream_copy_wait_param
649
- }
674
+ end
650
675
  con.write("\0"*20) # end of steram
651
676
 
652
677
  res = con.gets # STORED\r\n or error string
653
678
  Roma::Messaging::ConPool.instance.return_connection(nid,con)
654
679
  res.chomp! if res
680
+ if res_dump == false
681
+ @log.error("#{__method__}:each_vn_dump in hname=#{hname} vn=#{vn} nid=#{nid}")
682
+ return "CANCELED"
683
+ end
655
684
  res
656
685
  rescue =>e
657
686
  @log.error("#{e}\n#{$@}")
@@ -773,7 +802,7 @@ module Roma
773
802
  def asyncev_start_storage_flush_process(args)
774
803
  hname, dn = args
775
804
  @log.debug("#{__method__} #{args.inspect}")
776
-
805
+
777
806
  st = @storages[hname]
778
807
  if st.dbs[dn] != :safecopy_flushing
779
808
  @log.error("Can not flush storage. stat = #{st.dbs[dn]}")
@@ -796,7 +825,7 @@ module Roma
796
825
  def asyncev_start_storage_cachecleaning_process(args)
797
826
  hname, dn = args
798
827
  @log.debug("#{__method__} #{args.inspect}")
799
-
828
+
800
829
  st = @storages[hname]
801
830
  if st.dbs[dn] != :cachecleaning
802
831
  @log.error("Can not start cachecleaning process. stat = #{st.dbs[dn]}")
@@ -818,16 +847,16 @@ module Roma
818
847
  count = 0
819
848
  rcount = 0
820
849
  st = @storages[hname]
821
-
850
+
822
851
  @do_storage_cachecleaning_process = true
823
852
  loop do
824
853
  # get keys in a cache up to 100 kyes
825
854
  keys = st.get_keys_in_cache(dn)
826
855
  break if keys == nil || keys.length == 0
827
856
  break unless @do_storage_cachecleaning_process
828
-
857
+
829
858
  # @log.debug("#{__method__}:#{keys.length} keys found")
830
-
859
+
831
860
  # copy cache -> db
832
861
  st.each_cache_by_keys(dn, keys) do |vn, last, clk, expt, k, v|
833
862
  break unless @do_storage_cachecleaning_process
@@ -854,6 +883,91 @@ module Roma
854
883
  @do_storage_cachecleaning_process = false
855
884
  end
856
885
 
886
+ def asyncev_start_get_routing_event(args)
887
+ @log.debug("#{__method__} #{args}")
888
+ t = Thread::new do
889
+ begin
890
+ get_routing_event
891
+ rescue => e
892
+ @log.error("#{__method__}:#{e.inspect} #{$@}")
893
+ ensure
894
+ end
895
+ end
896
+ t[:name] = __method__
897
+ end
898
+
899
+ def get_routing_event
900
+ @log.info("#{__method__}:start.")
901
+
902
+ routing_path = Config::RTTABLE_PATH
903
+ f_list = Dir.glob("#{routing_path}/#{@stats.ap_str}*")
904
+
905
+ f_list.each{|fname|
906
+ IO.foreach(fname){|line|
907
+ if line =~ /join|leave/
908
+ @rttable.event.shift if @rttable.event.size >= @rttable.event_limit_line
909
+ @rttable.event << line.chomp
910
+ end
911
+ }
912
+ }
913
+
914
+ @log.info("#{__method__} has done.")
915
+ rescue =>e
916
+ @log.error("#{e}\n#{$@}")
917
+ end
918
+
919
+ def asyncev_start_get_logs(args)
920
+ @log.debug("#{__method__} #{args}")
921
+ t = Thread::new do
922
+ begin
923
+ get_logs(args)
924
+ rescue => e
925
+ @log.error("#{__method__}:#{e.inspect} #{$@}")
926
+ ensure
927
+ @stats.gui_run_gather_logs = false
928
+ end
929
+ end
930
+ t[:name] = __method__
931
+ end
932
+
933
+ def get_logs(args)
934
+ @log.debug("#{__method__}:start.")
935
+
936
+ log_path = Config::LOG_PATH
937
+ log_file = "#{log_path}/#{@stats.ap_str}.log"
938
+
939
+ raw_logs = []
940
+ start_time = Time.now
941
+ File.open(log_file){|f|
942
+ f.each_line{|line|
943
+ # hilatency check
944
+ ps = Time.now - start_time
945
+ if ps > 5
946
+ @log.warn("gather_logs process was failed.")
947
+ raise
948
+ end
949
+
950
+ raw_logs << line unless line.chomp == '.'
951
+ }
952
+ }
953
+
954
+ sliced_logs = []
955
+ if raw_logs.size > args[0]
956
+ sliced_logs = raw_logs.slice(-args[0]..-1)
957
+ else
958
+ raw_logs.shift # remove first line(date of log file was created)
959
+ sliced_logs = raw_logs
960
+ end
961
+
962
+ @rttable.logs = sliced_logs
963
+ @log.info("#{__method__} has done.")
964
+ rescue =>e
965
+ @rttable.logs = []
966
+ @log.error("#{e}\n#{$@}")
967
+ ensure
968
+ @stats.gui_run_gather_logs = false
969
+ end
970
+
857
971
  end # module AsyncProcess
858
972
 
859
973
  end # module Roma
@@ -119,6 +119,11 @@ module Roma
119
119
  stored.vn, stored.last, stored.clk, stored.expt, stored.value =
120
120
  @storages[params.hash_name].get_raw(params.vn, params.key, params.digest)
121
121
  stored = nil if stored.vn == nil || Time.now.to_i > stored.expt
122
+ if stored && @stats.wb_command_map.key?("#{cmd}__prev".to_sym)
123
+ Roma::WriteBehindProcess::push(params.hash_name,
124
+ @stats.wb_command_map["#{cmd}__prev".to_sym],
125
+ params.key, stored.value)
126
+ end
122
127
  ctx = CommandContext.new(s, params, stored)
123
128
 
124
129
  ret = instance_exec(ctx, &block)
@@ -137,7 +142,9 @@ module Roma
137
142
 
138
143
  if ret
139
144
  if @stats.wb_command_map.key?(cmd.to_sym)
140
- Roma::WriteBehindProcess::push(ctx.params.hash_name, @stats.wb_command_map[cmd.to_sym], ctx.params.key, ret[4])
145
+ Roma::WriteBehindProcess::push(ctx.params.hash_name,
146
+ @stats.wb_command_map[cmd.to_sym],
147
+ ctx.params.key, ret[4])
141
148
  end
142
149
  redundant(ctx.params.nodes[1..-1], ctx.params.hash_name,
143
150
  ctx.params.key, ctx.params.digest, ret[2],
@@ -223,8 +230,15 @@ module Roma
223
230
  stored.vn, stored.last, stored.clk, stored.expt, stored.value =
224
231
  @storages[params.hash_name].get_raw(params.vn, params.key, params.digest)
225
232
  stored = nil if stored.vn == nil || Time.now.to_i > stored.expt
226
- ctx = CommandContext.new(s, params, stored)
227
233
 
234
+ if stored && @stats.wb_command_map.key?("#{cmd}__prev".to_sym)
235
+ Roma::WriteBehindProcess::push(params.hash_name,
236
+ @stats.wb_command_map["#{cmd}__prev".to_sym],
237
+ params.key, stored.value)
238
+ end
239
+
240
+ ctx = CommandContext.new(s, params, stored)
241
+
228
242
  ret = instance_exec(ctx, &block)
229
243
  if ret.instance_of? Array
230
244
  flg, expt, value, count, msg = ret
@@ -241,7 +255,9 @@ module Roma
241
255
 
242
256
  if ret
243
257
  if @stats.wb_command_map.key?(cmd.to_sym)
244
- Roma::WriteBehindProcess::push(ctx.params.hash_name, @stats.wb_command_map[cmd.to_sym], ctx.params.key, ctx.params.value)
258
+ Roma::WriteBehindProcess::push(ctx.params.hash_name,
259
+ @stats.wb_command_map[cmd.to_sym],
260
+ ctx.params.key, ret[4])
245
261
  end
246
262
  redundant(ctx.params.nodes[1..-1], ctx.params.hash_name,
247
263
  ctx.params.key, ctx.params.digest, ret[2],
@@ -60,6 +60,19 @@ module Roma
60
60
  end
61
61
  end
62
62
 
63
+ def ev_enabled_repetition_in_routing?(s)
64
+ rt = @rttable
65
+ rd = @rttable.sub_nid_rd(@addr)
66
+ rt = Roma::Routing::RoutingTable.new(rd) if rd
67
+
68
+ if s.length == 1
69
+ repetition = rt.check_repetition_in_routing
70
+ send_data("#{repetition}\r\n")
71
+ else
72
+ send_data("CLIENT_ERROR\r\n")
73
+ end
74
+ end
75
+
63
76
  # setroute <vnode-id> <clock> <node-id> ...
64
77
  def ev_setroute(s)
65
78
  if s.length < 4
@@ -275,6 +288,21 @@ module Roma
275
288
  end
276
289
  end
277
290
 
291
+ # get_key_info <key>
292
+ def ev_get_key_info(s)
293
+ if s.length != 2
294
+ return send_data("CLIENT_ERROR number of arguments(0 for 1)\r\n")
295
+ end
296
+
297
+ d = Digest::SHA1.hexdigest(s[1]).hex % @rttable.hbits
298
+ vn = @rttable.get_vnode_id(d)
299
+ nodes = @rttable.search_nodes_for_write(vn)
300
+ send_data(sprintf("d = %s 0x%x\r\n",d,d))
301
+ send_data(sprintf("vn = %s 0x%x\r\n",vn,vn))
302
+ send_data("nodes = #{nodes.inspect}\r\n")
303
+ send_data("END\r\n")
304
+ end
305
+
278
306
  end # module RoutingCommandReceiver
279
307
  end # module Command
280
308
  end # module Roma
@@ -1052,7 +1052,7 @@ module Roma
1052
1052
  return send_data("CLIENT_ERROR storage[#{dn}] != :normal status\r\n")
1053
1053
  end
1054
1054
  if st.set_db_stat(dn, :safecopy_flushing) == false
1055
- return send_data("CLIENT_ERROR storage[#{dn}] != :normal status\r\n")
1055
+ return send_data("CLIENT_ERROR storage[#{dn}] status can't changed\r\n")
1056
1056
  end
1057
1057
  Roma::AsyncProcess::queue.push(Roma::AsyncMessage.new('start_storage_flush_process',[hname, dn]))
1058
1058
  elsif s[2] == 'normal'
@@ -1060,7 +1060,7 @@ module Roma
1060
1060
  return send_data("CLIENT_ERROR storage[#{dn}] != :safecopy_flushed status\r\n")
1061
1061
  end
1062
1062
  if st.set_db_stat(dn, :cachecleaning) == false
1063
- return send_data("CLIENT_ERROR storage[#{dn}] != :safecopy_flushed status\r\n")
1063
+ return send_data("CLIENT_ERROR storage[#{dn}] status can't changed\r\n")
1064
1064
  end
1065
1065
  Roma::AsyncProcess::queue.push(Roma::AsyncMessage.new('start_storage_cachecleaning_process',[hname, dn]))
1066
1066
  else
@@ -1070,6 +1070,68 @@ module Roma
1070
1070
  send_data("PUSHED\r\n")
1071
1071
  end
1072
1072
 
1073
+ # set_gui_run_snapshot [true|false]
1074
+ def ev_set_gui_run_snapshot(s)
1075
+ if s.length != 2
1076
+ return send_data("CLIENT_ERROR number of arguments\n\r")
1077
+ end
1078
+
1079
+ case s[1]
1080
+ when 'true'
1081
+ @stats.gui_run_snapshot = true
1082
+ send_data("STORED\r\n")
1083
+ when 'false'
1084
+ @stats.gui_run_snapshot = false
1085
+ send_data("STORED\r\n")
1086
+ else
1087
+ return send_data("CLIENT_ERROR value must be true or false\r\n")
1088
+ end
1089
+ end
1090
+
1091
+ # set_gui_last_snapshot_date [%Y/%m/%d %H:%M:%S]
1092
+ def ev_set_gui_last_snapshot(s)
1093
+ if s.length != 2
1094
+ return send_data("CLIENT_ERROR number of arguments\n\r")
1095
+ end
1096
+ if s[1] !~ /^\d+\/\d+\/\d+T\d+:\d+:\d+$/
1097
+ return send_data("CLIENT_ERROR format is [%Y/%m/%dT%H:%M:%S]\r\n")
1098
+ end
1099
+ res = broadcast_cmd("rset_gui_last_snapshot #{s[1]}\r\n")
1100
+ @stats.gui_last_snapshot = s[1]
1101
+ res[@stats.ap_str] = "PUSHED"
1102
+ send_data("#{res}\r\n")
1103
+ end
1104
+
1105
+ # rset_gui_last_snapshot(s)
1106
+ def ev_rset_gui_last_snapshot(s)
1107
+ if s.length != 2
1108
+ return send_data("CLIENT_ERROR number of arguments\n\r")
1109
+ end
1110
+ if s[1] !~ /^\d+\/\d+\/\d+T\d+:\d+:\d+$/
1111
+ return send_data("CLIENT_ERROR format is [%Y/%m/%dT%H:%M:%S]\r\n")
1112
+ end
1113
+ @stats.gui_last_snapshot = s[1]
1114
+ send_data("PUSHED\r\n")
1115
+ end
1116
+
1117
+ # set_cleanup_regexp <regexp>
1118
+ def ev_set_cleanup_regexp(s)
1119
+ if s.length != 2
1120
+ return send_data("CLIENT_ERROR number of arguments #{s.length-1} to 1\r\n")
1121
+ end
1122
+
1123
+ # failover check
1124
+ unless @rttable.enabled_failover
1125
+ return send_data("CLIENT_ERROR failover disable now!!\r\n")
1126
+ end
1127
+
1128
+ @storages.each{|hname,st|
1129
+ st.cleanup_regexp = s[1]
1130
+ st.stop_clean_up
1131
+ send_data("STORED\r\n")
1132
+ }
1133
+ end
1134
+
1073
1135
  private
1074
1136
 
1075
1137
  def dcnice(p)