roma 0.8.2 → 0.8.10

Sign up to get free protection for your applications and to get access to all the features.
Files changed (114) hide show
  1. data/CHANG +326 -0
  2. data/CHANGELOG +132 -0
  3. data/{README.rdoc → FETCH_HEAD} +0 -0
  4. data/{LICENSE.rdoc → LICENSE} +0 -1
  5. data/README +17 -0
  6. data/Rakefile +33 -18
  7. data/ruby/server/bin/chg_redundancy +10 -0
  8. data/ruby/server/bin/key_access +7 -0
  9. data/ruby/server/bin/key_list +7 -0
  10. data/ruby/server/bin/mkconfig +19 -0
  11. data/{bin → ruby/server/bin}/mkrecent +0 -1
  12. data/{bin → ruby/server/bin}/mkroute +0 -1
  13. data/ruby/server/bin/multi_commander +19 -0
  14. data/ruby/server/bin/recoverlost +10 -0
  15. data/ruby/server/bin/recoverlost_alist +10 -0
  16. data/ruby/server/bin/recoverlost_alist_all +10 -0
  17. data/ruby/server/bin/recoverlost_alist_keys +10 -0
  18. data/{bin/recoverlost → ruby/server/bin/roma_watcher} +1 -2
  19. data/ruby/server/bin/romad +36 -0
  20. data/{bin → ruby/server/bin}/sample_watcher +0 -1
  21. data/{bin → ruby/server/bin}/sample_watcher2 +0 -1
  22. data/{bin/simple_bench → ruby/server/bin/sample_watcher3} +1 -2
  23. data/ruby/server/bin/simple_bench +26 -0
  24. data/{bin → ruby/server/bin}/ssroute +0 -1
  25. data/ruby/server/bin/test-scenario +11 -0
  26. data/{bin → ruby/server/bin}/tribunus +0 -1
  27. data/{lib → ruby/server/lib}/roma/async_process.rb +67 -15
  28. data/{lib → ruby/server/lib}/roma/command/bg_command_receiver.rb +1 -1
  29. data/ruby/server/lib/roma/command/command_definition.rb +422 -0
  30. data/ruby/server/lib/roma/command/mh_command_receiver.rb +127 -0
  31. data/ruby/server/lib/roma/command/receiver.rb +64 -0
  32. data/{lib → ruby/server/lib}/roma/command/rt_command_receiver.rb +6 -1
  33. data/ruby/server/lib/roma/command/sys_command_receiver.rb +609 -0
  34. data/{lib → ruby/server/lib}/roma/command/util_command_receiver.rb +15 -5
  35. data/{lib → ruby/server/lib}/roma/command/vn_command_receiver.rb +12 -4
  36. data/{lib → ruby/server/lib}/roma/command_plugin.rb +0 -0
  37. data/ruby/server/lib/roma/config.rb +84 -0
  38. data/{lib → ruby/server/lib}/roma/event/con_pool.rb +12 -1
  39. data/ruby/server/lib/roma/event/handler.rb +256 -0
  40. data/ruby/server/lib/roma/live_patch-20120302-001.rb +107 -0
  41. data/ruby/server/lib/roma/logging/rlogger.rb +163 -0
  42. data/ruby/server/lib/roma/messaging/con_pool.rb +92 -0
  43. data/{lib → ruby/server/lib}/roma/plugin/plugin_alist.rb +118 -240
  44. data/ruby/server/lib/roma/plugin/plugin_debug.rb +31 -0
  45. data/ruby/server/lib/roma/plugin/plugin_map.rb +177 -0
  46. data/ruby/server/lib/roma/plugin/plugin_mapcount.rb +185 -0
  47. data/{lib/roma/command/st_command_receiver.rb → ruby/server/lib/roma/plugin/plugin_storage.rb} +170 -146
  48. data/ruby/server/lib/roma/plugin/plugin_stub.rb +283 -0
  49. data/{lib → ruby/server/lib}/roma/plugin/plugin_test.rb +0 -0
  50. data/{lib → ruby/server/lib}/roma/romad.rb +221 -94
  51. data/{lib → ruby/server/lib}/roma/routing/cb_rttable.rb +4 -6
  52. data/{lib → ruby/server/lib}/roma/routing/merkle_tree.rb +0 -0
  53. data/ruby/server/lib/roma/routing/routing_data.rb +307 -0
  54. data/{lib → ruby/server/lib}/roma/routing/rttable.rb +4 -0
  55. data/{lib → ruby/server/lib}/roma/stats.rb +19 -3
  56. data/{lib → ruby/server/lib}/roma/storage/basic_storage.rb +25 -26
  57. data/{lib → ruby/server/lib}/roma/storage/dbm_storage.rb +1 -23
  58. data/{lib → ruby/server/lib}/roma/storage/dummy_storage.rb +0 -0
  59. data/{lib → ruby/server/lib}/roma/storage/rh_storage.rb +0 -0
  60. data/{lib → ruby/server/lib}/roma/storage/sqlite3_storage.rb +0 -0
  61. data/{lib → ruby/server/lib}/roma/storage/tc_storage.rb +62 -2
  62. data/ruby/server/lib/roma/tools/chg_redundancy.rb +36 -0
  63. data/ruby/server/lib/roma/tools/key_access.rb +105 -0
  64. data/ruby/server/lib/roma/tools/key_list.rb +94 -0
  65. data/ruby/server/lib/roma/tools/mkconfig.rb +535 -0
  66. data/{lib → ruby/server/lib}/roma/tools/mkrecent.rb +0 -0
  67. data/{lib → ruby/server/lib}/roma/tools/mkroute.rb +0 -0
  68. data/ruby/server/lib/roma/tools/multi_commander.rb +45 -0
  69. data/{lib → ruby/server/lib}/roma/tools/recoverlost.rb +0 -0
  70. data/{lib → ruby/server/lib}/roma/tools/recoverlost_alist.rb +0 -0
  71. data/ruby/server/lib/roma/tools/recoverlost_alist_all.rb +8 -0
  72. data/ruby/server/lib/roma/tools/recoverlost_alist_keys.rb +16 -0
  73. data/ruby/server/lib/roma/tools/recoverlost_lib.rb +349 -0
  74. data/ruby/server/lib/roma/tools/roma_watcher.rb +150 -0
  75. data/ruby/server/lib/roma/tools/roma_watcher_config.yml.example +20 -0
  76. data/{lib → ruby/server/lib}/roma/tools/sample_watcher.rb +3 -1
  77. data/{lib → ruby/server/lib}/roma/tools/sample_watcher2.rb +3 -1
  78. data/ruby/server/lib/roma/tools/sample_watcher3.rb +49 -0
  79. data/{lib → ruby/server/lib}/roma/tools/simple_bench.rb +2 -0
  80. data/ruby/server/lib/roma/tools/simple_bench2.rb +78 -0
  81. data/{lib → ruby/server/lib}/roma/tools/ssroute.rb +0 -0
  82. data/ruby/server/lib/roma/tools/test-scenario.rb +327 -0
  83. data/{lib → ruby/server/lib}/roma/tools/tribunus.rb +0 -0
  84. data/ruby/server/lib/roma/version.rb +4 -0
  85. data/{lib → ruby/server/lib}/roma/write_behind.rb +1 -0
  86. data/ruby/server/test/config4mhash.rb +68 -0
  87. data/ruby/server/test/config4storage_error.rb +69 -0
  88. data/{lib/roma/config.rb → ruby/server/test/config4test.rb} +6 -3
  89. data/{test → ruby/server/test}/rcirb.rb +0 -1
  90. data/{test → ruby/server/test}/roma-test-utils.rb +21 -8
  91. data/{test → ruby/server/test}/run-test.rb +3 -2
  92. data/ruby/server/test/storage_error_storage.rb +37 -0
  93. data/ruby/server/test/t_command_definition.rb +326 -0
  94. data/{test → ruby/server/test}/t_cpdata.rb +9 -3
  95. data/{test → ruby/server/test}/t_listplugin.rb +48 -12
  96. data/ruby/server/test/t_mapcountplugin.rb +231 -0
  97. data/ruby/server/test/t_mapplugin.rb +131 -0
  98. data/ruby/server/test/t_mhash.rb +222 -0
  99. data/ruby/server/test/t_rclient.rb +199 -0
  100. data/{test → ruby/server/test}/t_routing_data.rb +56 -0
  101. data/{test → ruby/server/test}/t_storage.rb +107 -111
  102. data/ruby/server/test/t_storage_error.rb +61 -0
  103. data/ruby/server/test/t_writebehind.rb +374 -0
  104. metadata +150 -82
  105. data/bin/recoverlost_alist +0 -8
  106. data/bin/romad +0 -7
  107. data/lib/roma/command/mh_command_receiver.rb +0 -117
  108. data/lib/roma/command/receiver.rb +0 -287
  109. data/lib/roma/event/handler.rb +0 -159
  110. data/lib/roma/plugin/plugin_debug.rb +0 -19
  111. data/lib/roma/tools/recoverlost_lib.rb +0 -217
  112. data/lib/roma/version.rb +0 -4
  113. data/test/t_rclient.rb +0 -318
  114. data/test/t_writebehind.rb +0 -200
@@ -0,0 +1,107 @@
1
+ #!/usr/bin/env ruby
2
+ # eval begin;load('roma/live_patch-20120302-001.rb');rescue Excption=>e;e.inspect;end
3
+ #
4
+ log = Roma::Logging::RLogger.instance
5
+ log.info("live_patch-20120302-001:loading")
6
+
7
+ Roma::Command::Receiver.class_eval{
8
+ def forward_get_context(nid, k)
9
+ con = get_connection(nid)
10
+ con.send("fget_context #{k}\r\n")
11
+ res = con.gets
12
+ if res == nil
13
+ @rttable.proc_failed(nid)
14
+ @log.error("forward get_context failed:nid=#{nid} key=#{k}")
15
+ return nil
16
+ elsif res == "NOT_EXIST\r\n"
17
+ elsif res.start_with?("ERROR")
18
+ @rttable.proc_succeed(nid)
19
+ con.close_connection
20
+ return nil
21
+ else
22
+ while((buf = con.gets) != "END\r\n")
23
+ res << buf
24
+ end
25
+ res << "END\r\n"
26
+ end
27
+ return_connection(nid, con)
28
+ @rttable.proc_succeed(nid)
29
+ res
30
+ rescue => e
31
+ @rttable.proc_failed(nid) if e.message != "no connection"
32
+ @log.error("#{e.inspect}/#{$@}")
33
+ @log.error("forward get_context failed:nid=#{nid} key=#{k}")
34
+ nil
35
+ end
36
+
37
+ def ev_get_context(s)
38
+ key,hname = s[1].split("\e")
39
+ hname ||= @defhash
40
+ d = Digest::SHA1.hexdigest(key).hex % @rttable.hbits
41
+ vn = @rttable.get_vnode_id(d)
42
+ nodes = @rttable.search_nodes_for_write(vn)
43
+ if nodes[0] != @nid
44
+ @log.warn("forward #{s[0]} key=#{key} vn=#{vn} to #{nodes[0]}")
45
+ res = forward_get_context(nodes[0], s[1])
46
+ if res
47
+ return send_data("#{res}")
48
+ end
49
+ return send_data("SERVER_ERROR Message forward failed.\r\n")
50
+ end
51
+
52
+ unless @storages.key?(hname)
53
+ send_data("SERVER_ERROR #{hname} dose not exists.\r\n")
54
+ return
55
+ end
56
+
57
+ ret = @storages[hname].get_raw(vn, key, d)
58
+ if ret
59
+ send_data("nid = #{@nid}\r\n")
60
+ send_data("vn = #{ret[0]}\r\n")
61
+ send_data("t = #{ret[1]}\r\n")
62
+ send_data("clk = #{ret[2]}\r\n")
63
+ send_data("expt = #{ret[3]}\r\n")
64
+ send_data("v.length = #{ret[4].length}\r\n") if ret[4]
65
+ send_data("END\r\n")
66
+ else
67
+ return send_data("NOT_EXIST\r\n")
68
+ end
69
+ end
70
+
71
+ def ev_fget_context(s)
72
+ key,hname = s[1].split("\e")
73
+ hname ||= @defhash
74
+ d = Digest::SHA1.hexdigest(key).hex % @rttable.hbits
75
+ vn = @rttable.get_vnode_id(d)
76
+ nodes = @rttable.search_nodes_for_write(vn)
77
+ if nodes.include?(@nid) == false
78
+ @log.error("#{s[0]} failed key = #{s[1]} vn = #{vn}")
79
+ return send_data("SERVER_ERROR Routing table is inconsistent.\r\n")
80
+ end
81
+
82
+ unless @storages.key?(hname)
83
+ send_data("SERVER_ERROR #{hname} dose not exists.\r\n")
84
+ return
85
+ end
86
+
87
+ ret = @storages[hname].get_raw(vn, key, d)
88
+ if ret
89
+ send_data("nid = #{@nid}\r\n")
90
+ send_data("vn = #{ret[0]}\r\n")
91
+ send_data("t = #{ret[1]}\r\n")
92
+ send_data("clk = #{ret[2]}\r\n")
93
+ send_data("expt = #{ret[3]}\r\n")
94
+ send_data("v.length = #{ret[4].length}\r\n") if ret[4]
95
+ send_data("END\r\n")
96
+ else
97
+ return send_data("NOT_EXIST\r\n")
98
+ end
99
+ end
100
+ }
101
+
102
+ Roma::Event::Handler.class_eval{
103
+ @@ev_list['get_context'] = :ev_get_context
104
+ @@ev_list['fget_context'] = :ev_fget_context
105
+ }
106
+
107
+ log.info("live_patch-20120302-001:has loaded")
@@ -0,0 +1,163 @@
1
+ # ROMA's logger.
2
+ #
3
+ # rlogger.rb - it is an extension to a standard logger for ROMA
4
+ #
5
+ require 'logger'
6
+
7
+ module Roma
8
+ module Logging
9
+
10
+ class RLogger
11
+ VERSION = '0.0.1'
12
+
13
+ module Severity
14
+ TRACE = -1
15
+ DEBUG = 0
16
+ INFO = 1
17
+ WARN = 2
18
+ ERROR = 3
19
+ FATAL = 4
20
+ UNKNOWN = 5
21
+ end
22
+ include Severity
23
+
24
+ module ExtLogDev
25
+ def extendLogDev()
26
+ if @logdev
27
+ @logdev.extend(ExtShiftAge)
28
+ end
29
+ end
30
+ end
31
+
32
+ module ExtShiftAge
33
+ private
34
+ def shift_log_period(now)
35
+ postfix = previous_period_end(now).strftime("%Y%m%d%H%M")
36
+ age_file = "#{@filename}.#{postfix}"
37
+ if FileTest.exist?(age_file)
38
+ raise RuntimeError.new("'#{age_file}' already exists.")
39
+ end
40
+ @dev.close
41
+ File.rename(@filename, age_file)
42
+ @dev = create_logfile(@filename)
43
+ return true
44
+ end
45
+
46
+ def previous_period_end(now)
47
+ ret = nil
48
+ case @shift_age
49
+ when /^min$/
50
+ t = now - 1 * 60
51
+ ret = Time.mktime(t.year, t.month, t.mday, t.hour, t.min, 59)
52
+ when /^hour$/
53
+ t = now - 1 * 60 * 60
54
+ ret = Time.mktime(t.year, t.month, t.mday, t.hour, 59, 59)
55
+ when /^daily$/
56
+ ret = eod(now - 1 * SiD)
57
+ when /^weekly$/
58
+ ret = eod(now - ((now.wday + 1) * SiD))
59
+ when /^monthly$/
60
+ ret = eod(now - now.mday * SiD)
61
+ else
62
+ ret = now
63
+ end
64
+ return ret
65
+ end
66
+ end
67
+
68
+ module ExtTrace
69
+ private
70
+ SEV_LABEL = %w{DEBUG INFO WARN ERROR FATAL ANY}
71
+
72
+ def format_severity(severity)
73
+ if @level <= RLogger::TRACE and severity <= RLogger::TRACE
74
+ return 'TRACE'
75
+ else
76
+ return SEV_LABEL[severity] || 'ANY'
77
+ end
78
+ end
79
+ end
80
+
81
+ @@singleton_instance = nil
82
+
83
+ def self.create_singleton_instance(logdev, shift_age = 0, shift_size = 1048576)
84
+ @@singleton_instance = RLogger.new(logdev, shift_age, shift_size)
85
+ private_class_method :new, :allocate
86
+ end
87
+
88
+ def self.instance
89
+ @@singleton_instance
90
+ end
91
+
92
+ def initialize(logdev, shift_age = 0, shift_size = 1048576)
93
+ @wrap_logger = Logger.new(logdev, shift_age, shift_size)
94
+ @wrap_logger.extend(ExtTrace)
95
+ @wrap_logger.extend(ExtLogDev)
96
+ @wrap_logger.extendLogDev()
97
+ end
98
+
99
+ def level=(severity)
100
+ @wrap_logger.level = severity
101
+ end
102
+
103
+ def trace?; @wrap_logger.level <= TRACE; end
104
+
105
+ def debug?; @wrap_logger.debug?; end
106
+
107
+ def info?; @wrap_logger.info?; end
108
+
109
+ def warn?; @wrap_logger.warn?; end
110
+
111
+ def error?; @wrap_logger.error?; end
112
+
113
+ def fatal?; @wrap_logger.fatal?; end
114
+
115
+ def trace(progname = nil, &block)
116
+ @wrap_logger.add(TRACE, nil, progname, &block)
117
+ end
118
+
119
+ def debug(progname = nil, &block)
120
+ @wrap_logger.debug(progname, &block)
121
+ end
122
+
123
+ def info(progname = nil, &block)
124
+ @wrap_logger.info(progname, &block)
125
+ end
126
+
127
+ def warn(progname = nil, &block)
128
+ @wrap_logger.warn(progname, &block)
129
+ end
130
+
131
+ def error(progname = nil, &block)
132
+ @wrap_logger.error(progname, &block)
133
+ end
134
+
135
+ def fatal(progname = nil, &block)
136
+ @wrap_logger.fatal(progname, &block)
137
+ end
138
+
139
+ def unknown(progname = nil, &block)
140
+ @wrap_logger.unknow(progname, &block)
141
+ end
142
+
143
+ def close; @wrap_logger.close; end
144
+
145
+ end # class RLogger
146
+
147
+
148
+ class StderrWrapper < IO
149
+
150
+ def initialize
151
+ @log = RLogger.instance
152
+ end
153
+
154
+ def write(s)
155
+ @log.error("STDERR:#{s}")
156
+ end
157
+
158
+ def flush; end
159
+
160
+ end # class StderrWrapper < IO
161
+
162
+ end # module Logging
163
+ end # module Roma
@@ -0,0 +1,92 @@
1
+ require 'thread'
2
+ require 'socket'
3
+ require 'singleton'
4
+
5
+ module Roma
6
+ module Messaging
7
+
8
+ class ConPool
9
+ include Singleton
10
+
11
+ attr_accessor :maxlength
12
+ attr_accessor :expire_time
13
+
14
+ def initialize(maxlength = 10, expire_time = 30)
15
+ @pool = {}
16
+ @maxlength = maxlength
17
+ @expire_time = expire_time
18
+ @lock = Mutex.new
19
+ end
20
+
21
+ def get_connection(ap)
22
+ ret,last = @pool[ap].shift if @pool.key?(ap) && @pool[ap].length > 0
23
+ if ret && last < Time.now - @expire_time
24
+ ret.close
25
+ ret = nil
26
+ Logging::RLogger.instance.info("connection expired at #{ap},remains #{@pool[ap].length}")
27
+ end
28
+ return create_connection(ap) unless ret
29
+ ret
30
+ rescue => e
31
+ Logging::RLogger.instance.error("#{__FILE__}:#{__LINE__}:#{e}")
32
+ nil
33
+ end
34
+
35
+ def return_connection(ap, con)
36
+ if select([con],nil,nil,0.0001)
37
+ con.gets
38
+ con.close
39
+ return
40
+ end
41
+
42
+ if @pool.key?(ap) && @pool[ap].length > 0
43
+ if @pool[ap].length > @maxlength
44
+ con.close
45
+ else
46
+ @pool[ap] << [con, Time.now]
47
+ end
48
+ else
49
+ @pool[ap] = [[con, Time.now]]
50
+ end
51
+ rescue => e
52
+ Logging::RLogger.instance.error("#{__FILE__}:#{__LINE__}:#{e}")
53
+ end
54
+
55
+ def create_connection(ap)
56
+ addr, port = ap.split(/[:_]/)
57
+ TCPSocket.new(addr, port)
58
+ end
59
+
60
+ def delete_connection(ap)
61
+ @pool.delete(ap)
62
+ end
63
+
64
+ def close_all
65
+ @pool.each_key{|ap| close_at(ap) }
66
+ end
67
+
68
+ def close_same_host(ap)
69
+ host,port = ap.split(/[:_]/)
70
+ @pool.each_key{|eap|
71
+ close_at(eap) if eap.split(/[:_]/)[0] == host
72
+ }
73
+ end
74
+
75
+ def close_at(ap)
76
+ return unless @pool.key?(ap)
77
+ @lock.synchronize {
78
+ while(@pool[ap].length > 0)
79
+ begin
80
+ @pool[ap].shift[0].close
81
+ rescue =>e
82
+ Roma::Logging::RLogger.instance.error("#{__FILE__}:#{__LINE__}:#{e}")
83
+ end
84
+ end
85
+ @pool.delete(ap)
86
+ }
87
+ end
88
+
89
+ end # class ConPool
90
+
91
+ end # module Messaging
92
+ end # module Roma
@@ -48,9 +48,12 @@ module Roma
48
48
  expt = 0x7fffffff
49
49
  ret = @storages[hname].set(vn, k, d, expt ,Marshal.dump([[],[]]))
50
50
  @stats.delete_count += 1
51
- Roma::WriteBehindProcess::push(hname, 3, k, ddata)
51
+
52
52
 
53
53
  if ret
54
+ if @stats.wb_command_map.key?(:alist_clear)
55
+ Roma::WriteBehindProcess::push(hname, @stats.wb_command_map[:alist_clear], k, ddata)
56
+ end
54
57
  redundant(nodes[1..-1], hname, k, d, ret[2], expt, ret[4])
55
58
  send_data("CLEARED\r\n")
56
59
  else
@@ -86,9 +89,10 @@ module Roma
86
89
  ret = @storages[hname].set(vn, k, d, expt ,Marshal.dump(v))
87
90
  @stats.delete_count += 1
88
91
 
89
- Roma::WriteBehindProcess::push(hname, 2, k, data)
90
-
91
92
  if ret
93
+ if @stats.wb_command_map.key?(:alist_delete)
94
+ Roma::WriteBehindProcess::push(hname, @stats.wb_command_map[:alist_delete], k, data)
95
+ end
92
96
  redundant(nodes[1..-1], hname, k, d, ret[2], expt, ret[4])
93
97
  send_data("DELETED\r\n")
94
98
  else
@@ -119,9 +123,10 @@ module Roma
119
123
  ret = @storages[hname].set(vn, k, d, expt ,Marshal.dump(v))
120
124
  @stats.delete_count += 1
121
125
 
122
- Roma::WriteBehindProcess::push(hname, 2, k, dret)
123
-
124
126
  if ret
127
+ if @stats.wb_command_map.key?(:alist_delete_at)
128
+ Roma::WriteBehindProcess::push(hname, @stats.wb_command_map[:alist_delete_at], k, dret)
129
+ end
125
130
  redundant(nodes[1..-1], hname, k, d, ret[2], expt, ret[4])
126
131
  send_data("DELETED\r\n")
127
132
  else
@@ -358,6 +363,9 @@ module Roma
358
363
  @stats.write_count += 1
359
364
 
360
365
  if ret
366
+ if @stats.wb_command_map.key?(:alist_insert)
367
+ Roma::WriteBehindProcess::push(hname, @stats.wb_command_map[:alist_insert], k, data)
368
+ end
361
369
  redundant(nodes[1..-1], hname, k, d, ret[2], expt, ret[4])
362
370
  send_data("STORED\r\n")
363
371
  else
@@ -398,6 +406,9 @@ module Roma
398
406
  @stats.write_count += 1
399
407
 
400
408
  if ret
409
+ if @stats.wb_command_map.key?(:alist_sized_insert)
410
+ Roma::WriteBehindProcess::push(hname, @stats.wb_command_map[:alist_sized_insert], k, data)
411
+ end
401
412
  redundant(nodes[1..-1], hname, k, d, ret[2], expt, ret[4])
402
413
  send_data("STORED\r\n")
403
414
  else
@@ -440,9 +451,10 @@ module Roma
440
451
  ret = @storages[hname].set(vn, k, d, expt ,Marshal.dump(v))
441
452
  @stats.write_count += 1
442
453
 
443
- Roma::WriteBehindProcess::push(hname, 1, k, data)
444
-
445
454
  if ret
455
+ if @stats.wb_command_map.key?(:alist_swap_and_insert)
456
+ Roma::WriteBehindProcess::push(hname, @stats.wb_command_map[:alist_swap_and_insert], k, data)
457
+ end
446
458
  redundant(nodes[1..-1], hname, k, d, ret[2], expt, ret[4])
447
459
  send_data("STORED\r\n")
448
460
  else
@@ -487,9 +499,10 @@ module Roma
487
499
  ret = @storages[hname].set(vn, k, d, expt ,Marshal.dump(v))
488
500
  @stats.write_count += 1
489
501
 
490
- Roma::WriteBehindProcess::push(hname, 1, k, data)
491
-
492
502
  if ret
503
+ if @stats.wb_command_map.key?(:alist_swap_and_sized_insert)
504
+ Roma::WriteBehindProcess::push(hname, @stats.wb_command_map[:alist_swap_and_sized_insert], k, data)
505
+ end
493
506
  redundant(nodes[1..-1], hname, k, d, ret[2], expt, ret[4])
494
507
  send_data("STORED\r\n")
495
508
  else
@@ -535,9 +548,10 @@ module Roma
535
548
  ret = @storages[hname].set(vn, k, d, expt ,Marshal.dump(v))
536
549
  @stats.write_count += 1
537
550
 
538
- Roma::WriteBehindProcess::push(hname, 1, k, data)
539
-
540
551
  if ret
552
+ if @stats.wb_command_map.key?(:alist_expired_swap_and_insert)
553
+ Roma::WriteBehindProcess::push(hname, @stats.wb_command_map[:alist_expired_swap_and_insert], k, data)
554
+ end
541
555
  redundant(nodes[1..-1], hname, k, d, ret[2], expt, ret[4])
542
556
  send_data("STORED\r\n")
543
557
  else
@@ -585,9 +599,10 @@ module Roma
585
599
  ret = @storages[hname].set(vn, k, d, expt ,Marshal.dump(v))
586
600
  @stats.write_count += 1
587
601
 
588
- Roma::WriteBehindProcess::push(hname, 1, k, data)
589
-
590
602
  if ret
603
+ if @stats.wb_command_map.key?(:alist_expired_swap_and_sized_insert)
604
+ Roma::WriteBehindProcess::push(hname, @stats.wb_command_map[:alist_expired_swap_and_sized_insert], k, data)
605
+ end
591
606
  redundant(nodes[1..-1], hname, k, d, ret[2], expt, ret[4])
592
607
  send_data("STORED\r\n")
593
608
  else
@@ -831,7 +846,6 @@ module Roma
831
846
  data = read_bytes(s[2].to_i)
832
847
  read_bytes(2)
833
848
  if nodes[0] != @nid
834
- @log.debug("forward to #{nodes[0]}");
835
849
  return forward2(nodes[0], s, data)
836
850
  end
837
851
 
@@ -849,6 +863,9 @@ module Roma
849
863
  @stats.write_count += 1
850
864
 
851
865
  if ret
866
+ if @stats.wb_command_map.key?(:alist_push)
867
+ Roma::WriteBehindProcess::push(hname, @stats.wb_command_map[:alist_push], k, data)
868
+ end
852
869
  redundant(nodes[1..-1], hname, k, d, ret[2], expt, ret[4])
853
870
  send_data("STORED\r\n")
854
871
  else
@@ -889,7 +906,11 @@ module Roma
889
906
  ret = @storages[hname].set(vn, k, d, expt ,Marshal.dump(v))
890
907
  @stats.write_count += 1
891
908
 
909
+
892
910
  if ret
911
+ if @stats.wb_command_map.key?(:alist_sized_push)
912
+ Roma::WriteBehindProcess::push(hname, @stats.wb_command_map[:alist_sized_push], k, data)
913
+ end
893
914
  redundant(nodes[1..-1], hname, k, d, ret[2], expt, ret[4])
894
915
  send_data("STORED\r\n")
895
916
  else
@@ -931,6 +952,9 @@ module Roma
931
952
  @stats.write_count += 1
932
953
 
933
954
  if ret
955
+ if @stats.wb_command_map.key?(:alist_swap_and_push)
956
+ Roma::WriteBehindProcess::push(hname, @stats.wb_command_map[:alist_swap_and_push], k, data)
957
+ end
934
958
  redundant(nodes[1..-1], hname, k, d, ret[2], expt, ret[4])
935
959
  send_data("STORED\r\n")
936
960
  else
@@ -978,6 +1002,9 @@ module Roma
978
1002
  @stats.write_count += 1
979
1003
 
980
1004
  if ret
1005
+ if @stats.wb_command_map.key?(:alist_swap_and_sized_push)
1006
+ Roma::WriteBehindProcess::push(hname, @stats.wb_command_map[:alist_swap_and_sized_push], k, data)
1007
+ end
981
1008
  redundant(nodes[1..-1], hname, k, d, ret[2], expt, ret[4])
982
1009
  send_data("STORED\r\n")
983
1010
  else
@@ -1024,6 +1051,9 @@ module Roma
1024
1051
  @stats.write_count += 1
1025
1052
 
1026
1053
  if ret
1054
+ if @stats.wb_command_map.key?(:alist_expired_swap_and_push)
1055
+ Roma::WriteBehindProcess::push(hname, @stats.wb_command_map[:alist_expired_swap_and_push], k, data)
1056
+ end
1027
1057
  redundant(nodes[1..-1], hname, k, d, ret[2], expt, ret[4])
1028
1058
  send_data("STORED\r\n")
1029
1059
  else
@@ -1075,6 +1105,9 @@ module Roma
1075
1105
  @stats.write_count += 1
1076
1106
 
1077
1107
  if ret
1108
+ if @stats.wb_command_map.key?(:alist_expired_swap_and_sized_push)
1109
+ Roma::WriteBehindProcess::push(hname, @stats.wb_command_map[:alist_expired_swap_and_sized_push], k, data)
1110
+ end
1078
1111
  redundant(nodes[1..-1], hname, k, d, ret[2], expt, ret[4])
1079
1112
  send_data("STORED\r\n")
1080
1113
  else
@@ -1086,6 +1119,44 @@ module Roma
1086
1119
  @log.error("#{e} #{$@}")
1087
1120
  end
1088
1121
 
1122
+ # alist_update_at <key> <index> <bytes>[forward]\r\n
1123
+ # <data block>\r\n
1124
+ #
1125
+ # (STORED|NOT_STORED|NOT_FOUND|SERVER_ERROR <error message>)\r\n
1126
+ def ev_alist_update_at(s)
1127
+ hname, k, d, vn, nodes = calc_hash(s[1])
1128
+ data = read_bytes(s[3].to_i)
1129
+ read_bytes(2)
1130
+ return forward2(nodes[0], s, data) if nodes[0] != @nid
1131
+
1132
+ ddata = @storages[hname].get(vn, k, d)
1133
+ return send_data("NOT_FOUND\r\n") unless ddata
1134
+
1135
+ v = Marshal.load(ddata)
1136
+
1137
+ idx = s[2].to_i
1138
+ return send_data("NOT_FOUND\r\n") if idx < 0 || v[0].length <= idx
1139
+ v[0][idx] = data
1140
+ v[1][idx] = Time.now.to_i
1141
+
1142
+ expt = 0x7fffffff
1143
+ ret = @storages[hname].set(vn, k, d, expt ,Marshal.dump(v))
1144
+ @stats.write_count += 1
1145
+
1146
+ if ret
1147
+ if @stats.wb_command_map.key?(:alist_update_at)
1148
+ Roma::WriteBehindProcess::push(hname, @stats.wb_command_map[:alist_update_at], k, data)
1149
+ end
1150
+ redundant(nodes[1..-1], hname, k, d, ret[2], expt, ret[4])
1151
+ send_data("STORED\r\n")
1152
+ else
1153
+ send_data("NOT_STORED\r\n")
1154
+ end
1155
+ rescue => e
1156
+ msg = "SERVER_ERROR #{e} #{$@}".tr("\r\n"," ")
1157
+ send_data("#{msg}\r\n")
1158
+ @log.error("#{e} #{$@}")
1159
+ end
1089
1160
 
1090
1161
  # alist_shift <key> [forward]\r\n
1091
1162
  #
@@ -1298,6 +1369,8 @@ module Roma
1298
1369
  return send_data("SERVER_ERROR Routing table is inconsistent.\r\n")
1299
1370
  end
1300
1371
 
1372
+ @log.warn("forward #{rs} to #{nid}");
1373
+
1301
1374
  buf = ''
1302
1375
  rs.each{|ss| buf << "#{ss} " }
1303
1376
  buf << "forward\r\n"
@@ -1309,26 +1382,42 @@ module Roma
1309
1382
  con = get_connection(nid)
1310
1383
  con.send(buf)
1311
1384
 
1385
+ buf = con.gets
1386
+ if buf == nil
1387
+ @rttable.proc_failed(nid)
1388
+ @log.error("forward get failed:nid=#{nid} rs=#{rs} #{$@}")
1389
+ return send_data("SERVER_ERROR Message forward failed.\r\n")
1390
+ elsif buf.start_with?("ERROR")
1391
+ @rttable.proc_succeed(nid)
1392
+ con.close_connection
1393
+ @log.error("forward get failed:nid=#{nid} rs=#{rs} #{$@}")
1394
+ return send_data("SERVER_ERROR Message forward failed.\r\n")
1395
+ elsif buf.start_with?("VALUE") == false
1396
+ return_connection(nid, con)
1397
+ @rttable.proc_succeed(nid)
1398
+ return send_data(buf)
1399
+ end
1400
+
1312
1401
  res = ''
1313
- while (buf = con.gets)!="END\r\n"
1402
+ begin
1314
1403
  res << buf
1315
1404
  s = buf.split(/ /)
1316
1405
  if s[0] != 'VALUE'
1406
+ return_connection(nid, con)
1407
+ @rttable.proc_succeed(nid)
1317
1408
  return send_data(buf)
1318
1409
  end
1319
- res << con.read_bytes(s[3].to_i + 2)
1320
- end
1410
+ res << con.read_bytes(s[3].to_i + 2)
1411
+ end while (buf = con.gets)!="END\r\n"
1412
+
1321
1413
  res << "END\r\n"
1322
1414
 
1323
1415
  return_connection(nid, con)
1324
1416
  @rttable.proc_succeed(nid)
1325
- if res
1326
- send_data(res)
1327
- else
1328
- send_data("SERVER_ERROR Message forward failed.\r\n")
1329
- end
1417
+
1418
+ send_data(res)
1330
1419
  rescue => e
1331
- @rttable.proc_failed(nid)
1420
+ @rttable.proc_failed(nid) if e.message != "no connection"
1332
1421
  @log.error("forward get failed:nid=#{nid} rs=#{rs} #{e} #{$@}")
1333
1422
  send_data("SERVER_ERROR Message forward failed.\r\n")
1334
1423
  end
@@ -1339,6 +1428,8 @@ module Roma
1339
1428
  return send_data("SERVER_ERROR Routing table is inconsistent.\r\n")
1340
1429
  end
1341
1430
 
1431
+ @log.warn("forward #{rs} to #{nid}");
1432
+
1342
1433
  buf = ''
1343
1434
  rs.each{|ss| buf << "#{ss} " }
1344
1435
  buf << "forward\r\n"
@@ -1348,225 +1439,12 @@ module Roma
1348
1439
  end
1349
1440
 
1350
1441
  res = send_cmd(nid, buf)
1351
- return send_data("#{res}\r\n") if res
1352
- return send_data("SERVER_ERROR Message forward failed.\r\n")
1442
+ if res == nil || res.start_with?("ERROR")
1443
+ return send_data("SERVER_ERROR Message forward failed.\r\n")
1444
+ end
1445
+ send_data("#{res}\r\n")
1353
1446
  end
1354
1447
 
1355
1448
  end # PluginAshiatoList
1356
1449
  end # CommandPlugin
1357
-
1358
-
1359
- module ClientPlugin
1360
-
1361
- module PluginAshiatoList
1362
-
1363
- def alist_at(key, index)
1364
- ret = sender(:value_list_receiver, key, nil, "alist_at %s #{index}")
1365
- return nil if ret.length == 0
1366
- ret[0]
1367
- end
1368
-
1369
- def alist_clear(key)
1370
- sender(:oneline_receiver, key, nil, "alist_clear %s")
1371
- end
1372
-
1373
- def alist_delete(key, value)
1374
- value_validator(value)
1375
- sender(:oneline_receiver, key, value, "alist_delete %s #{value.length}")
1376
- end
1377
-
1378
- def alist_delete_at(key, index)
1379
- sender(:oneline_receiver, key, nil, "alist_delete_at %s #{index}")
1380
- end
1381
-
1382
- def alist_empty?(key)
1383
- sender(:oneline_receiver, key, nil, "alist_empty? %s")
1384
- end
1385
-
1386
- def alist_first(key)
1387
- ret = sender(:value_list_receiver, key, nil, "alist_first %s")
1388
- return nil if ret.length == 0
1389
- ret[0]
1390
- end
1391
-
1392
- def alist_gets(key, range=nil)
1393
- if range
1394
- ret = sender(:value_list_receiver, key, nil, "alist_gets %s #{range}")
1395
- else
1396
- ret = sender(:value_list_receiver, key, nil, "alist_gets %s")
1397
- end
1398
- return nil if ret.length == 0
1399
- ret[0] = ret[0].to_i
1400
- ret
1401
- end
1402
-
1403
- def alist_gets_with_time(key, range=nil)
1404
- if range
1405
- ret = sender(:value_list_receiver, key, nil, "alist_gets_with_time %s #{range}")
1406
- else
1407
- ret = sender(:value_list_receiver, key, nil, "alist_gets_with_time %s")
1408
- end
1409
- return nil if ret.length == 0
1410
- ret[0] = ret[0].to_i
1411
- ret
1412
- end
1413
-
1414
- def alist_include?(key, value)
1415
- sender(:oneline_receiver, key, value, "alist_include? %s #{value.length}")
1416
- end
1417
-
1418
- def alist_index(key, value)
1419
- value_validator(value)
1420
- ret = sender(:oneline_receiver, key, value, "alist_index %s #{value.length}")
1421
- return ret.to_i if ret =~ /\d+/
1422
- return nil if ret=='nil'
1423
- ret
1424
- end
1425
-
1426
- def alist_insert(key, index, value)
1427
- value_validator(value)
1428
- sender(:oneline_receiver, key, value, "alist_insert %s #{index} #{value.length}")
1429
- end
1430
-
1431
- def alist_sized_insert(key, array_size, value)
1432
- sender(:oneline_receiver, key, value, "alist_sized_insert %s #{array_size} #{value.length}")
1433
- end
1434
-
1435
- def alist_swap_and_insert(key, value)
1436
- sender(:oneline_receiver, key, value, "alist_swap_and_insert %s #{value.length}")
1437
- end
1438
-
1439
- def alist_swap_and_sized_insert(key, array_size, value)
1440
- value_validator(value)
1441
- sender(:oneline_receiver, key, value, "alist_swap_and_sized_insert %s #{array_size} #{value.length}")
1442
- end
1443
-
1444
- def alist_expired_swap_and_insert(key, expt, value)
1445
- value_validator(value)
1446
- sender(:oneline_receiver, key, value,
1447
- "alist_expired_swap_and_insert %s #{expt} #{value.length}")
1448
- end
1449
-
1450
- def alist_expired_swap_and_sized_insert(key, expt, array_size, value)
1451
- value_validator(value)
1452
- sender(:oneline_receiver, key, value,
1453
- "alist_expired_swap_and_sized_insert %s #{expt} #{array_size} #{value.length}")
1454
- end
1455
-
1456
- def alist_join(key, sep, range=nil)
1457
- if range
1458
- ret = sender(:value_list_receiver, key, sep, "alist_join %s #{sep.length} #{range}")
1459
- else
1460
- ret = sender(:value_list_receiver, key, sep, "alist_join %s #{sep.length}")
1461
- end
1462
- return nil if ret.length == 0
1463
- ret[0] = ret[0].to_i
1464
- ret
1465
- end
1466
-
1467
- def alist_join_with_time(key, sep, range=nil)
1468
- if range
1469
- ret = sender(:value_list_receiver, key, sep,
1470
- "alist_join_with_time %s #{sep.length} #{range}")
1471
- else
1472
- ret = sender(:value_list_receiver, key, sep,
1473
- "alist_join_with_time %s #{sep.length}")
1474
- end
1475
- return nil if ret.length == 0
1476
- ret[0] = ret[0].to_i
1477
- ret
1478
- end
1479
-
1480
- def alist_to_json(key, range=nil)
1481
- if range
1482
- ret = sender(:value_list_receiver, key, nil, "alist_to_json %s #{range}")
1483
- else
1484
- ret = sender(:value_list_receiver, key, nil, "alist_to_json %s")
1485
- end
1486
- return nil if ret.length == 0
1487
- ret[0]
1488
- end
1489
-
1490
- def alist_last(key)
1491
- ret = sender(:value_list_receiver, key, nil, "alist_last %s")
1492
- return nil if ret.length == 0
1493
- ret[0]
1494
- end
1495
-
1496
- def alist_length(key)
1497
- ret = sender(:oneline_receiver, key, nil, "alist_length %s")
1498
- return ret.to_i if ret =~ /\d+/
1499
- ret
1500
- end
1501
-
1502
- def alist_pop(key)
1503
- ret = sender(:value_list_receiver, key, nil, "alist_pop %s")
1504
- return nil if ret.length == 0
1505
- ret[0]
1506
- end
1507
-
1508
- def alist_push(key, value)
1509
- value_validator(value)
1510
- sender(:oneline_receiver, key, value, "alist_push %s #{value.length}")
1511
- end
1512
-
1513
- def alist_sized_push(key, array_size, value)
1514
- value_validator(value)
1515
- sender(:oneline_receiver, key, value,
1516
- "alist_sized_push %s #{array_size} #{value.length}")
1517
- end
1518
-
1519
- def alist_swap_and_push(key, value)
1520
- value_validator(value)
1521
- sender(:oneline_receiver, key, value, "alist_swap_and_push %s #{value.length}")
1522
- end
1523
-
1524
- def alist_swap_and_sized_push(key, array_size, value)
1525
- value_validator(value)
1526
- sender(:oneline_receiver, key, value,
1527
- "alist_swap_and_sized_push %s #{array_size} #{value.length}")
1528
- end
1529
-
1530
- def alist_expired_swap_and_push(key, expt, value)
1531
- value_validator(value)
1532
- sender(:oneline_receiver, key, value,
1533
- "alist_expired_swap_and_push %s #{expt} #{value.length}")
1534
- end
1535
-
1536
- def alist_expired_swap_and_sized_push(key, expt, array_size, value)
1537
- value_validator(value)
1538
- sender(:oneline_receiver, key, value,
1539
- "alist_expired_swap_and_sized_push %s #{expt} #{array_size} #{value.length}")
1540
- end
1541
-
1542
- def alist_shift(key)
1543
- ret = sender(:value_list_receiver, key, nil, "alist_shift %s")
1544
- return nil if ret.length == 0
1545
- ret[0]
1546
- end
1547
-
1548
- def alist_to_s(key, range=nil)
1549
- if range
1550
- ret = sender(:value_list_receiver, key, nil, "alist_to_s %s #{range}")
1551
- else
1552
- ret = sender(:value_list_receiver, key, nil, "alist_to_s %s")
1553
- end
1554
- return ret if ret.instance_of?(String)
1555
- return nil if ret.length == 0
1556
- ret[0] = ret[0].to_i
1557
- ret[1] = eval(ret[1])
1558
- ret
1559
- end
1560
-
1561
- private
1562
-
1563
- def value_validator(value)
1564
- if value == nil || !value.instance_of?(String)
1565
- raise "value must be a String object."
1566
- end
1567
- end
1568
-
1569
- end # PluginAshiatoList
1570
- end # ClientPlugin
1571
-
1572
1450
  end # Roma