roma 0.8.2 → 0.8.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (114) hide show
  1. data/CHANG +326 -0
  2. data/CHANGELOG +132 -0
  3. data/{README.rdoc → FETCH_HEAD} +0 -0
  4. data/{LICENSE.rdoc → LICENSE} +0 -1
  5. data/README +17 -0
  6. data/Rakefile +33 -18
  7. data/ruby/server/bin/chg_redundancy +10 -0
  8. data/ruby/server/bin/key_access +7 -0
  9. data/ruby/server/bin/key_list +7 -0
  10. data/ruby/server/bin/mkconfig +19 -0
  11. data/{bin → ruby/server/bin}/mkrecent +0 -1
  12. data/{bin → ruby/server/bin}/mkroute +0 -1
  13. data/ruby/server/bin/multi_commander +19 -0
  14. data/ruby/server/bin/recoverlost +10 -0
  15. data/ruby/server/bin/recoverlost_alist +10 -0
  16. data/ruby/server/bin/recoverlost_alist_all +10 -0
  17. data/ruby/server/bin/recoverlost_alist_keys +10 -0
  18. data/{bin/recoverlost → ruby/server/bin/roma_watcher} +1 -2
  19. data/ruby/server/bin/romad +36 -0
  20. data/{bin → ruby/server/bin}/sample_watcher +0 -1
  21. data/{bin → ruby/server/bin}/sample_watcher2 +0 -1
  22. data/{bin/simple_bench → ruby/server/bin/sample_watcher3} +1 -2
  23. data/ruby/server/bin/simple_bench +26 -0
  24. data/{bin → ruby/server/bin}/ssroute +0 -1
  25. data/ruby/server/bin/test-scenario +11 -0
  26. data/{bin → ruby/server/bin}/tribunus +0 -1
  27. data/{lib → ruby/server/lib}/roma/async_process.rb +67 -15
  28. data/{lib → ruby/server/lib}/roma/command/bg_command_receiver.rb +1 -1
  29. data/ruby/server/lib/roma/command/command_definition.rb +422 -0
  30. data/ruby/server/lib/roma/command/mh_command_receiver.rb +127 -0
  31. data/ruby/server/lib/roma/command/receiver.rb +64 -0
  32. data/{lib → ruby/server/lib}/roma/command/rt_command_receiver.rb +6 -1
  33. data/ruby/server/lib/roma/command/sys_command_receiver.rb +609 -0
  34. data/{lib → ruby/server/lib}/roma/command/util_command_receiver.rb +15 -5
  35. data/{lib → ruby/server/lib}/roma/command/vn_command_receiver.rb +12 -4
  36. data/{lib → ruby/server/lib}/roma/command_plugin.rb +0 -0
  37. data/ruby/server/lib/roma/config.rb +84 -0
  38. data/{lib → ruby/server/lib}/roma/event/con_pool.rb +12 -1
  39. data/ruby/server/lib/roma/event/handler.rb +256 -0
  40. data/ruby/server/lib/roma/live_patch-20120302-001.rb +107 -0
  41. data/ruby/server/lib/roma/logging/rlogger.rb +163 -0
  42. data/ruby/server/lib/roma/messaging/con_pool.rb +92 -0
  43. data/{lib → ruby/server/lib}/roma/plugin/plugin_alist.rb +118 -240
  44. data/ruby/server/lib/roma/plugin/plugin_debug.rb +31 -0
  45. data/ruby/server/lib/roma/plugin/plugin_map.rb +177 -0
  46. data/ruby/server/lib/roma/plugin/plugin_mapcount.rb +185 -0
  47. data/{lib/roma/command/st_command_receiver.rb → ruby/server/lib/roma/plugin/plugin_storage.rb} +170 -146
  48. data/ruby/server/lib/roma/plugin/plugin_stub.rb +283 -0
  49. data/{lib → ruby/server/lib}/roma/plugin/plugin_test.rb +0 -0
  50. data/{lib → ruby/server/lib}/roma/romad.rb +221 -94
  51. data/{lib → ruby/server/lib}/roma/routing/cb_rttable.rb +4 -6
  52. data/{lib → ruby/server/lib}/roma/routing/merkle_tree.rb +0 -0
  53. data/ruby/server/lib/roma/routing/routing_data.rb +307 -0
  54. data/{lib → ruby/server/lib}/roma/routing/rttable.rb +4 -0
  55. data/{lib → ruby/server/lib}/roma/stats.rb +19 -3
  56. data/{lib → ruby/server/lib}/roma/storage/basic_storage.rb +25 -26
  57. data/{lib → ruby/server/lib}/roma/storage/dbm_storage.rb +1 -23
  58. data/{lib → ruby/server/lib}/roma/storage/dummy_storage.rb +0 -0
  59. data/{lib → ruby/server/lib}/roma/storage/rh_storage.rb +0 -0
  60. data/{lib → ruby/server/lib}/roma/storage/sqlite3_storage.rb +0 -0
  61. data/{lib → ruby/server/lib}/roma/storage/tc_storage.rb +62 -2
  62. data/ruby/server/lib/roma/tools/chg_redundancy.rb +36 -0
  63. data/ruby/server/lib/roma/tools/key_access.rb +105 -0
  64. data/ruby/server/lib/roma/tools/key_list.rb +94 -0
  65. data/ruby/server/lib/roma/tools/mkconfig.rb +535 -0
  66. data/{lib → ruby/server/lib}/roma/tools/mkrecent.rb +0 -0
  67. data/{lib → ruby/server/lib}/roma/tools/mkroute.rb +0 -0
  68. data/ruby/server/lib/roma/tools/multi_commander.rb +45 -0
  69. data/{lib → ruby/server/lib}/roma/tools/recoverlost.rb +0 -0
  70. data/{lib → ruby/server/lib}/roma/tools/recoverlost_alist.rb +0 -0
  71. data/ruby/server/lib/roma/tools/recoverlost_alist_all.rb +8 -0
  72. data/ruby/server/lib/roma/tools/recoverlost_alist_keys.rb +16 -0
  73. data/ruby/server/lib/roma/tools/recoverlost_lib.rb +349 -0
  74. data/ruby/server/lib/roma/tools/roma_watcher.rb +150 -0
  75. data/ruby/server/lib/roma/tools/roma_watcher_config.yml.example +20 -0
  76. data/{lib → ruby/server/lib}/roma/tools/sample_watcher.rb +3 -1
  77. data/{lib → ruby/server/lib}/roma/tools/sample_watcher2.rb +3 -1
  78. data/ruby/server/lib/roma/tools/sample_watcher3.rb +49 -0
  79. data/{lib → ruby/server/lib}/roma/tools/simple_bench.rb +2 -0
  80. data/ruby/server/lib/roma/tools/simple_bench2.rb +78 -0
  81. data/{lib → ruby/server/lib}/roma/tools/ssroute.rb +0 -0
  82. data/ruby/server/lib/roma/tools/test-scenario.rb +327 -0
  83. data/{lib → ruby/server/lib}/roma/tools/tribunus.rb +0 -0
  84. data/ruby/server/lib/roma/version.rb +4 -0
  85. data/{lib → ruby/server/lib}/roma/write_behind.rb +1 -0
  86. data/ruby/server/test/config4mhash.rb +68 -0
  87. data/ruby/server/test/config4storage_error.rb +69 -0
  88. data/{lib/roma/config.rb → ruby/server/test/config4test.rb} +6 -3
  89. data/{test → ruby/server/test}/rcirb.rb +0 -1
  90. data/{test → ruby/server/test}/roma-test-utils.rb +21 -8
  91. data/{test → ruby/server/test}/run-test.rb +3 -2
  92. data/ruby/server/test/storage_error_storage.rb +37 -0
  93. data/ruby/server/test/t_command_definition.rb +326 -0
  94. data/{test → ruby/server/test}/t_cpdata.rb +9 -3
  95. data/{test → ruby/server/test}/t_listplugin.rb +48 -12
  96. data/ruby/server/test/t_mapcountplugin.rb +231 -0
  97. data/ruby/server/test/t_mapplugin.rb +131 -0
  98. data/ruby/server/test/t_mhash.rb +222 -0
  99. data/ruby/server/test/t_rclient.rb +199 -0
  100. data/{test → ruby/server/test}/t_routing_data.rb +56 -0
  101. data/{test → ruby/server/test}/t_storage.rb +107 -111
  102. data/ruby/server/test/t_storage_error.rb +61 -0
  103. data/ruby/server/test/t_writebehind.rb +374 -0
  104. metadata +150 -82
  105. data/bin/recoverlost_alist +0 -8
  106. data/bin/romad +0 -7
  107. data/lib/roma/command/mh_command_receiver.rb +0 -117
  108. data/lib/roma/command/receiver.rb +0 -287
  109. data/lib/roma/event/handler.rb +0 -159
  110. data/lib/roma/plugin/plugin_debug.rb +0 -19
  111. data/lib/roma/tools/recoverlost_lib.rb +0 -217
  112. data/lib/roma/version.rb +0 -4
  113. data/test/t_rclient.rb +0 -318
  114. data/test/t_writebehind.rb +0 -200
@@ -0,0 +1,107 @@
1
+ #!/usr/bin/env ruby
2
+ # eval begin;load('roma/live_patch-20120302-001.rb');rescue Excption=>e;e.inspect;end
3
+ #
4
+ log = Roma::Logging::RLogger.instance
5
+ log.info("live_patch-20120302-001:loading")
6
+
7
+ Roma::Command::Receiver.class_eval{
8
+ def forward_get_context(nid, k)
9
+ con = get_connection(nid)
10
+ con.send("fget_context #{k}\r\n")
11
+ res = con.gets
12
+ if res == nil
13
+ @rttable.proc_failed(nid)
14
+ @log.error("forward get_context failed:nid=#{nid} key=#{k}")
15
+ return nil
16
+ elsif res == "NOT_EXIST\r\n"
17
+ elsif res.start_with?("ERROR")
18
+ @rttable.proc_succeed(nid)
19
+ con.close_connection
20
+ return nil
21
+ else
22
+ while((buf = con.gets) != "END\r\n")
23
+ res << buf
24
+ end
25
+ res << "END\r\n"
26
+ end
27
+ return_connection(nid, con)
28
+ @rttable.proc_succeed(nid)
29
+ res
30
+ rescue => e
31
+ @rttable.proc_failed(nid) if e.message != "no connection"
32
+ @log.error("#{e.inspect}/#{$@}")
33
+ @log.error("forward get_context failed:nid=#{nid} key=#{k}")
34
+ nil
35
+ end
36
+
37
+ def ev_get_context(s)
38
+ key,hname = s[1].split("\e")
39
+ hname ||= @defhash
40
+ d = Digest::SHA1.hexdigest(key).hex % @rttable.hbits
41
+ vn = @rttable.get_vnode_id(d)
42
+ nodes = @rttable.search_nodes_for_write(vn)
43
+ if nodes[0] != @nid
44
+ @log.warn("forward #{s[0]} key=#{key} vn=#{vn} to #{nodes[0]}")
45
+ res = forward_get_context(nodes[0], s[1])
46
+ if res
47
+ return send_data("#{res}")
48
+ end
49
+ return send_data("SERVER_ERROR Message forward failed.\r\n")
50
+ end
51
+
52
+ unless @storages.key?(hname)
53
+ send_data("SERVER_ERROR #{hname} dose not exists.\r\n")
54
+ return
55
+ end
56
+
57
+ ret = @storages[hname].get_raw(vn, key, d)
58
+ if ret
59
+ send_data("nid = #{@nid}\r\n")
60
+ send_data("vn = #{ret[0]}\r\n")
61
+ send_data("t = #{ret[1]}\r\n")
62
+ send_data("clk = #{ret[2]}\r\n")
63
+ send_data("expt = #{ret[3]}\r\n")
64
+ send_data("v.length = #{ret[4].length}\r\n") if ret[4]
65
+ send_data("END\r\n")
66
+ else
67
+ return send_data("NOT_EXIST\r\n")
68
+ end
69
+ end
70
+
71
+ def ev_fget_context(s)
72
+ key,hname = s[1].split("\e")
73
+ hname ||= @defhash
74
+ d = Digest::SHA1.hexdigest(key).hex % @rttable.hbits
75
+ vn = @rttable.get_vnode_id(d)
76
+ nodes = @rttable.search_nodes_for_write(vn)
77
+ if nodes.include?(@nid) == false
78
+ @log.error("#{s[0]} failed key = #{s[1]} vn = #{vn}")
79
+ return send_data("SERVER_ERROR Routing table is inconsistent.\r\n")
80
+ end
81
+
82
+ unless @storages.key?(hname)
83
+ send_data("SERVER_ERROR #{hname} dose not exists.\r\n")
84
+ return
85
+ end
86
+
87
+ ret = @storages[hname].get_raw(vn, key, d)
88
+ if ret
89
+ send_data("nid = #{@nid}\r\n")
90
+ send_data("vn = #{ret[0]}\r\n")
91
+ send_data("t = #{ret[1]}\r\n")
92
+ send_data("clk = #{ret[2]}\r\n")
93
+ send_data("expt = #{ret[3]}\r\n")
94
+ send_data("v.length = #{ret[4].length}\r\n") if ret[4]
95
+ send_data("END\r\n")
96
+ else
97
+ return send_data("NOT_EXIST\r\n")
98
+ end
99
+ end
100
+ }
101
+
102
+ Roma::Event::Handler.class_eval{
103
+ @@ev_list['get_context'] = :ev_get_context
104
+ @@ev_list['fget_context'] = :ev_fget_context
105
+ }
106
+
107
+ log.info("live_patch-20120302-001:has loaded")
@@ -0,0 +1,163 @@
1
+ # ROMA's logger.
2
+ #
3
+ # rlogger.rb - it is an extension to a standard logger for ROMA
4
+ #
5
+ require 'logger'
6
+
7
+ module Roma
8
+ module Logging
9
+
10
+ class RLogger
11
+ VERSION = '0.0.1'
12
+
13
+ module Severity
14
+ TRACE = -1
15
+ DEBUG = 0
16
+ INFO = 1
17
+ WARN = 2
18
+ ERROR = 3
19
+ FATAL = 4
20
+ UNKNOWN = 5
21
+ end
22
+ include Severity
23
+
24
+ module ExtLogDev
25
+ def extendLogDev()
26
+ if @logdev
27
+ @logdev.extend(ExtShiftAge)
28
+ end
29
+ end
30
+ end
31
+
32
+ module ExtShiftAge
33
+ private
34
+ def shift_log_period(now)
35
+ postfix = previous_period_end(now).strftime("%Y%m%d%H%M")
36
+ age_file = "#{@filename}.#{postfix}"
37
+ if FileTest.exist?(age_file)
38
+ raise RuntimeError.new("'#{age_file}' already exists.")
39
+ end
40
+ @dev.close
41
+ File.rename(@filename, age_file)
42
+ @dev = create_logfile(@filename)
43
+ return true
44
+ end
45
+
46
+ def previous_period_end(now)
47
+ ret = nil
48
+ case @shift_age
49
+ when /^min$/
50
+ t = now - 1 * 60
51
+ ret = Time.mktime(t.year, t.month, t.mday, t.hour, t.min, 59)
52
+ when /^hour$/
53
+ t = now - 1 * 60 * 60
54
+ ret = Time.mktime(t.year, t.month, t.mday, t.hour, 59, 59)
55
+ when /^daily$/
56
+ ret = eod(now - 1 * SiD)
57
+ when /^weekly$/
58
+ ret = eod(now - ((now.wday + 1) * SiD))
59
+ when /^monthly$/
60
+ ret = eod(now - now.mday * SiD)
61
+ else
62
+ ret = now
63
+ end
64
+ return ret
65
+ end
66
+ end
67
+
68
+ module ExtTrace
69
+ private
70
+ SEV_LABEL = %w{DEBUG INFO WARN ERROR FATAL ANY}
71
+
72
+ def format_severity(severity)
73
+ if @level <= RLogger::TRACE and severity <= RLogger::TRACE
74
+ return 'TRACE'
75
+ else
76
+ return SEV_LABEL[severity] || 'ANY'
77
+ end
78
+ end
79
+ end
80
+
81
+ @@singleton_instance = nil
82
+
83
+ def self.create_singleton_instance(logdev, shift_age = 0, shift_size = 1048576)
84
+ @@singleton_instance = RLogger.new(logdev, shift_age, shift_size)
85
+ private_class_method :new, :allocate
86
+ end
87
+
88
+ def self.instance
89
+ @@singleton_instance
90
+ end
91
+
92
+ def initialize(logdev, shift_age = 0, shift_size = 1048576)
93
+ @wrap_logger = Logger.new(logdev, shift_age, shift_size)
94
+ @wrap_logger.extend(ExtTrace)
95
+ @wrap_logger.extend(ExtLogDev)
96
+ @wrap_logger.extendLogDev()
97
+ end
98
+
99
+ def level=(severity)
100
+ @wrap_logger.level = severity
101
+ end
102
+
103
+ def trace?; @wrap_logger.level <= TRACE; end
104
+
105
+ def debug?; @wrap_logger.debug?; end
106
+
107
+ def info?; @wrap_logger.info?; end
108
+
109
+ def warn?; @wrap_logger.warn?; end
110
+
111
+ def error?; @wrap_logger.error?; end
112
+
113
+ def fatal?; @wrap_logger.fatal?; end
114
+
115
+ def trace(progname = nil, &block)
116
+ @wrap_logger.add(TRACE, nil, progname, &block)
117
+ end
118
+
119
+ def debug(progname = nil, &block)
120
+ @wrap_logger.debug(progname, &block)
121
+ end
122
+
123
+ def info(progname = nil, &block)
124
+ @wrap_logger.info(progname, &block)
125
+ end
126
+
127
+ def warn(progname = nil, &block)
128
+ @wrap_logger.warn(progname, &block)
129
+ end
130
+
131
+ def error(progname = nil, &block)
132
+ @wrap_logger.error(progname, &block)
133
+ end
134
+
135
+ def fatal(progname = nil, &block)
136
+ @wrap_logger.fatal(progname, &block)
137
+ end
138
+
139
+ def unknown(progname = nil, &block)
140
+ @wrap_logger.unknow(progname, &block)
141
+ end
142
+
143
+ def close; @wrap_logger.close; end
144
+
145
+ end # class RLogger
146
+
147
+
148
+ class StderrWrapper < IO
149
+
150
+ def initialize
151
+ @log = RLogger.instance
152
+ end
153
+
154
+ def write(s)
155
+ @log.error("STDERR:#{s}")
156
+ end
157
+
158
+ def flush; end
159
+
160
+ end # class StderrWrapper < IO
161
+
162
+ end # module Logging
163
+ end # module Roma
@@ -0,0 +1,92 @@
1
+ require 'thread'
2
+ require 'socket'
3
+ require 'singleton'
4
+
5
+ module Roma
6
+ module Messaging
7
+
8
+ class ConPool
9
+ include Singleton
10
+
11
+ attr_accessor :maxlength
12
+ attr_accessor :expire_time
13
+
14
+ def initialize(maxlength = 10, expire_time = 30)
15
+ @pool = {}
16
+ @maxlength = maxlength
17
+ @expire_time = expire_time
18
+ @lock = Mutex.new
19
+ end
20
+
21
+ def get_connection(ap)
22
+ ret,last = @pool[ap].shift if @pool.key?(ap) && @pool[ap].length > 0
23
+ if ret && last < Time.now - @expire_time
24
+ ret.close
25
+ ret = nil
26
+ Logging::RLogger.instance.info("connection expired at #{ap},remains #{@pool[ap].length}")
27
+ end
28
+ return create_connection(ap) unless ret
29
+ ret
30
+ rescue => e
31
+ Logging::RLogger.instance.error("#{__FILE__}:#{__LINE__}:#{e}")
32
+ nil
33
+ end
34
+
35
+ def return_connection(ap, con)
36
+ if select([con],nil,nil,0.0001)
37
+ con.gets
38
+ con.close
39
+ return
40
+ end
41
+
42
+ if @pool.key?(ap) && @pool[ap].length > 0
43
+ if @pool[ap].length > @maxlength
44
+ con.close
45
+ else
46
+ @pool[ap] << [con, Time.now]
47
+ end
48
+ else
49
+ @pool[ap] = [[con, Time.now]]
50
+ end
51
+ rescue => e
52
+ Logging::RLogger.instance.error("#{__FILE__}:#{__LINE__}:#{e}")
53
+ end
54
+
55
+ def create_connection(ap)
56
+ addr, port = ap.split(/[:_]/)
57
+ TCPSocket.new(addr, port)
58
+ end
59
+
60
+ def delete_connection(ap)
61
+ @pool.delete(ap)
62
+ end
63
+
64
+ def close_all
65
+ @pool.each_key{|ap| close_at(ap) }
66
+ end
67
+
68
+ def close_same_host(ap)
69
+ host,port = ap.split(/[:_]/)
70
+ @pool.each_key{|eap|
71
+ close_at(eap) if eap.split(/[:_]/)[0] == host
72
+ }
73
+ end
74
+
75
+ def close_at(ap)
76
+ return unless @pool.key?(ap)
77
+ @lock.synchronize {
78
+ while(@pool[ap].length > 0)
79
+ begin
80
+ @pool[ap].shift[0].close
81
+ rescue =>e
82
+ Roma::Logging::RLogger.instance.error("#{__FILE__}:#{__LINE__}:#{e}")
83
+ end
84
+ end
85
+ @pool.delete(ap)
86
+ }
87
+ end
88
+
89
+ end # class ConPool
90
+
91
+ end # module Messaging
92
+ end # module Roma
@@ -48,9 +48,12 @@ module Roma
48
48
  expt = 0x7fffffff
49
49
  ret = @storages[hname].set(vn, k, d, expt ,Marshal.dump([[],[]]))
50
50
  @stats.delete_count += 1
51
- Roma::WriteBehindProcess::push(hname, 3, k, ddata)
51
+
52
52
 
53
53
  if ret
54
+ if @stats.wb_command_map.key?(:alist_clear)
55
+ Roma::WriteBehindProcess::push(hname, @stats.wb_command_map[:alist_clear], k, ddata)
56
+ end
54
57
  redundant(nodes[1..-1], hname, k, d, ret[2], expt, ret[4])
55
58
  send_data("CLEARED\r\n")
56
59
  else
@@ -86,9 +89,10 @@ module Roma
86
89
  ret = @storages[hname].set(vn, k, d, expt ,Marshal.dump(v))
87
90
  @stats.delete_count += 1
88
91
 
89
- Roma::WriteBehindProcess::push(hname, 2, k, data)
90
-
91
92
  if ret
93
+ if @stats.wb_command_map.key?(:alist_delete)
94
+ Roma::WriteBehindProcess::push(hname, @stats.wb_command_map[:alist_delete], k, data)
95
+ end
92
96
  redundant(nodes[1..-1], hname, k, d, ret[2], expt, ret[4])
93
97
  send_data("DELETED\r\n")
94
98
  else
@@ -119,9 +123,10 @@ module Roma
119
123
  ret = @storages[hname].set(vn, k, d, expt ,Marshal.dump(v))
120
124
  @stats.delete_count += 1
121
125
 
122
- Roma::WriteBehindProcess::push(hname, 2, k, dret)
123
-
124
126
  if ret
127
+ if @stats.wb_command_map.key?(:alist_delete_at)
128
+ Roma::WriteBehindProcess::push(hname, @stats.wb_command_map[:alist_delete_at], k, dret)
129
+ end
125
130
  redundant(nodes[1..-1], hname, k, d, ret[2], expt, ret[4])
126
131
  send_data("DELETED\r\n")
127
132
  else
@@ -358,6 +363,9 @@ module Roma
358
363
  @stats.write_count += 1
359
364
 
360
365
  if ret
366
+ if @stats.wb_command_map.key?(:alist_insert)
367
+ Roma::WriteBehindProcess::push(hname, @stats.wb_command_map[:alist_insert], k, data)
368
+ end
361
369
  redundant(nodes[1..-1], hname, k, d, ret[2], expt, ret[4])
362
370
  send_data("STORED\r\n")
363
371
  else
@@ -398,6 +406,9 @@ module Roma
398
406
  @stats.write_count += 1
399
407
 
400
408
  if ret
409
+ if @stats.wb_command_map.key?(:alist_sized_insert)
410
+ Roma::WriteBehindProcess::push(hname, @stats.wb_command_map[:alist_sized_insert], k, data)
411
+ end
401
412
  redundant(nodes[1..-1], hname, k, d, ret[2], expt, ret[4])
402
413
  send_data("STORED\r\n")
403
414
  else
@@ -440,9 +451,10 @@ module Roma
440
451
  ret = @storages[hname].set(vn, k, d, expt ,Marshal.dump(v))
441
452
  @stats.write_count += 1
442
453
 
443
- Roma::WriteBehindProcess::push(hname, 1, k, data)
444
-
445
454
  if ret
455
+ if @stats.wb_command_map.key?(:alist_swap_and_insert)
456
+ Roma::WriteBehindProcess::push(hname, @stats.wb_command_map[:alist_swap_and_insert], k, data)
457
+ end
446
458
  redundant(nodes[1..-1], hname, k, d, ret[2], expt, ret[4])
447
459
  send_data("STORED\r\n")
448
460
  else
@@ -487,9 +499,10 @@ module Roma
487
499
  ret = @storages[hname].set(vn, k, d, expt ,Marshal.dump(v))
488
500
  @stats.write_count += 1
489
501
 
490
- Roma::WriteBehindProcess::push(hname, 1, k, data)
491
-
492
502
  if ret
503
+ if @stats.wb_command_map.key?(:alist_swap_and_sized_insert)
504
+ Roma::WriteBehindProcess::push(hname, @stats.wb_command_map[:alist_swap_and_sized_insert], k, data)
505
+ end
493
506
  redundant(nodes[1..-1], hname, k, d, ret[2], expt, ret[4])
494
507
  send_data("STORED\r\n")
495
508
  else
@@ -535,9 +548,10 @@ module Roma
535
548
  ret = @storages[hname].set(vn, k, d, expt ,Marshal.dump(v))
536
549
  @stats.write_count += 1
537
550
 
538
- Roma::WriteBehindProcess::push(hname, 1, k, data)
539
-
540
551
  if ret
552
+ if @stats.wb_command_map.key?(:alist_expired_swap_and_insert)
553
+ Roma::WriteBehindProcess::push(hname, @stats.wb_command_map[:alist_expired_swap_and_insert], k, data)
554
+ end
541
555
  redundant(nodes[1..-1], hname, k, d, ret[2], expt, ret[4])
542
556
  send_data("STORED\r\n")
543
557
  else
@@ -585,9 +599,10 @@ module Roma
585
599
  ret = @storages[hname].set(vn, k, d, expt ,Marshal.dump(v))
586
600
  @stats.write_count += 1
587
601
 
588
- Roma::WriteBehindProcess::push(hname, 1, k, data)
589
-
590
602
  if ret
603
+ if @stats.wb_command_map.key?(:alist_expired_swap_and_sized_insert)
604
+ Roma::WriteBehindProcess::push(hname, @stats.wb_command_map[:alist_expired_swap_and_sized_insert], k, data)
605
+ end
591
606
  redundant(nodes[1..-1], hname, k, d, ret[2], expt, ret[4])
592
607
  send_data("STORED\r\n")
593
608
  else
@@ -831,7 +846,6 @@ module Roma
831
846
  data = read_bytes(s[2].to_i)
832
847
  read_bytes(2)
833
848
  if nodes[0] != @nid
834
- @log.debug("forward to #{nodes[0]}");
835
849
  return forward2(nodes[0], s, data)
836
850
  end
837
851
 
@@ -849,6 +863,9 @@ module Roma
849
863
  @stats.write_count += 1
850
864
 
851
865
  if ret
866
+ if @stats.wb_command_map.key?(:alist_push)
867
+ Roma::WriteBehindProcess::push(hname, @stats.wb_command_map[:alist_push], k, data)
868
+ end
852
869
  redundant(nodes[1..-1], hname, k, d, ret[2], expt, ret[4])
853
870
  send_data("STORED\r\n")
854
871
  else
@@ -889,7 +906,11 @@ module Roma
889
906
  ret = @storages[hname].set(vn, k, d, expt ,Marshal.dump(v))
890
907
  @stats.write_count += 1
891
908
 
909
+
892
910
  if ret
911
+ if @stats.wb_command_map.key?(:alist_sized_push)
912
+ Roma::WriteBehindProcess::push(hname, @stats.wb_command_map[:alist_sized_push], k, data)
913
+ end
893
914
  redundant(nodes[1..-1], hname, k, d, ret[2], expt, ret[4])
894
915
  send_data("STORED\r\n")
895
916
  else
@@ -931,6 +952,9 @@ module Roma
931
952
  @stats.write_count += 1
932
953
 
933
954
  if ret
955
+ if @stats.wb_command_map.key?(:alist_swap_and_push)
956
+ Roma::WriteBehindProcess::push(hname, @stats.wb_command_map[:alist_swap_and_push], k, data)
957
+ end
934
958
  redundant(nodes[1..-1], hname, k, d, ret[2], expt, ret[4])
935
959
  send_data("STORED\r\n")
936
960
  else
@@ -978,6 +1002,9 @@ module Roma
978
1002
  @stats.write_count += 1
979
1003
 
980
1004
  if ret
1005
+ if @stats.wb_command_map.key?(:alist_swap_and_sized_push)
1006
+ Roma::WriteBehindProcess::push(hname, @stats.wb_command_map[:alist_swap_and_sized_push], k, data)
1007
+ end
981
1008
  redundant(nodes[1..-1], hname, k, d, ret[2], expt, ret[4])
982
1009
  send_data("STORED\r\n")
983
1010
  else
@@ -1024,6 +1051,9 @@ module Roma
1024
1051
  @stats.write_count += 1
1025
1052
 
1026
1053
  if ret
1054
+ if @stats.wb_command_map.key?(:alist_expired_swap_and_push)
1055
+ Roma::WriteBehindProcess::push(hname, @stats.wb_command_map[:alist_expired_swap_and_push], k, data)
1056
+ end
1027
1057
  redundant(nodes[1..-1], hname, k, d, ret[2], expt, ret[4])
1028
1058
  send_data("STORED\r\n")
1029
1059
  else
@@ -1075,6 +1105,9 @@ module Roma
1075
1105
  @stats.write_count += 1
1076
1106
 
1077
1107
  if ret
1108
+ if @stats.wb_command_map.key?(:alist_expired_swap_and_sized_push)
1109
+ Roma::WriteBehindProcess::push(hname, @stats.wb_command_map[:alist_expired_swap_and_sized_push], k, data)
1110
+ end
1078
1111
  redundant(nodes[1..-1], hname, k, d, ret[2], expt, ret[4])
1079
1112
  send_data("STORED\r\n")
1080
1113
  else
@@ -1086,6 +1119,44 @@ module Roma
1086
1119
  @log.error("#{e} #{$@}")
1087
1120
  end
1088
1121
 
1122
+ # alist_update_at <key> <index> <bytes>[forward]\r\n
1123
+ # <data block>\r\n
1124
+ #
1125
+ # (STORED|NOT_STORED|NOT_FOUND|SERVER_ERROR <error message>)\r\n
1126
+ def ev_alist_update_at(s)
1127
+ hname, k, d, vn, nodes = calc_hash(s[1])
1128
+ data = read_bytes(s[3].to_i)
1129
+ read_bytes(2)
1130
+ return forward2(nodes[0], s, data) if nodes[0] != @nid
1131
+
1132
+ ddata = @storages[hname].get(vn, k, d)
1133
+ return send_data("NOT_FOUND\r\n") unless ddata
1134
+
1135
+ v = Marshal.load(ddata)
1136
+
1137
+ idx = s[2].to_i
1138
+ return send_data("NOT_FOUND\r\n") if idx < 0 || v[0].length <= idx
1139
+ v[0][idx] = data
1140
+ v[1][idx] = Time.now.to_i
1141
+
1142
+ expt = 0x7fffffff
1143
+ ret = @storages[hname].set(vn, k, d, expt ,Marshal.dump(v))
1144
+ @stats.write_count += 1
1145
+
1146
+ if ret
1147
+ if @stats.wb_command_map.key?(:alist_update_at)
1148
+ Roma::WriteBehindProcess::push(hname, @stats.wb_command_map[:alist_update_at], k, data)
1149
+ end
1150
+ redundant(nodes[1..-1], hname, k, d, ret[2], expt, ret[4])
1151
+ send_data("STORED\r\n")
1152
+ else
1153
+ send_data("NOT_STORED\r\n")
1154
+ end
1155
+ rescue => e
1156
+ msg = "SERVER_ERROR #{e} #{$@}".tr("\r\n"," ")
1157
+ send_data("#{msg}\r\n")
1158
+ @log.error("#{e} #{$@}")
1159
+ end
1089
1160
 
1090
1161
  # alist_shift <key> [forward]\r\n
1091
1162
  #
@@ -1298,6 +1369,8 @@ module Roma
1298
1369
  return send_data("SERVER_ERROR Routing table is inconsistent.\r\n")
1299
1370
  end
1300
1371
 
1372
+ @log.warn("forward #{rs} to #{nid}");
1373
+
1301
1374
  buf = ''
1302
1375
  rs.each{|ss| buf << "#{ss} " }
1303
1376
  buf << "forward\r\n"
@@ -1309,26 +1382,42 @@ module Roma
1309
1382
  con = get_connection(nid)
1310
1383
  con.send(buf)
1311
1384
 
1385
+ buf = con.gets
1386
+ if buf == nil
1387
+ @rttable.proc_failed(nid)
1388
+ @log.error("forward get failed:nid=#{nid} rs=#{rs} #{$@}")
1389
+ return send_data("SERVER_ERROR Message forward failed.\r\n")
1390
+ elsif buf.start_with?("ERROR")
1391
+ @rttable.proc_succeed(nid)
1392
+ con.close_connection
1393
+ @log.error("forward get failed:nid=#{nid} rs=#{rs} #{$@}")
1394
+ return send_data("SERVER_ERROR Message forward failed.\r\n")
1395
+ elsif buf.start_with?("VALUE") == false
1396
+ return_connection(nid, con)
1397
+ @rttable.proc_succeed(nid)
1398
+ return send_data(buf)
1399
+ end
1400
+
1312
1401
  res = ''
1313
- while (buf = con.gets)!="END\r\n"
1402
+ begin
1314
1403
  res << buf
1315
1404
  s = buf.split(/ /)
1316
1405
  if s[0] != 'VALUE'
1406
+ return_connection(nid, con)
1407
+ @rttable.proc_succeed(nid)
1317
1408
  return send_data(buf)
1318
1409
  end
1319
- res << con.read_bytes(s[3].to_i + 2)
1320
- end
1410
+ res << con.read_bytes(s[3].to_i + 2)
1411
+ end while (buf = con.gets)!="END\r\n"
1412
+
1321
1413
  res << "END\r\n"
1322
1414
 
1323
1415
  return_connection(nid, con)
1324
1416
  @rttable.proc_succeed(nid)
1325
- if res
1326
- send_data(res)
1327
- else
1328
- send_data("SERVER_ERROR Message forward failed.\r\n")
1329
- end
1417
+
1418
+ send_data(res)
1330
1419
  rescue => e
1331
- @rttable.proc_failed(nid)
1420
+ @rttable.proc_failed(nid) if e.message != "no connection"
1332
1421
  @log.error("forward get failed:nid=#{nid} rs=#{rs} #{e} #{$@}")
1333
1422
  send_data("SERVER_ERROR Message forward failed.\r\n")
1334
1423
  end
@@ -1339,6 +1428,8 @@ module Roma
1339
1428
  return send_data("SERVER_ERROR Routing table is inconsistent.\r\n")
1340
1429
  end
1341
1430
 
1431
+ @log.warn("forward #{rs} to #{nid}");
1432
+
1342
1433
  buf = ''
1343
1434
  rs.each{|ss| buf << "#{ss} " }
1344
1435
  buf << "forward\r\n"
@@ -1348,225 +1439,12 @@ module Roma
1348
1439
  end
1349
1440
 
1350
1441
  res = send_cmd(nid, buf)
1351
- return send_data("#{res}\r\n") if res
1352
- return send_data("SERVER_ERROR Message forward failed.\r\n")
1442
+ if res == nil || res.start_with?("ERROR")
1443
+ return send_data("SERVER_ERROR Message forward failed.\r\n")
1444
+ end
1445
+ send_data("#{res}\r\n")
1353
1446
  end
1354
1447
 
1355
1448
  end # PluginAshiatoList
1356
1449
  end # CommandPlugin
1357
-
1358
-
1359
- module ClientPlugin
1360
-
1361
- module PluginAshiatoList
1362
-
1363
- def alist_at(key, index)
1364
- ret = sender(:value_list_receiver, key, nil, "alist_at %s #{index}")
1365
- return nil if ret.length == 0
1366
- ret[0]
1367
- end
1368
-
1369
- def alist_clear(key)
1370
- sender(:oneline_receiver, key, nil, "alist_clear %s")
1371
- end
1372
-
1373
- def alist_delete(key, value)
1374
- value_validator(value)
1375
- sender(:oneline_receiver, key, value, "alist_delete %s #{value.length}")
1376
- end
1377
-
1378
- def alist_delete_at(key, index)
1379
- sender(:oneline_receiver, key, nil, "alist_delete_at %s #{index}")
1380
- end
1381
-
1382
- def alist_empty?(key)
1383
- sender(:oneline_receiver, key, nil, "alist_empty? %s")
1384
- end
1385
-
1386
- def alist_first(key)
1387
- ret = sender(:value_list_receiver, key, nil, "alist_first %s")
1388
- return nil if ret.length == 0
1389
- ret[0]
1390
- end
1391
-
1392
- def alist_gets(key, range=nil)
1393
- if range
1394
- ret = sender(:value_list_receiver, key, nil, "alist_gets %s #{range}")
1395
- else
1396
- ret = sender(:value_list_receiver, key, nil, "alist_gets %s")
1397
- end
1398
- return nil if ret.length == 0
1399
- ret[0] = ret[0].to_i
1400
- ret
1401
- end
1402
-
1403
- def alist_gets_with_time(key, range=nil)
1404
- if range
1405
- ret = sender(:value_list_receiver, key, nil, "alist_gets_with_time %s #{range}")
1406
- else
1407
- ret = sender(:value_list_receiver, key, nil, "alist_gets_with_time %s")
1408
- end
1409
- return nil if ret.length == 0
1410
- ret[0] = ret[0].to_i
1411
- ret
1412
- end
1413
-
1414
- def alist_include?(key, value)
1415
- sender(:oneline_receiver, key, value, "alist_include? %s #{value.length}")
1416
- end
1417
-
1418
- def alist_index(key, value)
1419
- value_validator(value)
1420
- ret = sender(:oneline_receiver, key, value, "alist_index %s #{value.length}")
1421
- return ret.to_i if ret =~ /\d+/
1422
- return nil if ret=='nil'
1423
- ret
1424
- end
1425
-
1426
- def alist_insert(key, index, value)
1427
- value_validator(value)
1428
- sender(:oneline_receiver, key, value, "alist_insert %s #{index} #{value.length}")
1429
- end
1430
-
1431
- def alist_sized_insert(key, array_size, value)
1432
- sender(:oneline_receiver, key, value, "alist_sized_insert %s #{array_size} #{value.length}")
1433
- end
1434
-
1435
- def alist_swap_and_insert(key, value)
1436
- sender(:oneline_receiver, key, value, "alist_swap_and_insert %s #{value.length}")
1437
- end
1438
-
1439
- def alist_swap_and_sized_insert(key, array_size, value)
1440
- value_validator(value)
1441
- sender(:oneline_receiver, key, value, "alist_swap_and_sized_insert %s #{array_size} #{value.length}")
1442
- end
1443
-
1444
- def alist_expired_swap_and_insert(key, expt, value)
1445
- value_validator(value)
1446
- sender(:oneline_receiver, key, value,
1447
- "alist_expired_swap_and_insert %s #{expt} #{value.length}")
1448
- end
1449
-
1450
- def alist_expired_swap_and_sized_insert(key, expt, array_size, value)
1451
- value_validator(value)
1452
- sender(:oneline_receiver, key, value,
1453
- "alist_expired_swap_and_sized_insert %s #{expt} #{array_size} #{value.length}")
1454
- end
1455
-
1456
- def alist_join(key, sep, range=nil)
1457
- if range
1458
- ret = sender(:value_list_receiver, key, sep, "alist_join %s #{sep.length} #{range}")
1459
- else
1460
- ret = sender(:value_list_receiver, key, sep, "alist_join %s #{sep.length}")
1461
- end
1462
- return nil if ret.length == 0
1463
- ret[0] = ret[0].to_i
1464
- ret
1465
- end
1466
-
1467
- def alist_join_with_time(key, sep, range=nil)
1468
- if range
1469
- ret = sender(:value_list_receiver, key, sep,
1470
- "alist_join_with_time %s #{sep.length} #{range}")
1471
- else
1472
- ret = sender(:value_list_receiver, key, sep,
1473
- "alist_join_with_time %s #{sep.length}")
1474
- end
1475
- return nil if ret.length == 0
1476
- ret[0] = ret[0].to_i
1477
- ret
1478
- end
1479
-
1480
- def alist_to_json(key, range=nil)
1481
- if range
1482
- ret = sender(:value_list_receiver, key, nil, "alist_to_json %s #{range}")
1483
- else
1484
- ret = sender(:value_list_receiver, key, nil, "alist_to_json %s")
1485
- end
1486
- return nil if ret.length == 0
1487
- ret[0]
1488
- end
1489
-
1490
- def alist_last(key)
1491
- ret = sender(:value_list_receiver, key, nil, "alist_last %s")
1492
- return nil if ret.length == 0
1493
- ret[0]
1494
- end
1495
-
1496
- def alist_length(key)
1497
- ret = sender(:oneline_receiver, key, nil, "alist_length %s")
1498
- return ret.to_i if ret =~ /\d+/
1499
- ret
1500
- end
1501
-
1502
- def alist_pop(key)
1503
- ret = sender(:value_list_receiver, key, nil, "alist_pop %s")
1504
- return nil if ret.length == 0
1505
- ret[0]
1506
- end
1507
-
1508
- def alist_push(key, value)
1509
- value_validator(value)
1510
- sender(:oneline_receiver, key, value, "alist_push %s #{value.length}")
1511
- end
1512
-
1513
- def alist_sized_push(key, array_size, value)
1514
- value_validator(value)
1515
- sender(:oneline_receiver, key, value,
1516
- "alist_sized_push %s #{array_size} #{value.length}")
1517
- end
1518
-
1519
- def alist_swap_and_push(key, value)
1520
- value_validator(value)
1521
- sender(:oneline_receiver, key, value, "alist_swap_and_push %s #{value.length}")
1522
- end
1523
-
1524
- def alist_swap_and_sized_push(key, array_size, value)
1525
- value_validator(value)
1526
- sender(:oneline_receiver, key, value,
1527
- "alist_swap_and_sized_push %s #{array_size} #{value.length}")
1528
- end
1529
-
1530
- def alist_expired_swap_and_push(key, expt, value)
1531
- value_validator(value)
1532
- sender(:oneline_receiver, key, value,
1533
- "alist_expired_swap_and_push %s #{expt} #{value.length}")
1534
- end
1535
-
1536
- def alist_expired_swap_and_sized_push(key, expt, array_size, value)
1537
- value_validator(value)
1538
- sender(:oneline_receiver, key, value,
1539
- "alist_expired_swap_and_sized_push %s #{expt} #{array_size} #{value.length}")
1540
- end
1541
-
1542
- def alist_shift(key)
1543
- ret = sender(:value_list_receiver, key, nil, "alist_shift %s")
1544
- return nil if ret.length == 0
1545
- ret[0]
1546
- end
1547
-
1548
- def alist_to_s(key, range=nil)
1549
- if range
1550
- ret = sender(:value_list_receiver, key, nil, "alist_to_s %s #{range}")
1551
- else
1552
- ret = sender(:value_list_receiver, key, nil, "alist_to_s %s")
1553
- end
1554
- return ret if ret.instance_of?(String)
1555
- return nil if ret.length == 0
1556
- ret[0] = ret[0].to_i
1557
- ret[1] = eval(ret[1])
1558
- ret
1559
- end
1560
-
1561
- private
1562
-
1563
- def value_validator(value)
1564
- if value == nil || !value.instance_of?(String)
1565
- raise "value must be a String object."
1566
- end
1567
- end
1568
-
1569
- end # PluginAshiatoList
1570
- end # ClientPlugin
1571
-
1572
1450
  end # Roma