roma 0.8.13 → 0.8.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -90,7 +90,7 @@ module Roma
90
90
 
91
91
  def get_connection(ap)
92
92
  ret = @pool[ap].shift if @pool.key?(ap) && @pool[ap].length > 0
93
- if ret && ret.last_access < Time.now - @expire_time
93
+ if ret && @expire_time != 0 && ret.last_access < Time.now - @expire_time
94
94
  ret.close_connection if ret.connected
95
95
  ret = nil
96
96
  Logging::RLogger.instance.info("EM connection expired at #{ap},remains #{@pool[ap].length}")
@@ -50,7 +50,15 @@ module Roma
50
50
  @@connection_expire_time
51
51
  end
52
52
 
53
- attr_accessor :timeout
53
+ @@timeout = 10
54
+ def self.timeout=(n)
55
+ @@timeout = n
56
+ end
57
+
58
+ def self.timeout
59
+ @@timeout
60
+ end
61
+
54
62
  attr_reader :connected
55
63
  attr_reader :lastcmd
56
64
  attr_reader :last_access
@@ -71,7 +79,6 @@ module Roma
71
79
 
72
80
  @storages = storages
73
81
  @rttable = rttable
74
- @timeout = 10
75
82
  @log = Roma::Logging::RLogger.instance
76
83
  @last_access = Time.now
77
84
  end
@@ -96,7 +103,6 @@ module Roma
96
103
  end
97
104
 
98
105
  def unbind
99
- @log.debug("Roma::Event::Handler.unbind called")
100
106
  @connected=false
101
107
  begin
102
108
  @fiber.resume
@@ -144,7 +150,7 @@ module Roma
144
150
 
145
151
  def dispatcher
146
152
  @stats = Roma::Stats.instance
147
- @log.debug("Roma::Event::Handler.dipatcher called")
153
+ #@log.debug("Roma::Event::Handler.dipatcher called")
148
154
  while(@connected) do
149
155
  @enter_time = nil
150
156
  next unless s=gets
@@ -224,7 +230,7 @@ module Roma
224
230
  else
225
231
  remain = size - @rbuf.size
226
232
  Fiber.yield(remain)
227
- if Time.now.to_i - t > @timeout * mult
233
+ if Time.now.to_i - t > @@timeout * mult
228
234
  @log.warn("#{__FILE__}:#{__LINE__}:#{@addr}:#{@port} read_bytes time out");
229
235
  close_connection
230
236
  return nil
@@ -21,7 +21,7 @@ module Roma
21
21
 
22
22
  def get_connection(ap)
23
23
  ret,last = @pool[ap].shift if @pool.key?(ap) && @pool[ap].length > 0
24
- if ret && last < Time.now - @expire_time
24
+ if ret && @expire_time != 0 && last < Time.now - @expire_time
25
25
  ret.close
26
26
  ret = nil
27
27
  Logging::RLogger.instance.info("connection expired at #{ap},remains #{@pool[ap].length}")
@@ -521,6 +521,8 @@ module Roma
521
521
  @stats.run_storage_clean_up
522
522
  stop_clean_up
523
523
  end
524
+ rescue Exception =>e
525
+ @log.error("#{e}\n#{$@}")
524
526
  end
525
527
 
526
528
  def timer_event_10sec
@@ -539,7 +541,7 @@ module Roma
539
541
  @log.warn("failover disable now!!")
540
542
  else
541
543
  version_check
542
- @rttable.delete_old_trans
544
+ @rttable.delete_old_trans(@stats.routing_trans_timeout)
543
545
  start_sync_routing_process
544
546
  end
545
547
 
@@ -63,6 +63,14 @@ module Roma
63
63
  attr_accessor :latency_data
64
64
  #attr_accessor :latency_denominator
65
65
 
66
+ # for vnode copy parameter
67
+ attr_accessor :spushv_klength_warn
68
+ attr_accessor :spushv_vlength_warn
69
+ attr_accessor :spushv_read_timeout
70
+ attr_accessor :reqpushv_timeout_count
71
+
72
+ attr_accessor :routing_trans_timeout
73
+
66
74
  def initialize
67
75
  @config_path = nil
68
76
  @run_recover = false
@@ -90,8 +98,13 @@ module Roma
90
98
  @wb_command_map = {}
91
99
  @latency_log = false
92
100
  @latency_check_cmd =["get", "set", "delete"]
93
- @latency_check_time_count = nil
101
+ @latency_check_time_count = false
94
102
  @latency_data = Hash.new { |hash,key| hash[key] = {} } #double hash
103
+ @spushv_klength_warn = 1024 # 1kB
104
+ @spushv_vlength_warn = 1024 * 1024 # 1MB
105
+ @spushv_read_timeout = 100
106
+ @reqpushv_timeout_count = 300 # 0.1 * 300 sec
107
+ @routing_trans_timeout = 3600 * 3 # 3hr
95
108
  end
96
109
 
97
110
  def ap_str
@@ -132,6 +145,11 @@ module Roma
132
145
  ret['stats.latency_log'] = @latency_log
133
146
  ret['stats.latency_check_cmd'] = @latency_check_cmd
134
147
  ret['stats.latency_check_time_count'] = @latency_check_time_count
148
+ ret['stats.spushv_klength_warn'] = @spushv_klength_warn
149
+ ret['stats.spushv_vlength_warn'] = @spushv_vlength_warn
150
+ ret['stats.spushv_read_timeout'] = @spushv_read_timeout
151
+ ret['stats.reqpushv_timeout_count'] = @reqpushv_timeout_count
152
+ ret['stats.routing_trans_timeout'] = @routing_trans_timeout
135
153
  ret
136
154
  end
137
155
 
@@ -1,4 +1,5 @@
1
1
  require 'digest/sha1'
2
+ require 'thread'
2
3
 
3
4
  module Roma
4
5
  module Storage
@@ -7,11 +8,11 @@ module Roma
7
8
 
8
9
  class BasicStorage
9
10
 
10
- attr :hdb
11
- attr :hdiv
12
- attr :ext_name
13
-
11
+ attr_reader :hdb
12
+ attr_reader :hdiv
13
+ attr_reader :ext_name
14
14
  attr_reader :error_message
15
+ attr_reader :dbs
15
16
 
16
17
  attr_writer :vn_list
17
18
  attr_writer :storage_path
@@ -23,8 +24,17 @@ module Roma
23
24
  attr_accessor :each_clean_up_sleep
24
25
  attr_accessor :logic_clock_expire
25
26
 
27
+ attr_accessor :do_each_vn_dump
28
+
26
29
  def initialize
30
+ # database handler
27
31
  @hdb = []
32
+ # database cache handler
33
+ @hdbc = []
34
+ # status of a database
35
+ @dbs = []
36
+ @log_fd = nil
37
+
28
38
  @hdiv = Hash.new(0)
29
39
 
30
40
  @ext_name = 'db'
@@ -35,6 +45,10 @@ module Roma
35
45
  @each_vn_dump_sleep_count = 100
36
46
  @each_clean_up_sleep = 0.01
37
47
  @logic_clock_expire = 300
48
+
49
+ @each_cache_lock = Mutex::new
50
+ @each_clean_up_lock = Mutex::new
51
+ @stat_lock = Mutex::new
38
52
  end
39
53
 
40
54
  def get_stat
@@ -46,6 +60,7 @@ module Roma
46
60
  ret['storage.each_vn_dump_sleep_count'] = @each_vn_dump_sleep_count
47
61
  ret['storage.each_clean_up_sleep'] = @each_clean_up_sleep
48
62
  ret['storage.logic_clock_expire'] = @logic_clock_expire
63
+ ret['storage.safecopy_stats'] = @dbs.inspect
49
64
  ret
50
65
  end
51
66
 
@@ -71,27 +86,45 @@ module Roma
71
86
  end
72
87
  protected :create_div_hash
73
88
 
74
- def opendb
75
- create_div_hash
89
+ def mkdir_p(md_path)
76
90
  path = ''
77
- @storage_path.split('/').each{|p|
78
- if p.length==0
91
+ md_path.split('/').each do |p|
92
+ if p.length == 0
79
93
  path = '/'
80
94
  next
81
95
  end
82
96
  path << p
83
97
  Dir::mkdir(path) unless File.exist?(path)
84
98
  path << '/'
85
- }
86
- @divnum.times{ |i|
99
+ end
100
+ end
101
+ protected :mkdir_p
102
+
103
+ def opendb
104
+ create_div_hash
105
+ mkdir_p(@storage_path)
106
+ @divnum.times do |i|
107
+ # open database file
87
108
  @hdb[i] = open_db("#{@storage_path}/#{i}.#{@ext_name}")
88
- }
109
+ # check cache file
110
+ if File.exist?(cache_file_name(i))
111
+ @hdbc[i] = open_db(cache_file_name(i))
112
+ stop_clean_up { @dbs[i] = :safecopy_flushed }
113
+ else
114
+ @dbs[i] = :normal
115
+ @hdbc[i] = nil
116
+ end
117
+ end
118
+ open_log
89
119
  end
90
120
 
91
121
  def closedb
92
122
  stop_clean_up
93
123
  buf = @hdb; @hdb = []
94
- buf.each{ |hdb| close_db(hdb) }
124
+ buf.each{ |h| close_db(h) if h }
125
+ buf = @hdbc; @hdbc = []
126
+ buf.each{ |h| close_db(h) if h }
127
+ close_log
95
128
  end
96
129
 
97
130
 
@@ -117,15 +150,41 @@ module Roma
117
150
  end
118
151
  private :pack_header, :unpack_header, :pack_data, :unpack_data
119
152
 
153
+ def db_get(vn, k)
154
+ n = @hdiv[vn]
155
+ d = @hdb[n].get(k)
156
+ return d if @dbs[n] == :normal
157
+
158
+ c = @hdbc[n].get(k)
159
+ return d unless c # in case of out of :normal status
160
+
161
+ if @dbs[n] == :cachecleaning && d
162
+ # in case of existing value is both @hdb and @hdbc
163
+ vn, lat, clk, expt = unpack_header(d)
164
+ cvn, clat, cclk, cexpt = unpack_header(c)
165
+ return d if cmp_clk(clk, cclk) > 0 # if @hdb newer than @hdbc
166
+ end
167
+ c
168
+ end
169
+
170
+ def db_put(vn, k, v)
171
+ n = @hdiv[vn]
172
+ if @dbs[n] == :safecopy_flushing || @dbs[n] == :safecopy_flushed
173
+ ret = @hdbc[n].put(k, v)
174
+ else
175
+ ret = @hdb[n].put(k, v)
176
+ end
177
+ ret
178
+ end
120
179
 
121
180
  def get_context(vn, k, d)
122
- buf = @hdb[@hdiv[vn]].get(k)
181
+ buf = db_get(vn, k)
123
182
  return nil unless buf
124
183
  unpack_header(buf)
125
184
  end
126
185
 
127
186
  def cas(vn, k, d, clk, expt, v)
128
- buf = @hdb[@hdiv[vn]].get(k)
187
+ buf = db_get(vn ,k)
129
188
  return :not_found unless buf
130
189
  t = Time.now.to_i
131
190
  data = unpack_data(buf)
@@ -133,12 +192,12 @@ module Roma
133
192
  return :exists if clk != data[2]
134
193
  clk = (data[2] + 1) & 0xffffffff
135
194
  ret = [vn, t, clk, expt, v]
136
- return ret if @hdb[@hdiv[vn]].put(k, pack_data(*ret))
195
+ return ret if db_put(vn, k, pack_data(*ret))
137
196
  nil
138
197
  end
139
198
 
140
199
  def rset(vn, k, d, clk, expt, v)
141
- buf = @hdb[@hdiv[vn]].get(k)
200
+ buf = db_get(vn, k)
142
201
  t = Time.now.to_i
143
202
  if buf
144
203
  data = unpack_data(buf)
@@ -149,13 +208,13 @@ module Roma
149
208
  end
150
209
 
151
210
  ret = [vn, t, clk, expt, v]
152
- return ret if @hdb[@hdiv[vn]].put(k, pack_data(*ret))
211
+ return ret if db_put(vn, k, pack_data(*ret))
153
212
  @error_message = "error:put"
154
213
  nil
155
214
  end
156
215
 
157
216
  def set(vn, k, d, expt, v)
158
- buf = @hdb[@hdiv[vn]].get(k)
217
+ buf = db_get(vn, k)
159
218
  clk = 0
160
219
  if buf
161
220
  data = unpack_data(buf)
@@ -163,12 +222,12 @@ module Roma
163
222
  end
164
223
 
165
224
  ret = [vn, Time.now.to_i, clk, expt, v]
166
- return ret if @hdb[@hdiv[vn]].put(k, pack_data(*ret))
225
+ return ret if db_put(vn , k, pack_data(*ret))
167
226
  nil
168
227
  end
169
228
 
170
229
  def add(vn, k, d, expt, v)
171
- buf = @hdb[@hdiv[vn]].get(k)
230
+ buf = db_get(vn, k)
172
231
  clk = 0
173
232
  if buf
174
233
  vn, t, clk, expt2, v2 = unpack_data(buf)
@@ -178,12 +237,12 @@ module Roma
178
237
 
179
238
  # not exist
180
239
  ret = [vn, Time.now.to_i, clk, expt, v]
181
- return ret if @hdb[@hdiv[vn]].put(k, pack_data(*ret))
240
+ return ret if db_put(vn, k, pack_data(*ret))
182
241
  nil
183
242
  end
184
243
 
185
244
  def replace(vn, k, d, expt, v)
186
- buf = @hdb[@hdiv[vn]].get(k)
245
+ buf = db_get(vn, k)
187
246
  return nil unless buf
188
247
 
189
248
  # buf != nil
@@ -192,12 +251,12 @@ module Roma
192
251
  clk = (clk + 1) & 0xffffffff
193
252
 
194
253
  ret = [vn, Time.now.to_i, clk, expt, v]
195
- return ret if @hdb[@hdiv[vn]].put(k, pack_data(*ret))
254
+ return ret if db_put(vn, k, pack_data(*ret))
196
255
  nil
197
256
  end
198
257
 
199
258
  def append(vn, k, d, expt, v)
200
- buf = @hdb[@hdiv[vn]].get(k)
259
+ buf = db_get(vn, k)
201
260
  return nil unless buf
202
261
 
203
262
  # buf != nil
@@ -206,12 +265,12 @@ module Roma
206
265
  clk = (clk + 1) & 0xffffffff
207
266
 
208
267
  ret = [vn, Time.now.to_i, clk, expt, v2 + v]
209
- return ret if @hdb[@hdiv[vn]].put(k, pack_data(*ret))
268
+ return ret if db_put(vn, k, pack_data(*ret))
210
269
  nil
211
270
  end
212
271
 
213
272
  def prepend(vn, k, d, expt, v)
214
- buf = @hdb[@hdiv[vn]].get(k)
273
+ buf = db_get(vn, k)
215
274
  return nil unless buf
216
275
 
217
276
  # buf != nil
@@ -220,12 +279,12 @@ module Roma
220
279
  clk = (clk + 1) & 0xffffffff
221
280
 
222
281
  ret = [vn, Time.now.to_i, clk, expt, v + v2]
223
- return ret if @hdb[@hdiv[vn]].put(k, pack_data(*ret))
282
+ return ret if db_put(vn, k, pack_data(*ret))
224
283
  nil
225
284
  end
226
285
 
227
286
  def get(vn, k, d)
228
- buf = @hdb[@hdiv[vn]].get(k)
287
+ buf = db_get(vn, k)
229
288
  return nil unless buf
230
289
  vn, t, clk, expt, v = unpack_data(buf)
231
290
 
@@ -234,7 +293,7 @@ module Roma
234
293
  end
235
294
 
236
295
  def get_raw(vn, k, d)
237
- buf = @hdb[@hdiv[vn]].get(k)
296
+ buf = db_get(vn, k)
238
297
  return nil unless buf
239
298
 
240
299
  unpack_data(buf)
@@ -249,7 +308,7 @@ module Roma
249
308
  end
250
309
 
251
310
  def rdelete(vn, k, d, clk)
252
- buf = @hdb[@hdiv[vn]].get(k)
311
+ buf = db_get(vn, k)
253
312
  t = Time.now.to_i
254
313
  if buf
255
314
  data = unpack_header(buf)
@@ -264,7 +323,7 @@ module Roma
264
323
  # [ 8..11] logical clock
265
324
  # [12..15] exptime(unix time) => 0
266
325
  ret = [vn, t, clk, 0]
267
- if @hdb[@hdiv[vn]].put(k,pack_header(*ret))
326
+ if db_put(vn, k, pack_header(*ret))
268
327
  return ret
269
328
  else
270
329
  return nil
@@ -272,7 +331,7 @@ module Roma
272
331
  end
273
332
 
274
333
  def delete(vn, k, d)
275
- buf = @hdb[@hdiv[vn]].get(k)
334
+ buf = db_get(vn, k)
276
335
  v = ret = nil
277
336
  clk = 0
278
337
  if buf
@@ -287,7 +346,7 @@ module Roma
287
346
  # [ 8..11] logical clock
288
347
  # [12..15] exptime(unix time) => 0
289
348
  ret = [vn, Time.now.to_i, clk, 0, v]
290
- if @hdb[@hdiv[vn]].put(k,pack_header(*ret[0..-2]))
349
+ if db_put(vn, k, pack_header(*ret[0..-2]))
291
350
  return ret
292
351
  else
293
352
  return nil
@@ -299,7 +358,7 @@ module Roma
299
358
  end
300
359
 
301
360
  def incr(vn, k, d, v)
302
- buf = @hdb[@hdiv[vn]].get(k)
361
+ buf = db_get(vn, k)
303
362
  return nil unless buf
304
363
 
305
364
  # buf != nil
@@ -312,12 +371,12 @@ module Roma
312
371
  v = v & 0xffffffffffffffff
313
372
 
314
373
  ret = [vn, Time.now.to_i, clk, expt2, v.to_s]
315
- return ret if @hdb[@hdiv[vn]].put(k, pack_data(*ret))
374
+ return ret if db_put(vn, k, pack_data(*ret))
316
375
  nil
317
376
  end
318
377
 
319
378
  def decr(vn, k, d, v)
320
- buf = @hdb[@hdiv[vn]].get(k)
379
+ buf = db_get(vn, k)
321
380
  return nil unless buf
322
381
 
323
382
  # buf != nil
@@ -330,19 +389,19 @@ module Roma
330
389
  v = v & 0xffffffffffffffff
331
390
 
332
391
  ret = [vn, Time.now.to_i, clk, expt2, v.to_s]
333
- return ret if @hdb[@hdiv[vn]].put(k, pack_data(*ret))
392
+ return ret if db_put(vn, k, pack_data(*ret))
334
393
  nil
335
394
  end
336
395
 
337
396
  # set expire time
338
397
  def set_expt(vn, k, d, expt)
339
- buf = @hdb[@hdiv[vn]].get(k)
398
+ buf = db_get(vn, k)
340
399
  if buf
341
400
  vn, t, clk, expt2, v = unpack_data(buf)
342
401
  return nil if Time.now.to_i > expt2
343
402
  clk = (clk + 1) & 0xffffffff
344
403
  ret = [vn, Time.now.to_i, clk, expt, v]
345
- return ret if @hdb[@hdiv[vn]].put(k, pack_data(*ret))
404
+ return ret if db_put(vn, k, pack_data(*ret))
346
405
  end
347
406
  nil
348
407
  end
@@ -353,19 +412,14 @@ module Roma
353
412
  res
354
413
  end
355
414
 
356
- def add_vnode(vn)
357
- end
358
-
359
- def del_vnode(vn)
360
- buf = get_vnode_hash(vn)
361
- buf.each_key{ |k| @hdb[@hdiv[vn]].out(k) }
362
- end
363
-
364
415
  def each_clean_up(t, vnhash)
365
416
  @do_clean_up = true
417
+ return unless @each_clean_up_lock.try_lock
366
418
  nt = Time.now.to_i
367
- @hdb.each{ |hdb|
368
- hdb.each{ |k, v|
419
+ @divnum.times do |i|
420
+ next if @dbs[i] != :normal
421
+ hdb = @hdb[i]
422
+ hdb.each do |k, v|
369
423
  return unless @do_clean_up # 1st check
370
424
  vn, last, clk, expt = unpack_header(v)
371
425
  vn_stat = vnhash[vn]
@@ -380,12 +434,22 @@ module Roma
380
434
  end
381
435
  return unless @do_clean_up # 2nd ckeck
382
436
  sleep @each_clean_up_sleep
383
- }
384
- }
437
+ end
438
+ end
439
+ ensure
440
+ @each_clean_up_lock.unlock if @each_clean_up_lock.locked?
385
441
  end
386
442
 
387
- def stop_clean_up
388
- @do_clean_up = false
443
+ def stop_clean_up(&block)
444
+ @do_clean_up = false
445
+ if block
446
+ @each_clean_up_lock.lock
447
+ begin
448
+ block.call
449
+ ensure
450
+ @each_clean_up_lock.unlock
451
+ end
452
+ end
389
453
  end
390
454
 
391
455
  def load(dmp)
@@ -414,19 +478,40 @@ module Roma
414
478
  end
415
479
 
416
480
  def load_stream_dump(vn, last, clk, expt, k, v)
417
- buf = @hdb[@hdiv[vn]].get(k)
481
+ buf = db_get(vn, k)
418
482
  if buf
419
483
  data = unpack_header(buf)
420
484
  if last - data[1] < @logic_clock_expire && cmp_clk(clk,data[2]) <= 0
421
485
  return nil
422
486
  end
423
487
  end
424
-
488
+
489
+ ret = [vn, last, clk, expt, v]
490
+ if expt == 0
491
+ # for the deleted mark
492
+ return ret if db_put(vn, k, pack_header(*ret[0..3]))
493
+ else
494
+ return ret if db_put(vn, k, pack_data(*ret))
495
+ end
496
+ nil
497
+ end
498
+
499
+ def load_stream_dump_for_cachecleaning(vn, last, clk, expt, k, v)
500
+ n = @hdiv[vn]
501
+ buf = @hdb[n].get(k)
502
+ if buf
503
+ data = unpack_header(buf)
504
+ if last - data[1] < @logic_clock_expire && cmp_clk(clk,data[2]) <= 0
505
+ return nil
506
+ end
507
+ end
508
+
425
509
  ret = [vn, last, clk, expt, v]
426
510
  if expt == 0
427
- return ret if @hdb[@hdiv[vn]].put(k, pack_header(*ret[0..3]))
511
+ # for the deleted mark
512
+ return ret if @hdb[n].put(k, pack_header(*ret[0..3]))
428
513
  else
429
- return ret if @hdb[@hdiv[vn]].put(k, pack_data(*ret))
514
+ return ret if @hdb[n].put(k, pack_data(*ret))
430
515
  end
431
516
  nil
432
517
  end
@@ -438,43 +523,68 @@ module Roma
438
523
  Marshal.dump(buf)
439
524
  end
440
525
 
441
- def dump_file(path,except_vnh = nil)
442
- pbuf = ''
443
- path.split('/').each{|p|
444
- pbuf << p
445
- begin
446
- Dir::mkdir(pbuf) unless File.exist?(pbuf)
447
- rescue
526
+ def each_vn_dump(target_vn)
527
+ @do_each_vn_dump = true
528
+ n = @hdiv[target_vn]
529
+ if @dbs[n] == :normal
530
+ # in case of :normal status
531
+ each_unpacked_db(target_vn, @hdb) do |vn, last, clk, expt, k, val|
532
+ return unless @do_each_vn_dump
533
+ yield vn_dump_pack(vn, last, clk, expt, k, val)
448
534
  end
449
- pbuf << '/'
450
- }
451
- @divnum.times{|i|
452
- f = open("#{path}/#{i}.dump","wb")
453
- each_hdb_dump(i,except_vnh){|data| f.write(data) }
454
- f.close
455
- }
456
- open("#{path}/eod","w"){|f|
457
- f.puts Time.now
458
- }
535
+ else
536
+ # in case of out of :normal status
537
+ @each_cache_lock.synchronize do
538
+ each_unpacked_db(target_vn, @hdbc) do |cvn, clast, cclk, cexpt, k, cval|
539
+ return unless @do_each_vn_dump
540
+ data = @hdb[n].get(k)
541
+ if data
542
+ vn, last, clk, expt, val = unpack_data(data)
543
+ #puts "#{k} #{clk} #{cclk} #{cmp_clk(clk, cclk)} #{val}"
544
+ if cmp_clk(clk, cclk) > 0
545
+ yield vn_dump_pack(vn, last, clk, expt, k, val)
546
+ else
547
+ yield vn_dump_pack(cvn, clast, cclk, cexpt, k, cval)
548
+ end
549
+ else
550
+ yield vn_dump_pack(cvn, clast, cclk, cexpt, k, cval)
551
+ end
552
+ end
553
+ end
554
+ each_unpacked_db(target_vn, @hdb) do |vn, last, clk, expt, k, val|
555
+ return unless @do_each_vn_dump
556
+ unless @hdbc[n].get(k)
557
+ yield vn_dump_pack(vn, last, clk, expt, k, val)
558
+ end
559
+ end
560
+ end
561
+ ensure
562
+ @do_each_vn_dump = false
459
563
  end
460
564
 
461
- def each_vn_dump(target_vn)
565
+ def vn_dump_pack(vn, last, clk, expt, k, val)
566
+ if val
567
+ return [vn, last, clk, expt, k.length, k, val.length, val].pack("NNNNNa#{k.length}Na#{val.length}")
568
+ else
569
+ return [vn, last, clk, expt, k.length, k, 0].pack("NNNNNa#{k.length}N")
570
+ end
571
+ end
572
+ private :vn_dump_pack
573
+
574
+ def each_unpacked_db(target_vn, db)
462
575
  count = 0
463
576
  tn = Time.now.to_i
464
- @hdb[@hdiv[target_vn]].each{|k,v|
577
+ db[@hdiv[target_vn]].each do |k,v|
465
578
  vn, last, clk, expt, val = unpack_data(v)
466
579
  if vn != target_vn || (expt != 0 && tn > expt)
467
580
  count += 1
468
581
  sleep @each_vn_dump_sleep if count % @each_vn_dump_sleep_count == 0
469
582
  next
470
583
  end
471
- if val
472
- yield [vn, last, clk, expt, k.length, k, val.length, val].pack("NNNNNa#{k.length}Na#{val.length}")
473
- else
474
- yield [vn, last, clk, expt, k.length, k, 0].pack("NNNNNa#{k.length}N")
475
- end
476
- }
584
+ yield vn, last, clk, expt, k, val
585
+ end
477
586
  end
587
+ private :each_unpacked_db
478
588
 
479
589
  def each_hdb_dump(i,except_vnh = nil)
480
590
  count = 0
@@ -490,6 +600,54 @@ module Roma
490
600
  }
491
601
  end
492
602
 
603
+ # Remove a key for the cache(@hdbc).
604
+ # +dn+:: number of database
605
+ # +key+:: key
606
+ def out_cache(dn, key)
607
+ @hdbc[dn].out(key)
608
+ end
609
+
610
+ # Calls the geven block,
611
+ # passes the cache(@hdbc) element.
612
+ # +dn+:: number of database
613
+ # +keys+:: key list
614
+ def each_cache_by_keys(dn, keys)
615
+ keys.each do |k|
616
+ v = @hdbc[dn].get(k)
617
+ vn, last, clk, expt, val = unpack_data(v)
618
+ yield [vn, last, clk, expt, k, val]
619
+ end
620
+ end
621
+
622
+ # Calls the geven block,
623
+ # passes the cache(@hdbc) element as the spushv command data format.
624
+ # +dn+:: number of database
625
+ # +keys+:: key list
626
+ def each_cache_dump_pack(dn, keys)
627
+ keys.each do |k|
628
+ v = @hdbc[dn].get(k)
629
+ vn, last, clk, expt, val = unpack_data(v)
630
+ yield vn_dump_pack(vn, last, clk, expt, k, val)
631
+ end
632
+ end
633
+
634
+ # Returns a key array in a cache(@hdbc).
635
+ # +dn+:: number of database
636
+ # +kn+:: number of keys which is return value
637
+ def get_keys_in_cache(dn, kn=100)
638
+ return nil if @do_each_vn_dump
639
+ ret = []
640
+ return ret unless @hdbc[dn]
641
+ count = 0
642
+ @each_cache_lock.synchronize do
643
+ @hdbc[dn].each do |k, v|
644
+ ret << k
645
+ break if (count+=1) >= kn
646
+ end
647
+ end
648
+ ret
649
+ end
650
+
493
651
  # Create vnode dump.
494
652
  def get_vnode_hash(vn)
495
653
  buf = {}
@@ -504,6 +662,108 @@ module Roma
504
662
  end
505
663
  private :get_vnode_hash
506
664
 
665
+ def flush_db(dn)
666
+ @hdb[dn].sync
667
+ end
668
+
669
+ def cache_file_name(dn)
670
+ "#{@storage_path}/#{dn}.cache.#{@ext_name}"
671
+ end
672
+
673
+ def set_db_stat(dn, stat)
674
+ @stat_lock.synchronize do
675
+ case @dbs[dn]
676
+ when :normal
677
+ if stat == :safecopy_flushing
678
+ # open cache
679
+ @hdbc[dn] = open_db(cache_file_name(dn))
680
+ stop_clean_up { @dbs[dn] = stat }
681
+ write_log("#{dn} #{stat.to_s}")
682
+ stat
683
+ else
684
+ false
685
+ end
686
+ when :safecopy_flushing
687
+ if stat == :safecopy_flushed
688
+ write_log("#{dn} #{stat.to_s}")
689
+ @dbs[dn] = stat
690
+ else
691
+ false
692
+ end
693
+ when :safecopy_flushed
694
+ if stat == :cachecleaning
695
+ write_log("#{dn} #{stat.to_s}")
696
+ @dbs[dn] = stat
697
+ else
698
+ false
699
+ end
700
+ when :cachecleaning
701
+ if stat == :normal
702
+ write_log("#{dn} #{stat.to_s}")
703
+ @dbs[dn] = stat
704
+ # remove cache
705
+ close_db(@hdbc[dn])
706
+ @hdbc[dn] = nil
707
+ if File.exist?("#{@storage_path}/#{dn}.cache.#{@ext_name}")
708
+ File.unlink("#{@storage_path}/#{dn}.cache.#{@ext_name}")
709
+ end
710
+ stat
711
+ elsif stat == :safecopy_flushing
712
+ write_log("#{dn} #{stat.to_s}")
713
+ @dbs[dn] = stat
714
+ else
715
+ false
716
+ end
717
+ else
718
+ false
719
+ end
720
+ end
721
+ end
722
+
723
+ def get_logfile_list
724
+ l={}
725
+ files=Dir.glob("#{@storage_path}/status.log.*")
726
+ files.each{ |file|
727
+ if /$.+status\.log\.(\d+)$/=~file
728
+ l[$1.to_i]=$&
729
+ end
730
+ }
731
+ # sorted by old order
732
+ l.to_a.sort{|a,b| a[0]<=>b[0]}
733
+ end
734
+
735
+ def open_log
736
+ logs = get_logfile_list
737
+ if logs.length == 0
738
+ @log_name="#{@storage_path}/status.log.1"
739
+ else
740
+ if File::stat("#{@fname}.#{logs.last[0]}").size == 0
741
+ @log_name="#{@fname}.#{logs.last[0]}"
742
+ else
743
+ @log_name="#{@fname}.#{logs.last[0]+1}"
744
+ end
745
+ end
746
+ @log_fd=File.open(@log_name,"a")
747
+ end
748
+
749
+ def write_log(line)
750
+ return unless @log_name
751
+ # log rotation
752
+ if File::stat(@log_name).size > 1000 * 1024
753
+ close_log
754
+ open_log
755
+ end
756
+ t = Time.now
757
+ tstr = "#{t.strftime('%Y-%m-%dT%H:%M:%S')}.#{t.usec}"
758
+ @log_fd.write("#{tstr} #{line}\n")
759
+ @log_fd.flush
760
+ end
761
+
762
+ def close_log
763
+ @log_fd.close if @log_fd
764
+ @log_fd = nil
765
+ end
766
+
507
767
  end # class BasicStorage
508
768
 
509
769
  end # module Storage