roma-storage 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/README.md ADDED
@@ -0,0 +1,43 @@
1
+ # Roma::Storage
2
+
3
+ Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/roma/storage`. To experiment with that code, run `bin/console` for an interactive prompt.
4
+
5
+ TODO: Delete this and the text above, and describe your gem
6
+
7
+ ## Installation
8
+
9
+ Add this line to your application's Gemfile:
10
+
11
+ ```ruby
12
+ gem 'roma-storage'
13
+ ```
14
+
15
+ And then execute:
16
+
17
+ $ bundle
18
+
19
+ Or install it yourself as:
20
+
21
+ $ gem install roma-storage
22
+
23
+ ## Usage
24
+
25
+ TODO: Write usage instructions here
26
+
27
+ ## Development
28
+
29
+ After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
30
+
31
+ To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
32
+
33
+ ## Contributing
34
+
35
+ Bug reports and pull requests are welcome on GitHub at https://github.com/satoryu/roma-storage. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
36
+
37
+ ## License
38
+
39
+ The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
40
+
41
+ ## Code of Conduct
42
+
43
+ Everyone interacting in the Roma::Storage project’s codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/satoryu/roma-storage/blob/master/CODE_OF_CONDUCT.md).
data/Rakefile ADDED
@@ -0,0 +1,6 @@
1
+ require "bundler/gem_tasks"
2
+ require "rspec/core/rake_task"
3
+
4
+ RSpec::Core::RakeTask.new(:spec)
5
+
6
+ task :default => :spec
data/bin/console ADDED
@@ -0,0 +1,14 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require "bundler/setup"
4
+ require "roma/storage"
5
+
6
+ # You can add fixtures and/or initialization code here to make experimenting
7
+ # with your gem easier. You can also use a different console, if you like.
8
+
9
+ # (If you use this, don't forget to add pry to your Gemfile!)
10
+ # require "pry"
11
+ # Pry.start
12
+
13
+ require "irb"
14
+ IRB.start(__FILE__)
data/bin/setup ADDED
@@ -0,0 +1,8 @@
1
+ #!/usr/bin/env bash
2
+ set -euo pipefail
3
+ IFS=$'\n\t'
4
+ set -vx
5
+
6
+ bundle install
7
+
8
+ # Do any other automated setup that you need to do here
@@ -0,0 +1,5 @@
1
+ require 'roma/storage/version'
2
+ require 'roma/storage/basic_storage'
3
+ require 'roma/storage/dbm_storage'
4
+ require 'roma/storage/dummy_storage'
5
+ require 'roma/storage/ruby_hash_storage'
@@ -0,0 +1,765 @@
1
+ require 'digest/sha1'
2
+ require 'thread'
3
+ require 'fileutils'
4
+
5
+ module Roma
6
+ module Storage
7
+
8
+ class StorageException < Exception; end
9
+
10
+ class BasicStorage
11
+
12
+ attr_reader :hdb
13
+ attr_reader :hdiv
14
+ attr_reader :ext_name
15
+ attr_reader :error_message
16
+ attr_reader :dbs
17
+
18
+ attr_writer :vn_list
19
+ attr_writer :storage_path
20
+ attr_writer :option
21
+
22
+ attr_accessor :st_class
23
+ attr_accessor :divnum
24
+ attr_accessor :each_vn_dump_sleep
25
+ attr_accessor :each_vn_dump_sleep_count
26
+ attr_accessor :each_clean_up_sleep
27
+ attr_accessor :cleanup_regexp
28
+ attr_accessor :logic_clock_expire
29
+
30
+ attr_accessor :do_each_vn_dump
31
+
32
+ def initialize
33
+ # database handler
34
+ @hdb = []
35
+ # database cache handler
36
+ @hdbc = []
37
+ # status of a database
38
+ @dbs = []
39
+ @log_fd = nil
40
+ # file number list of a each_vn_dump while
41
+ @each_vn_dump_vnodes = []
42
+
43
+ @hdiv = Hash.new(0)
44
+
45
+ @ext_name = 'db'
46
+
47
+ @st_class = nil
48
+ @divnum = 10
49
+
50
+ @each_vn_dump_sleep = 0.001
51
+ @each_vn_dump_sleep_count = 100
52
+ @each_clean_up_sleep = 0.01
53
+ @cleanup_regexp = nil
54
+ @logic_clock_expire = 300
55
+
56
+ @each_cache_lock = Mutex::new
57
+ @each_clean_up_lock = Mutex::new
58
+ @stat_lock = Mutex::new
59
+ end
60
+
61
+ def get_stat
62
+ ret = {}
63
+ ret['storage.storage_path'] = File.expand_path(@storage_path)
64
+ ret['storage.st_class'] = @st_class.to_s.match(/Roma::Storage::(.*)/)[1]
65
+ ret['storage.divnum'] = @divnum
66
+ ret['storage.option'] = @option
67
+ ret['storage.each_vn_dump_sleep'] = @each_vn_dump_sleep
68
+ ret['storage.each_vn_dump_sleep_count'] = @each_vn_dump_sleep_count
69
+ ret['storage.each_vn_dump_files'] = @each_vn_dump_files.inspect
70
+ ret['storage.each_clean_up_sleep'] = @each_clean_up_sleep
71
+ ret['storage.cleanup_regexp'] = @cleanup_regexp
72
+ ret['storage.logic_clock_expire'] = @logic_clock_expire
73
+ ret['storage.safecopy_stats'] = @dbs.inspect
74
+ ret
75
+ end
76
+
77
+ # Compare this clock with the specified.
78
+ #
79
+ # -1, 0 or 1 as +clk1+ is numerically less than, equal to,
80
+ # or greater than the +clk2+ given as the parameter.
81
+ #
82
+ # logical clock space is a 32bit ring.
83
+ def cmp_clk(clk1, clk2)
84
+ if (clk1-clk2).abs < 0x80000000 # 1<<31
85
+ clk1 <=> clk2
86
+ else
87
+ clk2 <=> clk1
88
+ end
89
+ end
90
+ private :cmp_clk
91
+
92
+ def create_div_hash
93
+ @vn_list.each{ |vn|
94
+ @hdiv[vn] = Digest::SHA1.hexdigest(vn.to_s).hex % @divnum
95
+ }
96
+ end
97
+ protected :create_div_hash
98
+
99
+ def opendb
100
+ create_div_hash
101
+ FileUtils.mkdir_p(@storage_path)
102
+ @divnum.times do |i|
103
+ # open database file
104
+ @hdb[i] = open_db("#{@storage_path}/#{i}.#{@ext_name}")
105
+ # check cache file
106
+ if File.exist?(cache_file_name(i))
107
+ @hdbc[i] = open_db(cache_file_name(i))
108
+ stop_clean_up { @dbs[i] = :safecopy_flushed }
109
+ else
110
+ @dbs[i] = :normal
111
+ @hdbc[i] = nil
112
+ end
113
+ end
114
+ open_log
115
+ end
116
+
117
+ def closedb
118
+ stop_clean_up
119
+ buf = @hdb; @hdb = []
120
+ buf.each{ |h| close_db(h) if h }
121
+ buf = @hdbc; @hdbc = []
122
+ buf.each{ |h| close_db(h) if h }
123
+ close_log
124
+ end
125
+
126
+
127
+ # [ 0.. 3] vn
128
+ # [ 4.. 7] physical clock (unix time)
129
+ # [ 8..11] logical clock
130
+ # [12..15] exptime(unix time)
131
+ # [16.. ] value data
132
+
133
+ PACK_HEADER_TEMPLATE='NNNN'
134
+ PACK_TEMPLATE=PACK_HEADER_TEMPLATE+'a*'
135
+ def pack_header(vn, physical_clock, logical_clock, expire)
136
+ [vn,physical_clock, logical_clock, expire].pack(PACK_HEADER_TEMPLATE)
137
+ end
138
+ def unpack_header(str)
139
+ str.unpack(PACK_HEADER_TEMPLATE)
140
+ end
141
+ def pack_data(vn, physical_clock, logical_clock, expire,value)
142
+ [vn,physical_clock, logical_clock, expire, value].pack(PACK_TEMPLATE)
143
+ end
144
+ def unpack_data(str)
145
+ str.unpack(PACK_TEMPLATE)
146
+ end
147
+ private :pack_header, :unpack_header, :pack_data, :unpack_data
148
+
149
+ def db_get(vn, k)
150
+ n = @hdiv[vn]
151
+ d = @hdb[n].get(k)
152
+ return d if @dbs[n] == :normal
153
+
154
+ c = @hdbc[n].get(k)
155
+ return d unless c # in case of out of :normal status
156
+
157
+ if @dbs[n] == :cachecleaning && d
158
+ # in case of existing value is both @hdb and @hdbc
159
+ vn, lat, clk, expt = unpack_header(d)
160
+ cvn, clat, cclk, cexpt = unpack_header(c)
161
+ return d if cmp_clk(clk, cclk) > 0 # if @hdb newer than @hdbc
162
+ end
163
+ c
164
+ end
165
+
166
+ def db_put(vn, k, v)
167
+ n = @hdiv[vn]
168
+ if @dbs[n] == :safecopy_flushing || @dbs[n] == :safecopy_flushed
169
+ ret = @hdbc[n].put(k, v)
170
+ else
171
+ ret = @hdb[n].put(k, v)
172
+ end
173
+ ret
174
+ end
175
+
176
+ def get_context(vn, k, d)
177
+ buf = db_get(vn, k)
178
+ return nil unless buf
179
+ unpack_header(buf)
180
+ end
181
+
182
+ def cas(vn, k, d, clk, expt, v)
183
+ buf = db_get(vn ,k)
184
+ return :not_found unless buf
185
+ t = Time.now.to_i
186
+ data = unpack_data(buf)
187
+ return :not_found if t > data[3]
188
+ return :exists if clk != data[2]
189
+ clk = (data[2] + 1) & 0xffffffff
190
+ ret = [vn, t, clk, expt, v]
191
+ return ret if db_put(vn, k, pack_data(*ret))
192
+ nil
193
+ end
194
+
195
+ def rset(vn, k, d, clk, expt, v)
196
+ buf = db_get(vn, k)
197
+ t = Time.now.to_i
198
+ if buf
199
+ data = unpack_data(buf)
200
+ if t - data[1] < @logic_clock_expire && cmp_clk(clk,data[2]) <= 0
201
+ @error_message = "error:#{t-data[1]} < #{@logic_clock_expire} && cmp_clk(#{clk},#{data[2]})<=0"
202
+ return nil
203
+ end
204
+ end
205
+
206
+ ret = [vn, t, clk, expt, v]
207
+ return ret if db_put(vn, k, pack_data(*ret))
208
+ @error_message = "error:put"
209
+ nil
210
+ end
211
+
212
+ def set(vn, k, d, expt, v)
213
+ buf = db_get(vn, k)
214
+ clk = 0
215
+ if buf
216
+ data = unpack_data(buf)
217
+ clk = (data[2] + 1) & 0xffffffff
218
+ end
219
+
220
+ ret = [vn, Time.now.to_i, clk, expt, v]
221
+ return ret if db_put(vn , k, pack_data(*ret))
222
+ nil
223
+ end
224
+
225
+ def add(vn, k, d, expt, v)
226
+ buf = db_get(vn, k)
227
+ clk = 0
228
+ if buf
229
+ vn, t, clk, expt2, v2 = unpack_data(buf)
230
+ return nil if Time.now.to_i <= expt2
231
+ clk = (clk + 1) & 0xffffffff
232
+ end
233
+
234
+ # not exist
235
+ ret = [vn, Time.now.to_i, clk, expt, v]
236
+ return ret if db_put(vn, k, pack_data(*ret))
237
+ nil
238
+ end
239
+
240
+ def replace(vn, k, d, expt, v)
241
+ buf = db_get(vn, k)
242
+ return nil unless buf
243
+
244
+ # buf != nil
245
+ vn, t, clk, expt2, v2 = unpack_data(buf)
246
+ return nil if Time.now.to_i > expt2
247
+ clk = (clk + 1) & 0xffffffff
248
+
249
+ ret = [vn, Time.now.to_i, clk, expt, v]
250
+ return ret if db_put(vn, k, pack_data(*ret))
251
+ nil
252
+ end
253
+
254
+ def append(vn, k, d, expt, v)
255
+ buf = db_get(vn, k)
256
+ return nil unless buf
257
+
258
+ # buf != nil
259
+ vn, t, clk, expt2, v2 = unpack_data(buf)
260
+ return nil if Time.now.to_i > expt2
261
+ clk = (clk + 1) & 0xffffffff
262
+
263
+ ret = [vn, Time.now.to_i, clk, expt, v2 + v]
264
+ return ret if db_put(vn, k, pack_data(*ret))
265
+ nil
266
+ end
267
+
268
+ def prepend(vn, k, d, expt, v)
269
+ buf = db_get(vn, k)
270
+ return nil unless buf
271
+
272
+ # buf != nil
273
+ vn, t, clk, expt2, v2 = unpack_data(buf)
274
+ return nil if Time.now.to_i > expt2
275
+ clk = (clk + 1) & 0xffffffff
276
+
277
+ ret = [vn, Time.now.to_i, clk, expt, v + v2]
278
+ return ret if db_put(vn, k, pack_data(*ret))
279
+ nil
280
+ end
281
+
282
+ def get(vn, k, d)
283
+ buf = db_get(vn, k)
284
+ return nil unless buf
285
+ vn, t, clk, expt, v = unpack_data(buf)
286
+
287
+ return nil if Time.now.to_i > expt
288
+ v
289
+ end
290
+
291
+ def get_raw(vn, k, d)
292
+ buf = db_get(vn, k)
293
+ return nil unless buf
294
+
295
+ unpack_data(buf)
296
+ end
297
+
298
+ def get_raw2(k)
299
+ @hdb.each{|hdb|
300
+ buf = hdb.get(k)
301
+ return unpack_data(buf) if buf
302
+ }
303
+ nil
304
+ end
305
+
306
+ def rdelete(vn, k, d, clk)
307
+ buf = db_get(vn, k)
308
+ t = Time.now.to_i
309
+ if buf
310
+ data = unpack_header(buf)
311
+ if t - data[1] < @logic_clock_expire && cmp_clk(clk,data[2]) <= 0
312
+ @error_message = "error:#{t-data[1]} < #{@logic_clock_expire} && cmp_clk(#{clk},#{data[2]})<=0"
313
+ return nil
314
+ end
315
+ end
316
+
317
+ # [ 0.. 3] vn
318
+ # [ 4.. 7] physical clock(unix time)
319
+ # [ 8..11] logical clock
320
+ # [12..15] exptime(unix time) => 0
321
+ ret = [vn, t, clk, 0]
322
+ if db_put(vn, k, pack_header(*ret))
323
+ return ret
324
+ else
325
+ return nil
326
+ end
327
+ end
328
+
329
+ def delete(vn, k, d)
330
+ buf = db_get(vn, k)
331
+ v = ret = nil
332
+ clk = 0
333
+ if buf
334
+ vn, t, clk, expt, v2 = unpack_data(buf)
335
+ return :deletemark if expt == 0
336
+ clk = (clk + 1) & 0xffffffff
337
+ v = v2 if v2 && v2.length != 0 && Time.now.to_i <= expt
338
+ end
339
+
340
+ # [ 0.. 3] vn
341
+ # [ 4.. 7] physical clock(unix time)
342
+ # [ 8..11] logical clock
343
+ # [12..15] exptime(unix time) => 0
344
+ ret = [vn, Time.now.to_i, clk, 0, v]
345
+ if db_put(vn, k, pack_header(*ret[0..-2]))
346
+ return ret
347
+ else
348
+ return nil
349
+ end
350
+ end
351
+
352
+ def out(vn, k, d)
353
+ @hdb[@hdiv[vn]].out(k)
354
+ end
355
+
356
+ def incr(vn, k, d, v)
357
+ buf = db_get(vn, k)
358
+ return nil unless buf
359
+
360
+ # buf != nil
361
+ vn, t, clk, expt2, v2 = unpack_data(buf)
362
+ return nil if Time.now.to_i > expt2
363
+ clk = (clk + 1) & 0xffffffff
364
+
365
+ v = (v2.to_i + v)
366
+ v = 0 if v < 0
367
+ v = v & 0xffffffffffffffff
368
+
369
+ ret = [vn, Time.now.to_i, clk, expt2, v.to_s]
370
+ return ret if db_put(vn, k, pack_data(*ret))
371
+ nil
372
+ end
373
+
374
+ def decr(vn, k, d, v)
375
+ buf = db_get(vn, k)
376
+ return nil unless buf
377
+
378
+ # buf != nil
379
+ vn, t, clk, expt2, v2 = unpack_data(buf)
380
+ return nil if Time.now.to_i > expt2
381
+ clk = (clk + 1) & 0xffffffff
382
+
383
+ v = (v2.to_i - v)
384
+ v = 0 if v < 0
385
+ v = v & 0xffffffffffffffff
386
+
387
+ ret = [vn, Time.now.to_i, clk, expt2, v.to_s]
388
+ return ret if db_put(vn, k, pack_data(*ret))
389
+ nil
390
+ end
391
+
392
+ # set expire time
393
+ def set_expt(vn, k, d, expt)
394
+ buf = db_get(vn, k)
395
+ if buf
396
+ vn, t, clk, expt2, v = unpack_data(buf)
397
+ return nil if Time.now.to_i > expt2
398
+ clk = (clk + 1) & 0xffffffff
399
+ ret = [vn, Time.now.to_i, clk, expt, v]
400
+ return ret if db_put(vn, k, pack_data(*ret))
401
+ end
402
+ nil
403
+ end
404
+
405
+ def true_length
406
+ res = 0
407
+ @hdb.each{ |hdb| res += hdb.rnum }
408
+ res
409
+ end
410
+
411
+ def each_clean_up(t, vnhash)
412
+ @do_clean_up = true
413
+
414
+ f = nil;
415
+ if @cleanup_regexp && File.exist?(@storage_path)
416
+ f = open(@storage_path + '/klist.txt','w')
417
+ end
418
+
419
+ return unless @each_clean_up_lock.try_lock
420
+ nt = Time.now.to_i
421
+ @divnum.times do |i|
422
+ next if @dbs[i] != :normal
423
+ hdb = @hdb[i]
424
+ hdb.each do |k, v|
425
+ return unless @do_clean_up # 1st check
426
+ vn, last, clk, expt = unpack_header(v)
427
+ vn_stat = vnhash[vn]
428
+ if f && @cleanup_regexp && k =~ /#{@cleanup_regexp}/
429
+ # write klist
430
+ f.puts("#{k},#{last},#{clk}") if hdb.get(k) == v
431
+ end
432
+ if vn_stat == :primary && ( (expt != 0 && nt > expt) || (expt == 0 && t > last) )
433
+ if yield k, vn
434
+ hdb.out(k) if hdb.get(k) == v
435
+ end
436
+ elsif vn_stat == nil && t > last
437
+ if yield k, vn
438
+ hdb.out(k) if hdb.get(k) == v
439
+ end
440
+ end
441
+ return unless @do_clean_up # 2nd ckeck
442
+ sleep @each_clean_up_sleep
443
+ end
444
+ end
445
+ ensure
446
+ @each_clean_up_lock.unlock if @each_clean_up_lock.locked?
447
+ if f
448
+ @cleanup_regexp = nil
449
+ f.close
450
+ end
451
+ end
452
+
453
+ def stop_clean_up(&block)
454
+ @do_clean_up = false
455
+ if block
456
+ @each_clean_up_lock.lock
457
+ begin
458
+ block.call
459
+ ensure
460
+ @each_clean_up_lock.unlock
461
+ end
462
+ end
463
+ end
464
+
465
+ def load(dmp)
466
+ n = 0
467
+ h = Marshal.load(dmp)
468
+ h.each_pair{ |k, v|
469
+ # remort data
470
+ r_vn, r_last, r_clk, r_expt = unpack_header(v)
471
+ raise "An invalid vnode number is include.key=#{k} vn=#{r_vn}" unless @hdiv.key?(r_vn)
472
+ local = @hdb[@hdiv[r_vn]].get(k)
473
+ if local == nil
474
+ n += 1
475
+ @hdb[@hdiv[r_vn]].put(k, v)
476
+ else
477
+ # local data
478
+ l_vn, l_last, l_clk, l_expt = unpack_data(local)
479
+ if r_last - l_last < @logic_clock_expire && cmp_clk(r_clk,l_clk) <= 0
480
+ else # remort is newer.
481
+ n += 1
482
+ @hdb[@hdiv[r_vn]].put(k, v)
483
+ end
484
+ end
485
+ sleep @each_vn_dump_sleep
486
+ }
487
+ n
488
+ end
489
+
490
+ def load_stream_dump(vn, last, clk, expt, k, v)
491
+ buf = db_get(vn, k)
492
+ if buf
493
+ data = unpack_header(buf)
494
+ if last - data[1] < @logic_clock_expire && cmp_clk(clk,data[2]) <= 0
495
+ return nil
496
+ end
497
+ end
498
+
499
+ ret = [vn, last, clk, expt, v]
500
+ if expt == 0
501
+ # for the deleted mark
502
+ return ret if db_put(vn, k, pack_header(*ret[0..3]))
503
+ else
504
+ return ret if db_put(vn, k, pack_data(*ret))
505
+ end
506
+ nil
507
+ end
508
+
509
+ def load_stream_dump_for_cachecleaning(vn, last, clk, expt, k, v)
510
+ n = @hdiv[vn]
511
+ buf = @hdb[n].get(k)
512
+ if buf
513
+ data = unpack_header(buf)
514
+ if last - data[1] < @logic_clock_expire && cmp_clk(clk,data[2]) <= 0
515
+ return nil
516
+ end
517
+ end
518
+
519
+ ret = [vn, last, clk, expt, v]
520
+ if expt == 0
521
+ # for the deleted mark
522
+ return ret if @hdb[n].put(k, pack_header(*ret[0..3]))
523
+ else
524
+ return ret if @hdb[n].put(k, pack_data(*ret))
525
+ end
526
+ nil
527
+ end
528
+
529
+ # Returns the vnode dump.
530
+ def dump(vn)
531
+ buf = get_vnode_hash(vn)
532
+ return nil if buf.length == 0
533
+ Marshal.dump(buf)
534
+ end
535
+
536
+ def each_vn_dump(target_vn)
537
+ n = @hdiv[target_vn]
538
+ @stat_lock.synchronize do
539
+ return false if @dbs[n] != :normal
540
+ return false if @each_vn_dump_vnodes.include?(target_vn)
541
+ @each_vn_dump_vnodes << target_vn
542
+ end
543
+
544
+ begin
545
+ @do_each_vn_dump = true
546
+ each_unpacked_db(target_vn, @hdb) do |vn, last, clk, expt, k, val|
547
+ return unless @do_each_vn_dump
548
+ yield vn_dump_pack(vn, last, clk, expt, k, val)
549
+ end
550
+ ensure
551
+ @each_vn_dump_vnodes.delete(target_vn)
552
+ end
553
+
554
+ true
555
+ end
556
+
557
+ def vn_dump_pack(vn, last, clk, expt, k, val)
558
+ if val
559
+ return [vn, last, clk, expt, k.length, k, val.length, val].pack("NNNNNa#{k.length}Na#{val.length}")
560
+ else
561
+ return [vn, last, clk, expt, k.length, k, 0].pack("NNNNNa#{k.length}N")
562
+ end
563
+ end
564
+ private :vn_dump_pack
565
+
566
+ def each_unpacked_db(target_vn, db)
567
+ count = 0
568
+ tn = Time.now.to_i
569
+ db[@hdiv[target_vn]].each do |k,v|
570
+ vn, last, clk, expt, val = unpack_data(v)
571
+ if vn != target_vn || (expt != 0 && tn > expt)
572
+ count += 1
573
+ sleep @each_vn_dump_sleep if count % @each_vn_dump_sleep_count == 0
574
+ next
575
+ end
576
+ yield vn, last, clk, expt, k, val
577
+ end
578
+ end
579
+ private :each_unpacked_db
580
+
581
+ def each_hdb_dump(i,except_vnh = nil)
582
+ count = 0
583
+ @hdb[i].each{|k,v|
584
+ vn, last, clk, expt, val = unpack_data(v)
585
+ if except_vnh && except_vnh.key?(vn) || Time.now.to_i > expt
586
+ count += 1
587
+ sleep @each_vn_dump_sleep if count % @each_vn_dump_sleep_count == 0
588
+ else
589
+ yield [vn, last, clk, expt, k.length, k, val.length, val].pack("NNNNNa#{k.length}Na#{val.length}")
590
+ sleep @each_vn_dump_sleep
591
+ end
592
+ }
593
+ end
594
+
595
+ # Remove a key for the cache(@hdbc).
596
+ # +dn+:: number of database
597
+ # +key+:: key
598
+ def out_cache(dn, key)
599
+ @hdbc[dn].out(key)
600
+ end
601
+
602
+ # Calls the geven block,
603
+ # passes the cache(@hdbc) element.
604
+ # +dn+:: number of database
605
+ # +keys+:: key list
606
+ def each_cache_by_keys(dn, keys)
607
+ keys.each do |k|
608
+ v = @hdbc[dn].get(k)
609
+ vn, last, clk, expt, val = unpack_data(v)
610
+ yield [vn, last, clk, expt, k, val]
611
+ end
612
+ end
613
+
614
+ # Calls the geven block,
615
+ # passes the cache(@hdbc) element as the spushv command data format.
616
+ # +dn+:: number of database
617
+ # +keys+:: key list
618
+ def each_cache_dump_pack(dn, keys)
619
+ keys.each do |k|
620
+ v = @hdbc[dn].get(k)
621
+ vn, last, clk, expt, val = unpack_data(v)
622
+ yield vn_dump_pack(vn, last, clk, expt, k, val)
623
+ end
624
+ end
625
+
626
+ # Returns a key array in a cache(@hdbc).
627
+ # +dn+:: number of database
628
+ # +kn+:: number of keys which is return value
629
+ def get_keys_in_cache(dn, kn=100)
630
+ return nil if @do_each_vn_dump
631
+ ret = []
632
+ return ret unless @hdbc[dn]
633
+ count = 0
634
+ @each_cache_lock.synchronize do
635
+ @hdbc[dn].each do |k, v|
636
+ ret << k
637
+ break if (count+=1) >= kn
638
+ end
639
+ end
640
+ ret
641
+ end
642
+
643
+ # Create vnode dump.
644
+ def get_vnode_hash(vn)
645
+ buf = {}
646
+ count = 0
647
+ @hdb[@hdiv[vn]].each{ |k, v|
648
+ count += 1
649
+ sleep @each_vn_dump_sleep if count % @each_vn_dump_sleep_count == 0
650
+ dat = unpack_data(v) #v.unpack('NNNN')
651
+ buf[k] = v if dat[0] == vn
652
+ }
653
+ return buf
654
+ end
655
+ private :get_vnode_hash
656
+
657
+ def flush_db(dn)
658
+ @hdb[dn].sync
659
+ end
660
+
661
+ def cache_file_name(dn)
662
+ "#{@storage_path}/#{dn}.cache.#{@ext_name}"
663
+ end
664
+
665
+ def set_db_stat(dn, stat)
666
+ @stat_lock.synchronize do
667
+ case @dbs[dn]
668
+ when :normal
669
+ @each_vn_dump_vnodes.each do |vn|
670
+ return false if dn == @hdiv[vn]
671
+ end
672
+ if stat == :safecopy_flushing
673
+ # open cache
674
+ @hdbc[dn] = open_db(cache_file_name(dn))
675
+ stop_clean_up { @dbs[dn] = stat }
676
+ write_log("#{dn} #{stat.to_s}")
677
+ stat
678
+ else
679
+ false
680
+ end
681
+ when :safecopy_flushing
682
+ if stat == :safecopy_flushed
683
+ write_log("#{dn} #{stat.to_s}")
684
+ @dbs[dn] = stat
685
+ else
686
+ false
687
+ end
688
+ when :safecopy_flushed
689
+ if stat == :cachecleaning
690
+ write_log("#{dn} #{stat.to_s}")
691
+ @dbs[dn] = stat
692
+ else
693
+ false
694
+ end
695
+ when :cachecleaning
696
+ if stat == :normal
697
+ write_log("#{dn} #{stat.to_s}")
698
+ @dbs[dn] = stat
699
+ # remove cache
700
+ close_db(@hdbc[dn])
701
+ @hdbc[dn] = nil
702
+ if File.exist?("#{@storage_path}/#{dn}.cache.#{@ext_name}")
703
+ File.unlink("#{@storage_path}/#{dn}.cache.#{@ext_name}")
704
+ end
705
+ stat
706
+ elsif stat == :safecopy_flushing
707
+ write_log("#{dn} #{stat.to_s}")
708
+ @dbs[dn] = stat
709
+ else
710
+ false
711
+ end
712
+ else
713
+ false
714
+ end
715
+ end
716
+ end
717
+
718
+ def get_logfile_list
719
+ l={}
720
+ files=Dir.glob("#{@storage_path}/status.log.*")
721
+ files.each{ |file|
722
+ if /$.+status\.log\.(\d+)$/=~file
723
+ l[$1.to_i]=$&
724
+ end
725
+ }
726
+ # sorted by old order
727
+ l.to_a.sort{|a,b| a[0]<=>b[0]}
728
+ end
729
+
730
+ def open_log
731
+ logs = get_logfile_list
732
+ if logs.length == 0
733
+ @log_name="#{@storage_path}/status.log.1"
734
+ else
735
+ if File::stat("#{@fname}.#{logs.last[0]}").size == 0
736
+ @log_name="#{@fname}.#{logs.last[0]}"
737
+ else
738
+ @log_name="#{@fname}.#{logs.last[0]+1}"
739
+ end
740
+ end
741
+ @log_fd=File.open(@log_name,"a")
742
+ end
743
+
744
+ def write_log(line)
745
+ return unless @log_name
746
+ # log rotation
747
+ if File::stat(@log_name).size > 1000 * 1024
748
+ close_log
749
+ open_log
750
+ end
751
+ t = Time.now
752
+ tstr = "#{t.strftime('%Y-%m-%dT%H:%M:%S')}.#{t.usec}"
753
+ @log_fd.write("#{tstr} #{line}\n")
754
+ @log_fd.flush
755
+ end
756
+
757
+ def close_log
758
+ @log_fd.close if @log_fd
759
+ @log_fd = nil
760
+ end
761
+
762
+ end # class BasicStorage
763
+
764
+ end # module Storage
765
+ end # module Roma