roma-storage 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +12 -0
- data/.rspec +3 -0
- data/.travis.yml +5 -0
- data/CODE_OF_CONDUCT.md +74 -0
- data/Gemfile +6 -0
- data/LICENSE.txt +674 -0
- data/README.md +43 -0
- data/Rakefile +6 -0
- data/bin/console +14 -0
- data/bin/setup +8 -0
- data/lib/roma/storage.rb +5 -0
- data/lib/roma/storage/basic_storage.rb +765 -0
- data/lib/roma/storage/dummy_storage.rb +42 -0
- data/lib/roma/storage/ruby_hash_storage.rb +114 -0
- data/lib/roma/storage/version.rb +5 -0
- data/roma-storage.gemspec +36 -0
- metadata +103 -0
data/README.md
ADDED
@@ -0,0 +1,43 @@
|
|
1
|
+
# Roma::Storage
|
2
|
+
|
3
|
+
Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/roma/storage`. To experiment with that code, run `bin/console` for an interactive prompt.
|
4
|
+
|
5
|
+
TODO: Delete this and the text above, and describe your gem
|
6
|
+
|
7
|
+
## Installation
|
8
|
+
|
9
|
+
Add this line to your application's Gemfile:
|
10
|
+
|
11
|
+
```ruby
|
12
|
+
gem 'roma-storage'
|
13
|
+
```
|
14
|
+
|
15
|
+
And then execute:
|
16
|
+
|
17
|
+
$ bundle
|
18
|
+
|
19
|
+
Or install it yourself as:
|
20
|
+
|
21
|
+
$ gem install roma-storage
|
22
|
+
|
23
|
+
## Usage
|
24
|
+
|
25
|
+
TODO: Write usage instructions here
|
26
|
+
|
27
|
+
## Development
|
28
|
+
|
29
|
+
After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
|
30
|
+
|
31
|
+
To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
|
32
|
+
|
33
|
+
## Contributing
|
34
|
+
|
35
|
+
Bug reports and pull requests are welcome on GitHub at https://github.com/satoryu/roma-storage. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
|
36
|
+
|
37
|
+
## License
|
38
|
+
|
39
|
+
The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
|
40
|
+
|
41
|
+
## Code of Conduct
|
42
|
+
|
43
|
+
Everyone interacting in the Roma::Storage project’s codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/satoryu/roma-storage/blob/master/CODE_OF_CONDUCT.md).
|
data/Rakefile
ADDED
data/bin/console
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require "bundler/setup"
|
4
|
+
require "roma/storage"
|
5
|
+
|
6
|
+
# You can add fixtures and/or initialization code here to make experimenting
|
7
|
+
# with your gem easier. You can also use a different console, if you like.
|
8
|
+
|
9
|
+
# (If you use this, don't forget to add pry to your Gemfile!)
|
10
|
+
# require "pry"
|
11
|
+
# Pry.start
|
12
|
+
|
13
|
+
require "irb"
|
14
|
+
IRB.start(__FILE__)
|
data/bin/setup
ADDED
data/lib/roma/storage.rb
ADDED
@@ -0,0 +1,765 @@
|
|
1
|
+
require 'digest/sha1'
|
2
|
+
require 'thread'
|
3
|
+
require 'fileutils'
|
4
|
+
|
5
|
+
module Roma
|
6
|
+
module Storage
|
7
|
+
|
8
|
+
class StorageException < Exception; end
|
9
|
+
|
10
|
+
class BasicStorage
|
11
|
+
|
12
|
+
attr_reader :hdb
|
13
|
+
attr_reader :hdiv
|
14
|
+
attr_reader :ext_name
|
15
|
+
attr_reader :error_message
|
16
|
+
attr_reader :dbs
|
17
|
+
|
18
|
+
attr_writer :vn_list
|
19
|
+
attr_writer :storage_path
|
20
|
+
attr_writer :option
|
21
|
+
|
22
|
+
attr_accessor :st_class
|
23
|
+
attr_accessor :divnum
|
24
|
+
attr_accessor :each_vn_dump_sleep
|
25
|
+
attr_accessor :each_vn_dump_sleep_count
|
26
|
+
attr_accessor :each_clean_up_sleep
|
27
|
+
attr_accessor :cleanup_regexp
|
28
|
+
attr_accessor :logic_clock_expire
|
29
|
+
|
30
|
+
attr_accessor :do_each_vn_dump
|
31
|
+
|
32
|
+
def initialize
|
33
|
+
# database handler
|
34
|
+
@hdb = []
|
35
|
+
# database cache handler
|
36
|
+
@hdbc = []
|
37
|
+
# status of a database
|
38
|
+
@dbs = []
|
39
|
+
@log_fd = nil
|
40
|
+
# file number list of a each_vn_dump while
|
41
|
+
@each_vn_dump_vnodes = []
|
42
|
+
|
43
|
+
@hdiv = Hash.new(0)
|
44
|
+
|
45
|
+
@ext_name = 'db'
|
46
|
+
|
47
|
+
@st_class = nil
|
48
|
+
@divnum = 10
|
49
|
+
|
50
|
+
@each_vn_dump_sleep = 0.001
|
51
|
+
@each_vn_dump_sleep_count = 100
|
52
|
+
@each_clean_up_sleep = 0.01
|
53
|
+
@cleanup_regexp = nil
|
54
|
+
@logic_clock_expire = 300
|
55
|
+
|
56
|
+
@each_cache_lock = Mutex::new
|
57
|
+
@each_clean_up_lock = Mutex::new
|
58
|
+
@stat_lock = Mutex::new
|
59
|
+
end
|
60
|
+
|
61
|
+
def get_stat
|
62
|
+
ret = {}
|
63
|
+
ret['storage.storage_path'] = File.expand_path(@storage_path)
|
64
|
+
ret['storage.st_class'] = @st_class.to_s.match(/Roma::Storage::(.*)/)[1]
|
65
|
+
ret['storage.divnum'] = @divnum
|
66
|
+
ret['storage.option'] = @option
|
67
|
+
ret['storage.each_vn_dump_sleep'] = @each_vn_dump_sleep
|
68
|
+
ret['storage.each_vn_dump_sleep_count'] = @each_vn_dump_sleep_count
|
69
|
+
ret['storage.each_vn_dump_files'] = @each_vn_dump_files.inspect
|
70
|
+
ret['storage.each_clean_up_sleep'] = @each_clean_up_sleep
|
71
|
+
ret['storage.cleanup_regexp'] = @cleanup_regexp
|
72
|
+
ret['storage.logic_clock_expire'] = @logic_clock_expire
|
73
|
+
ret['storage.safecopy_stats'] = @dbs.inspect
|
74
|
+
ret
|
75
|
+
end
|
76
|
+
|
77
|
+
# Compare this clock with the specified.
|
78
|
+
#
|
79
|
+
# -1, 0 or 1 as +clk1+ is numerically less than, equal to,
|
80
|
+
# or greater than the +clk2+ given as the parameter.
|
81
|
+
#
|
82
|
+
# logical clock space is a 32bit ring.
|
83
|
+
def cmp_clk(clk1, clk2)
|
84
|
+
if (clk1-clk2).abs < 0x80000000 # 1<<31
|
85
|
+
clk1 <=> clk2
|
86
|
+
else
|
87
|
+
clk2 <=> clk1
|
88
|
+
end
|
89
|
+
end
|
90
|
+
private :cmp_clk
|
91
|
+
|
92
|
+
def create_div_hash
|
93
|
+
@vn_list.each{ |vn|
|
94
|
+
@hdiv[vn] = Digest::SHA1.hexdigest(vn.to_s).hex % @divnum
|
95
|
+
}
|
96
|
+
end
|
97
|
+
protected :create_div_hash
|
98
|
+
|
99
|
+
def opendb
|
100
|
+
create_div_hash
|
101
|
+
FileUtils.mkdir_p(@storage_path)
|
102
|
+
@divnum.times do |i|
|
103
|
+
# open database file
|
104
|
+
@hdb[i] = open_db("#{@storage_path}/#{i}.#{@ext_name}")
|
105
|
+
# check cache file
|
106
|
+
if File.exist?(cache_file_name(i))
|
107
|
+
@hdbc[i] = open_db(cache_file_name(i))
|
108
|
+
stop_clean_up { @dbs[i] = :safecopy_flushed }
|
109
|
+
else
|
110
|
+
@dbs[i] = :normal
|
111
|
+
@hdbc[i] = nil
|
112
|
+
end
|
113
|
+
end
|
114
|
+
open_log
|
115
|
+
end
|
116
|
+
|
117
|
+
def closedb
|
118
|
+
stop_clean_up
|
119
|
+
buf = @hdb; @hdb = []
|
120
|
+
buf.each{ |h| close_db(h) if h }
|
121
|
+
buf = @hdbc; @hdbc = []
|
122
|
+
buf.each{ |h| close_db(h) if h }
|
123
|
+
close_log
|
124
|
+
end
|
125
|
+
|
126
|
+
|
127
|
+
# [ 0.. 3] vn
|
128
|
+
# [ 4.. 7] physical clock (unix time)
|
129
|
+
# [ 8..11] logical clock
|
130
|
+
# [12..15] exptime(unix time)
|
131
|
+
# [16.. ] value data
|
132
|
+
|
133
|
+
PACK_HEADER_TEMPLATE='NNNN'
|
134
|
+
PACK_TEMPLATE=PACK_HEADER_TEMPLATE+'a*'
|
135
|
+
def pack_header(vn, physical_clock, logical_clock, expire)
|
136
|
+
[vn,physical_clock, logical_clock, expire].pack(PACK_HEADER_TEMPLATE)
|
137
|
+
end
|
138
|
+
def unpack_header(str)
|
139
|
+
str.unpack(PACK_HEADER_TEMPLATE)
|
140
|
+
end
|
141
|
+
def pack_data(vn, physical_clock, logical_clock, expire,value)
|
142
|
+
[vn,physical_clock, logical_clock, expire, value].pack(PACK_TEMPLATE)
|
143
|
+
end
|
144
|
+
def unpack_data(str)
|
145
|
+
str.unpack(PACK_TEMPLATE)
|
146
|
+
end
|
147
|
+
private :pack_header, :unpack_header, :pack_data, :unpack_data
|
148
|
+
|
149
|
+
def db_get(vn, k)
|
150
|
+
n = @hdiv[vn]
|
151
|
+
d = @hdb[n].get(k)
|
152
|
+
return d if @dbs[n] == :normal
|
153
|
+
|
154
|
+
c = @hdbc[n].get(k)
|
155
|
+
return d unless c # in case of out of :normal status
|
156
|
+
|
157
|
+
if @dbs[n] == :cachecleaning && d
|
158
|
+
# in case of existing value is both @hdb and @hdbc
|
159
|
+
vn, lat, clk, expt = unpack_header(d)
|
160
|
+
cvn, clat, cclk, cexpt = unpack_header(c)
|
161
|
+
return d if cmp_clk(clk, cclk) > 0 # if @hdb newer than @hdbc
|
162
|
+
end
|
163
|
+
c
|
164
|
+
end
|
165
|
+
|
166
|
+
def db_put(vn, k, v)
|
167
|
+
n = @hdiv[vn]
|
168
|
+
if @dbs[n] == :safecopy_flushing || @dbs[n] == :safecopy_flushed
|
169
|
+
ret = @hdbc[n].put(k, v)
|
170
|
+
else
|
171
|
+
ret = @hdb[n].put(k, v)
|
172
|
+
end
|
173
|
+
ret
|
174
|
+
end
|
175
|
+
|
176
|
+
def get_context(vn, k, d)
|
177
|
+
buf = db_get(vn, k)
|
178
|
+
return nil unless buf
|
179
|
+
unpack_header(buf)
|
180
|
+
end
|
181
|
+
|
182
|
+
def cas(vn, k, d, clk, expt, v)
|
183
|
+
buf = db_get(vn ,k)
|
184
|
+
return :not_found unless buf
|
185
|
+
t = Time.now.to_i
|
186
|
+
data = unpack_data(buf)
|
187
|
+
return :not_found if t > data[3]
|
188
|
+
return :exists if clk != data[2]
|
189
|
+
clk = (data[2] + 1) & 0xffffffff
|
190
|
+
ret = [vn, t, clk, expt, v]
|
191
|
+
return ret if db_put(vn, k, pack_data(*ret))
|
192
|
+
nil
|
193
|
+
end
|
194
|
+
|
195
|
+
def rset(vn, k, d, clk, expt, v)
|
196
|
+
buf = db_get(vn, k)
|
197
|
+
t = Time.now.to_i
|
198
|
+
if buf
|
199
|
+
data = unpack_data(buf)
|
200
|
+
if t - data[1] < @logic_clock_expire && cmp_clk(clk,data[2]) <= 0
|
201
|
+
@error_message = "error:#{t-data[1]} < #{@logic_clock_expire} && cmp_clk(#{clk},#{data[2]})<=0"
|
202
|
+
return nil
|
203
|
+
end
|
204
|
+
end
|
205
|
+
|
206
|
+
ret = [vn, t, clk, expt, v]
|
207
|
+
return ret if db_put(vn, k, pack_data(*ret))
|
208
|
+
@error_message = "error:put"
|
209
|
+
nil
|
210
|
+
end
|
211
|
+
|
212
|
+
def set(vn, k, d, expt, v)
|
213
|
+
buf = db_get(vn, k)
|
214
|
+
clk = 0
|
215
|
+
if buf
|
216
|
+
data = unpack_data(buf)
|
217
|
+
clk = (data[2] + 1) & 0xffffffff
|
218
|
+
end
|
219
|
+
|
220
|
+
ret = [vn, Time.now.to_i, clk, expt, v]
|
221
|
+
return ret if db_put(vn , k, pack_data(*ret))
|
222
|
+
nil
|
223
|
+
end
|
224
|
+
|
225
|
+
def add(vn, k, d, expt, v)
|
226
|
+
buf = db_get(vn, k)
|
227
|
+
clk = 0
|
228
|
+
if buf
|
229
|
+
vn, t, clk, expt2, v2 = unpack_data(buf)
|
230
|
+
return nil if Time.now.to_i <= expt2
|
231
|
+
clk = (clk + 1) & 0xffffffff
|
232
|
+
end
|
233
|
+
|
234
|
+
# not exist
|
235
|
+
ret = [vn, Time.now.to_i, clk, expt, v]
|
236
|
+
return ret if db_put(vn, k, pack_data(*ret))
|
237
|
+
nil
|
238
|
+
end
|
239
|
+
|
240
|
+
def replace(vn, k, d, expt, v)
|
241
|
+
buf = db_get(vn, k)
|
242
|
+
return nil unless buf
|
243
|
+
|
244
|
+
# buf != nil
|
245
|
+
vn, t, clk, expt2, v2 = unpack_data(buf)
|
246
|
+
return nil if Time.now.to_i > expt2
|
247
|
+
clk = (clk + 1) & 0xffffffff
|
248
|
+
|
249
|
+
ret = [vn, Time.now.to_i, clk, expt, v]
|
250
|
+
return ret if db_put(vn, k, pack_data(*ret))
|
251
|
+
nil
|
252
|
+
end
|
253
|
+
|
254
|
+
def append(vn, k, d, expt, v)
|
255
|
+
buf = db_get(vn, k)
|
256
|
+
return nil unless buf
|
257
|
+
|
258
|
+
# buf != nil
|
259
|
+
vn, t, clk, expt2, v2 = unpack_data(buf)
|
260
|
+
return nil if Time.now.to_i > expt2
|
261
|
+
clk = (clk + 1) & 0xffffffff
|
262
|
+
|
263
|
+
ret = [vn, Time.now.to_i, clk, expt, v2 + v]
|
264
|
+
return ret if db_put(vn, k, pack_data(*ret))
|
265
|
+
nil
|
266
|
+
end
|
267
|
+
|
268
|
+
def prepend(vn, k, d, expt, v)
|
269
|
+
buf = db_get(vn, k)
|
270
|
+
return nil unless buf
|
271
|
+
|
272
|
+
# buf != nil
|
273
|
+
vn, t, clk, expt2, v2 = unpack_data(buf)
|
274
|
+
return nil if Time.now.to_i > expt2
|
275
|
+
clk = (clk + 1) & 0xffffffff
|
276
|
+
|
277
|
+
ret = [vn, Time.now.to_i, clk, expt, v + v2]
|
278
|
+
return ret if db_put(vn, k, pack_data(*ret))
|
279
|
+
nil
|
280
|
+
end
|
281
|
+
|
282
|
+
def get(vn, k, d)
|
283
|
+
buf = db_get(vn, k)
|
284
|
+
return nil unless buf
|
285
|
+
vn, t, clk, expt, v = unpack_data(buf)
|
286
|
+
|
287
|
+
return nil if Time.now.to_i > expt
|
288
|
+
v
|
289
|
+
end
|
290
|
+
|
291
|
+
def get_raw(vn, k, d)
|
292
|
+
buf = db_get(vn, k)
|
293
|
+
return nil unless buf
|
294
|
+
|
295
|
+
unpack_data(buf)
|
296
|
+
end
|
297
|
+
|
298
|
+
def get_raw2(k)
|
299
|
+
@hdb.each{|hdb|
|
300
|
+
buf = hdb.get(k)
|
301
|
+
return unpack_data(buf) if buf
|
302
|
+
}
|
303
|
+
nil
|
304
|
+
end
|
305
|
+
|
306
|
+
def rdelete(vn, k, d, clk)
|
307
|
+
buf = db_get(vn, k)
|
308
|
+
t = Time.now.to_i
|
309
|
+
if buf
|
310
|
+
data = unpack_header(buf)
|
311
|
+
if t - data[1] < @logic_clock_expire && cmp_clk(clk,data[2]) <= 0
|
312
|
+
@error_message = "error:#{t-data[1]} < #{@logic_clock_expire} && cmp_clk(#{clk},#{data[2]})<=0"
|
313
|
+
return nil
|
314
|
+
end
|
315
|
+
end
|
316
|
+
|
317
|
+
# [ 0.. 3] vn
|
318
|
+
# [ 4.. 7] physical clock(unix time)
|
319
|
+
# [ 8..11] logical clock
|
320
|
+
# [12..15] exptime(unix time) => 0
|
321
|
+
ret = [vn, t, clk, 0]
|
322
|
+
if db_put(vn, k, pack_header(*ret))
|
323
|
+
return ret
|
324
|
+
else
|
325
|
+
return nil
|
326
|
+
end
|
327
|
+
end
|
328
|
+
|
329
|
+
def delete(vn, k, d)
|
330
|
+
buf = db_get(vn, k)
|
331
|
+
v = ret = nil
|
332
|
+
clk = 0
|
333
|
+
if buf
|
334
|
+
vn, t, clk, expt, v2 = unpack_data(buf)
|
335
|
+
return :deletemark if expt == 0
|
336
|
+
clk = (clk + 1) & 0xffffffff
|
337
|
+
v = v2 if v2 && v2.length != 0 && Time.now.to_i <= expt
|
338
|
+
end
|
339
|
+
|
340
|
+
# [ 0.. 3] vn
|
341
|
+
# [ 4.. 7] physical clock(unix time)
|
342
|
+
# [ 8..11] logical clock
|
343
|
+
# [12..15] exptime(unix time) => 0
|
344
|
+
ret = [vn, Time.now.to_i, clk, 0, v]
|
345
|
+
if db_put(vn, k, pack_header(*ret[0..-2]))
|
346
|
+
return ret
|
347
|
+
else
|
348
|
+
return nil
|
349
|
+
end
|
350
|
+
end
|
351
|
+
|
352
|
+
def out(vn, k, d)
|
353
|
+
@hdb[@hdiv[vn]].out(k)
|
354
|
+
end
|
355
|
+
|
356
|
+
def incr(vn, k, d, v)
|
357
|
+
buf = db_get(vn, k)
|
358
|
+
return nil unless buf
|
359
|
+
|
360
|
+
# buf != nil
|
361
|
+
vn, t, clk, expt2, v2 = unpack_data(buf)
|
362
|
+
return nil if Time.now.to_i > expt2
|
363
|
+
clk = (clk + 1) & 0xffffffff
|
364
|
+
|
365
|
+
v = (v2.to_i + v)
|
366
|
+
v = 0 if v < 0
|
367
|
+
v = v & 0xffffffffffffffff
|
368
|
+
|
369
|
+
ret = [vn, Time.now.to_i, clk, expt2, v.to_s]
|
370
|
+
return ret if db_put(vn, k, pack_data(*ret))
|
371
|
+
nil
|
372
|
+
end
|
373
|
+
|
374
|
+
def decr(vn, k, d, v)
|
375
|
+
buf = db_get(vn, k)
|
376
|
+
return nil unless buf
|
377
|
+
|
378
|
+
# buf != nil
|
379
|
+
vn, t, clk, expt2, v2 = unpack_data(buf)
|
380
|
+
return nil if Time.now.to_i > expt2
|
381
|
+
clk = (clk + 1) & 0xffffffff
|
382
|
+
|
383
|
+
v = (v2.to_i - v)
|
384
|
+
v = 0 if v < 0
|
385
|
+
v = v & 0xffffffffffffffff
|
386
|
+
|
387
|
+
ret = [vn, Time.now.to_i, clk, expt2, v.to_s]
|
388
|
+
return ret if db_put(vn, k, pack_data(*ret))
|
389
|
+
nil
|
390
|
+
end
|
391
|
+
|
392
|
+
# set expire time
|
393
|
+
def set_expt(vn, k, d, expt)
|
394
|
+
buf = db_get(vn, k)
|
395
|
+
if buf
|
396
|
+
vn, t, clk, expt2, v = unpack_data(buf)
|
397
|
+
return nil if Time.now.to_i > expt2
|
398
|
+
clk = (clk + 1) & 0xffffffff
|
399
|
+
ret = [vn, Time.now.to_i, clk, expt, v]
|
400
|
+
return ret if db_put(vn, k, pack_data(*ret))
|
401
|
+
end
|
402
|
+
nil
|
403
|
+
end
|
404
|
+
|
405
|
+
def true_length
|
406
|
+
res = 0
|
407
|
+
@hdb.each{ |hdb| res += hdb.rnum }
|
408
|
+
res
|
409
|
+
end
|
410
|
+
|
411
|
+
def each_clean_up(t, vnhash)
|
412
|
+
@do_clean_up = true
|
413
|
+
|
414
|
+
f = nil;
|
415
|
+
if @cleanup_regexp && File.exist?(@storage_path)
|
416
|
+
f = open(@storage_path + '/klist.txt','w')
|
417
|
+
end
|
418
|
+
|
419
|
+
return unless @each_clean_up_lock.try_lock
|
420
|
+
nt = Time.now.to_i
|
421
|
+
@divnum.times do |i|
|
422
|
+
next if @dbs[i] != :normal
|
423
|
+
hdb = @hdb[i]
|
424
|
+
hdb.each do |k, v|
|
425
|
+
return unless @do_clean_up # 1st check
|
426
|
+
vn, last, clk, expt = unpack_header(v)
|
427
|
+
vn_stat = vnhash[vn]
|
428
|
+
if f && @cleanup_regexp && k =~ /#{@cleanup_regexp}/
|
429
|
+
# write klist
|
430
|
+
f.puts("#{k},#{last},#{clk}") if hdb.get(k) == v
|
431
|
+
end
|
432
|
+
if vn_stat == :primary && ( (expt != 0 && nt > expt) || (expt == 0 && t > last) )
|
433
|
+
if yield k, vn
|
434
|
+
hdb.out(k) if hdb.get(k) == v
|
435
|
+
end
|
436
|
+
elsif vn_stat == nil && t > last
|
437
|
+
if yield k, vn
|
438
|
+
hdb.out(k) if hdb.get(k) == v
|
439
|
+
end
|
440
|
+
end
|
441
|
+
return unless @do_clean_up # 2nd ckeck
|
442
|
+
sleep @each_clean_up_sleep
|
443
|
+
end
|
444
|
+
end
|
445
|
+
ensure
|
446
|
+
@each_clean_up_lock.unlock if @each_clean_up_lock.locked?
|
447
|
+
if f
|
448
|
+
@cleanup_regexp = nil
|
449
|
+
f.close
|
450
|
+
end
|
451
|
+
end
|
452
|
+
|
453
|
+
def stop_clean_up(&block)
|
454
|
+
@do_clean_up = false
|
455
|
+
if block
|
456
|
+
@each_clean_up_lock.lock
|
457
|
+
begin
|
458
|
+
block.call
|
459
|
+
ensure
|
460
|
+
@each_clean_up_lock.unlock
|
461
|
+
end
|
462
|
+
end
|
463
|
+
end
|
464
|
+
|
465
|
+
def load(dmp)
|
466
|
+
n = 0
|
467
|
+
h = Marshal.load(dmp)
|
468
|
+
h.each_pair{ |k, v|
|
469
|
+
# remort data
|
470
|
+
r_vn, r_last, r_clk, r_expt = unpack_header(v)
|
471
|
+
raise "An invalid vnode number is include.key=#{k} vn=#{r_vn}" unless @hdiv.key?(r_vn)
|
472
|
+
local = @hdb[@hdiv[r_vn]].get(k)
|
473
|
+
if local == nil
|
474
|
+
n += 1
|
475
|
+
@hdb[@hdiv[r_vn]].put(k, v)
|
476
|
+
else
|
477
|
+
# local data
|
478
|
+
l_vn, l_last, l_clk, l_expt = unpack_data(local)
|
479
|
+
if r_last - l_last < @logic_clock_expire && cmp_clk(r_clk,l_clk) <= 0
|
480
|
+
else # remort is newer.
|
481
|
+
n += 1
|
482
|
+
@hdb[@hdiv[r_vn]].put(k, v)
|
483
|
+
end
|
484
|
+
end
|
485
|
+
sleep @each_vn_dump_sleep
|
486
|
+
}
|
487
|
+
n
|
488
|
+
end
|
489
|
+
|
490
|
+
def load_stream_dump(vn, last, clk, expt, k, v)
|
491
|
+
buf = db_get(vn, k)
|
492
|
+
if buf
|
493
|
+
data = unpack_header(buf)
|
494
|
+
if last - data[1] < @logic_clock_expire && cmp_clk(clk,data[2]) <= 0
|
495
|
+
return nil
|
496
|
+
end
|
497
|
+
end
|
498
|
+
|
499
|
+
ret = [vn, last, clk, expt, v]
|
500
|
+
if expt == 0
|
501
|
+
# for the deleted mark
|
502
|
+
return ret if db_put(vn, k, pack_header(*ret[0..3]))
|
503
|
+
else
|
504
|
+
return ret if db_put(vn, k, pack_data(*ret))
|
505
|
+
end
|
506
|
+
nil
|
507
|
+
end
|
508
|
+
|
509
|
+
def load_stream_dump_for_cachecleaning(vn, last, clk, expt, k, v)
|
510
|
+
n = @hdiv[vn]
|
511
|
+
buf = @hdb[n].get(k)
|
512
|
+
if buf
|
513
|
+
data = unpack_header(buf)
|
514
|
+
if last - data[1] < @logic_clock_expire && cmp_clk(clk,data[2]) <= 0
|
515
|
+
return nil
|
516
|
+
end
|
517
|
+
end
|
518
|
+
|
519
|
+
ret = [vn, last, clk, expt, v]
|
520
|
+
if expt == 0
|
521
|
+
# for the deleted mark
|
522
|
+
return ret if @hdb[n].put(k, pack_header(*ret[0..3]))
|
523
|
+
else
|
524
|
+
return ret if @hdb[n].put(k, pack_data(*ret))
|
525
|
+
end
|
526
|
+
nil
|
527
|
+
end
|
528
|
+
|
529
|
+
# Returns the vnode dump.
|
530
|
+
def dump(vn)
|
531
|
+
buf = get_vnode_hash(vn)
|
532
|
+
return nil if buf.length == 0
|
533
|
+
Marshal.dump(buf)
|
534
|
+
end
|
535
|
+
|
536
|
+
def each_vn_dump(target_vn)
|
537
|
+
n = @hdiv[target_vn]
|
538
|
+
@stat_lock.synchronize do
|
539
|
+
return false if @dbs[n] != :normal
|
540
|
+
return false if @each_vn_dump_vnodes.include?(target_vn)
|
541
|
+
@each_vn_dump_vnodes << target_vn
|
542
|
+
end
|
543
|
+
|
544
|
+
begin
|
545
|
+
@do_each_vn_dump = true
|
546
|
+
each_unpacked_db(target_vn, @hdb) do |vn, last, clk, expt, k, val|
|
547
|
+
return unless @do_each_vn_dump
|
548
|
+
yield vn_dump_pack(vn, last, clk, expt, k, val)
|
549
|
+
end
|
550
|
+
ensure
|
551
|
+
@each_vn_dump_vnodes.delete(target_vn)
|
552
|
+
end
|
553
|
+
|
554
|
+
true
|
555
|
+
end
|
556
|
+
|
557
|
+
def vn_dump_pack(vn, last, clk, expt, k, val)
|
558
|
+
if val
|
559
|
+
return [vn, last, clk, expt, k.length, k, val.length, val].pack("NNNNNa#{k.length}Na#{val.length}")
|
560
|
+
else
|
561
|
+
return [vn, last, clk, expt, k.length, k, 0].pack("NNNNNa#{k.length}N")
|
562
|
+
end
|
563
|
+
end
|
564
|
+
private :vn_dump_pack
|
565
|
+
|
566
|
+
def each_unpacked_db(target_vn, db)
|
567
|
+
count = 0
|
568
|
+
tn = Time.now.to_i
|
569
|
+
db[@hdiv[target_vn]].each do |k,v|
|
570
|
+
vn, last, clk, expt, val = unpack_data(v)
|
571
|
+
if vn != target_vn || (expt != 0 && tn > expt)
|
572
|
+
count += 1
|
573
|
+
sleep @each_vn_dump_sleep if count % @each_vn_dump_sleep_count == 0
|
574
|
+
next
|
575
|
+
end
|
576
|
+
yield vn, last, clk, expt, k, val
|
577
|
+
end
|
578
|
+
end
|
579
|
+
private :each_unpacked_db
|
580
|
+
|
581
|
+
def each_hdb_dump(i,except_vnh = nil)
|
582
|
+
count = 0
|
583
|
+
@hdb[i].each{|k,v|
|
584
|
+
vn, last, clk, expt, val = unpack_data(v)
|
585
|
+
if except_vnh && except_vnh.key?(vn) || Time.now.to_i > expt
|
586
|
+
count += 1
|
587
|
+
sleep @each_vn_dump_sleep if count % @each_vn_dump_sleep_count == 0
|
588
|
+
else
|
589
|
+
yield [vn, last, clk, expt, k.length, k, val.length, val].pack("NNNNNa#{k.length}Na#{val.length}")
|
590
|
+
sleep @each_vn_dump_sleep
|
591
|
+
end
|
592
|
+
}
|
593
|
+
end
|
594
|
+
|
595
|
+
# Remove a key for the cache(@hdbc).
|
596
|
+
# +dn+:: number of database
|
597
|
+
# +key+:: key
|
598
|
+
def out_cache(dn, key)
|
599
|
+
@hdbc[dn].out(key)
|
600
|
+
end
|
601
|
+
|
602
|
+
# Calls the geven block,
|
603
|
+
# passes the cache(@hdbc) element.
|
604
|
+
# +dn+:: number of database
|
605
|
+
# +keys+:: key list
|
606
|
+
def each_cache_by_keys(dn, keys)
|
607
|
+
keys.each do |k|
|
608
|
+
v = @hdbc[dn].get(k)
|
609
|
+
vn, last, clk, expt, val = unpack_data(v)
|
610
|
+
yield [vn, last, clk, expt, k, val]
|
611
|
+
end
|
612
|
+
end
|
613
|
+
|
614
|
+
# Calls the geven block,
|
615
|
+
# passes the cache(@hdbc) element as the spushv command data format.
|
616
|
+
# +dn+:: number of database
|
617
|
+
# +keys+:: key list
|
618
|
+
def each_cache_dump_pack(dn, keys)
|
619
|
+
keys.each do |k|
|
620
|
+
v = @hdbc[dn].get(k)
|
621
|
+
vn, last, clk, expt, val = unpack_data(v)
|
622
|
+
yield vn_dump_pack(vn, last, clk, expt, k, val)
|
623
|
+
end
|
624
|
+
end
|
625
|
+
|
626
|
+
# Returns a key array in a cache(@hdbc).
|
627
|
+
# +dn+:: number of database
|
628
|
+
# +kn+:: number of keys which is return value
|
629
|
+
def get_keys_in_cache(dn, kn=100)
|
630
|
+
return nil if @do_each_vn_dump
|
631
|
+
ret = []
|
632
|
+
return ret unless @hdbc[dn]
|
633
|
+
count = 0
|
634
|
+
@each_cache_lock.synchronize do
|
635
|
+
@hdbc[dn].each do |k, v|
|
636
|
+
ret << k
|
637
|
+
break if (count+=1) >= kn
|
638
|
+
end
|
639
|
+
end
|
640
|
+
ret
|
641
|
+
end
|
642
|
+
|
643
|
+
# Create vnode dump.
|
644
|
+
def get_vnode_hash(vn)
|
645
|
+
buf = {}
|
646
|
+
count = 0
|
647
|
+
@hdb[@hdiv[vn]].each{ |k, v|
|
648
|
+
count += 1
|
649
|
+
sleep @each_vn_dump_sleep if count % @each_vn_dump_sleep_count == 0
|
650
|
+
dat = unpack_data(v) #v.unpack('NNNN')
|
651
|
+
buf[k] = v if dat[0] == vn
|
652
|
+
}
|
653
|
+
return buf
|
654
|
+
end
|
655
|
+
private :get_vnode_hash
|
656
|
+
|
657
|
+
def flush_db(dn)
|
658
|
+
@hdb[dn].sync
|
659
|
+
end
|
660
|
+
|
661
|
+
def cache_file_name(dn)
|
662
|
+
"#{@storage_path}/#{dn}.cache.#{@ext_name}"
|
663
|
+
end
|
664
|
+
|
665
|
+
def set_db_stat(dn, stat)
|
666
|
+
@stat_lock.synchronize do
|
667
|
+
case @dbs[dn]
|
668
|
+
when :normal
|
669
|
+
@each_vn_dump_vnodes.each do |vn|
|
670
|
+
return false if dn == @hdiv[vn]
|
671
|
+
end
|
672
|
+
if stat == :safecopy_flushing
|
673
|
+
# open cache
|
674
|
+
@hdbc[dn] = open_db(cache_file_name(dn))
|
675
|
+
stop_clean_up { @dbs[dn] = stat }
|
676
|
+
write_log("#{dn} #{stat.to_s}")
|
677
|
+
stat
|
678
|
+
else
|
679
|
+
false
|
680
|
+
end
|
681
|
+
when :safecopy_flushing
|
682
|
+
if stat == :safecopy_flushed
|
683
|
+
write_log("#{dn} #{stat.to_s}")
|
684
|
+
@dbs[dn] = stat
|
685
|
+
else
|
686
|
+
false
|
687
|
+
end
|
688
|
+
when :safecopy_flushed
|
689
|
+
if stat == :cachecleaning
|
690
|
+
write_log("#{dn} #{stat.to_s}")
|
691
|
+
@dbs[dn] = stat
|
692
|
+
else
|
693
|
+
false
|
694
|
+
end
|
695
|
+
when :cachecleaning
|
696
|
+
if stat == :normal
|
697
|
+
write_log("#{dn} #{stat.to_s}")
|
698
|
+
@dbs[dn] = stat
|
699
|
+
# remove cache
|
700
|
+
close_db(@hdbc[dn])
|
701
|
+
@hdbc[dn] = nil
|
702
|
+
if File.exist?("#{@storage_path}/#{dn}.cache.#{@ext_name}")
|
703
|
+
File.unlink("#{@storage_path}/#{dn}.cache.#{@ext_name}")
|
704
|
+
end
|
705
|
+
stat
|
706
|
+
elsif stat == :safecopy_flushing
|
707
|
+
write_log("#{dn} #{stat.to_s}")
|
708
|
+
@dbs[dn] = stat
|
709
|
+
else
|
710
|
+
false
|
711
|
+
end
|
712
|
+
else
|
713
|
+
false
|
714
|
+
end
|
715
|
+
end
|
716
|
+
end
|
717
|
+
|
718
|
+
def get_logfile_list
|
719
|
+
l={}
|
720
|
+
files=Dir.glob("#{@storage_path}/status.log.*")
|
721
|
+
files.each{ |file|
|
722
|
+
if /$.+status\.log\.(\d+)$/=~file
|
723
|
+
l[$1.to_i]=$&
|
724
|
+
end
|
725
|
+
}
|
726
|
+
# sorted by old order
|
727
|
+
l.to_a.sort{|a,b| a[0]<=>b[0]}
|
728
|
+
end
|
729
|
+
|
730
|
+
def open_log
|
731
|
+
logs = get_logfile_list
|
732
|
+
if logs.length == 0
|
733
|
+
@log_name="#{@storage_path}/status.log.1"
|
734
|
+
else
|
735
|
+
if File::stat("#{@fname}.#{logs.last[0]}").size == 0
|
736
|
+
@log_name="#{@fname}.#{logs.last[0]}"
|
737
|
+
else
|
738
|
+
@log_name="#{@fname}.#{logs.last[0]+1}"
|
739
|
+
end
|
740
|
+
end
|
741
|
+
@log_fd=File.open(@log_name,"a")
|
742
|
+
end
|
743
|
+
|
744
|
+
def write_log(line)
|
745
|
+
return unless @log_name
|
746
|
+
# log rotation
|
747
|
+
if File::stat(@log_name).size > 1000 * 1024
|
748
|
+
close_log
|
749
|
+
open_log
|
750
|
+
end
|
751
|
+
t = Time.now
|
752
|
+
tstr = "#{t.strftime('%Y-%m-%dT%H:%M:%S')}.#{t.usec}"
|
753
|
+
@log_fd.write("#{tstr} #{line}\n")
|
754
|
+
@log_fd.flush
|
755
|
+
end
|
756
|
+
|
757
|
+
def close_log
|
758
|
+
@log_fd.close if @log_fd
|
759
|
+
@log_fd = nil
|
760
|
+
end
|
761
|
+
|
762
|
+
end # class BasicStorage
|
763
|
+
|
764
|
+
end # module Storage
|
765
|
+
end # module Roma
|