perobs 2.5.0 → 3.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/perobs/Array.rb +1 -1
- data/lib/perobs/BTree.rb +233 -0
- data/lib/perobs/BTreeDB.rb +2 -0
- data/lib/perobs/BTreeNode.rb +706 -0
- data/lib/perobs/BTreeNodeCache.rb +107 -0
- data/lib/perobs/BTreeNodeLink.rb +141 -0
- data/lib/perobs/EquiBlobsFile.rb +570 -0
- data/lib/perobs/FlatFile.rb +179 -78
- data/lib/perobs/FlatFileBlobHeader.rb +92 -17
- data/lib/perobs/FlatFileDB.rb +16 -7
- data/lib/perobs/LockFile.rb +181 -0
- data/lib/perobs/Object.rb +2 -1
- data/lib/perobs/SpaceTree.rb +181 -0
- data/lib/perobs/SpaceTreeNode.rb +672 -0
- data/lib/perobs/SpaceTreeNodeCache.rb +76 -0
- data/lib/perobs/SpaceTreeNodeLink.rb +103 -0
- data/lib/perobs/Store.rb +27 -13
- data/lib/perobs/version.rb +1 -1
- data/test/BTree_spec.rb +128 -0
- data/test/EquiBlobsFile_spec.rb +199 -0
- data/test/FlatFileDB_spec.rb +63 -9
- data/test/LockFile_spec.rb +133 -0
- data/test/SpaceTree_spec.rb +245 -0
- data/test/Store_spec.rb +3 -0
- data/test/spec_helper.rb +13 -0
- metadata +21 -13
- data/lib/perobs/FixedSizeBlobFile.rb +0 -193
- data/lib/perobs/FreeSpaceManager.rb +0 -204
- data/lib/perobs/IndexTree.rb +0 -145
- data/lib/perobs/IndexTreeNode.rb +0 -316
- data/test/FixedSizeBlobFile_spec.rb +0 -91
- data/test/FreeSpaceManager_spec.rb +0 -91
- data/test/IndexTree_spec.rb +0 -118
@@ -0,0 +1,133 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
#
|
3
|
+
# Copyright (c) 2017 by Chris Schlaeger <chris@taskjuggler.org>
|
4
|
+
#
|
5
|
+
# MIT License
|
6
|
+
#
|
7
|
+
# Permission is hereby granted, free of charge, to any person obtaining
|
8
|
+
# a copy of this software and associated documentation files (the
|
9
|
+
# "Software"), to deal in the Software without restriction, including
|
10
|
+
# without limitation the rights to use, copy, modify, merge, publish,
|
11
|
+
# distribute, sublicense, and/or sell copies of the Software, and to
|
12
|
+
# permit persons to whom the Software is furnished to do so, subject to
|
13
|
+
# the following conditions:
|
14
|
+
#
|
15
|
+
# The above copyright notice and this permission notice shall be
|
16
|
+
# included in all copies or substantial portions of the Software.
|
17
|
+
#
|
18
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
19
|
+
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
20
|
+
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
21
|
+
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
22
|
+
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
23
|
+
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
24
|
+
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
25
|
+
|
26
|
+
require 'spec_helper'
|
27
|
+
|
28
|
+
require 'perobs/LockFile'
|
29
|
+
|
30
|
+
describe PEROBS::LockFile do
|
31
|
+
|
32
|
+
before(:each) do
|
33
|
+
@dir = Dir.mktmpdir('LockFile')
|
34
|
+
@file = File.join(@dir, 'LockFile.lock')
|
35
|
+
end
|
36
|
+
|
37
|
+
after(:each) do
|
38
|
+
FileUtils.rm_rf(@dir)
|
39
|
+
end
|
40
|
+
|
41
|
+
it 'should raise an error if the lock file directory does not exist' do
|
42
|
+
capture_io do
|
43
|
+
expect(PEROBS::LockFile.new('/foo/bar/foobar').lock).to be false
|
44
|
+
end
|
45
|
+
PEROBS.log.open($stderr)
|
46
|
+
end
|
47
|
+
|
48
|
+
it 'should support taking and releasing the lock' do
|
49
|
+
lock = PEROBS::LockFile.new(@file)
|
50
|
+
expect(lock.is_locked?).to be false
|
51
|
+
expect(lock.lock).to be true
|
52
|
+
expect(lock.is_locked?).to be true
|
53
|
+
expect(lock.unlock).to be true
|
54
|
+
end
|
55
|
+
|
56
|
+
it 'should fail the unlock after a forced unlock' do
|
57
|
+
lock = PEROBS::LockFile.new(@file)
|
58
|
+
expect(lock.lock).to be true
|
59
|
+
expect(lock.is_locked?).to be true
|
60
|
+
lock.forced_unlock
|
61
|
+
expect(lock.is_locked?).to be false
|
62
|
+
out = capture_io{ expect(lock.unlock).to be false }
|
63
|
+
expect(out).to include('There is no current lock to release')
|
64
|
+
end
|
65
|
+
|
66
|
+
it 'should fail if the lock is already taken' do
|
67
|
+
lock1 = PEROBS::LockFile.new(@file)
|
68
|
+
expect(lock1.lock).to be true
|
69
|
+
lock2 = PEROBS::LockFile.new(@file)
|
70
|
+
out = capture_io { expect(lock2.lock).to be false }
|
71
|
+
expect(out).to include('due to timeout')
|
72
|
+
expect(lock1.unlock).to be true
|
73
|
+
expect(lock2.lock).to be true
|
74
|
+
expect(lock2.unlock).to be true
|
75
|
+
end
|
76
|
+
|
77
|
+
it 'should wait for the old lockholder' do
|
78
|
+
pid = Process.fork do
|
79
|
+
lock1 = PEROBS::LockFile.new(@file)
|
80
|
+
expect(lock1.lock).to be true
|
81
|
+
sleep 5
|
82
|
+
expect(lock1.unlock).to be true
|
83
|
+
end
|
84
|
+
|
85
|
+
while !File.exist?(@file)
|
86
|
+
sleep 1
|
87
|
+
end
|
88
|
+
lock2 = PEROBS::LockFile.new(@file,
|
89
|
+
{ :max_retries => 100, :pause_secs => 0.5 })
|
90
|
+
expect(lock2.lock).to be true
|
91
|
+
expect(lock2.unlock).to be true
|
92
|
+
Process.wait(pid)
|
93
|
+
end
|
94
|
+
|
95
|
+
it 'should timeout waiting for the old lockholder' do
|
96
|
+
pid = Process.fork do
|
97
|
+
lock1 = PEROBS::LockFile.new(@file)
|
98
|
+
expect(lock1.lock).to be true
|
99
|
+
sleep 3
|
100
|
+
expect(lock1.unlock).to be true
|
101
|
+
end
|
102
|
+
|
103
|
+
while !File.exist?(@file)
|
104
|
+
sleep 1
|
105
|
+
end
|
106
|
+
lock2 = PEROBS::LockFile.new(@file,
|
107
|
+
{ :max_retries => 2, :pause_secs => 0.5 })
|
108
|
+
out = capture_io { expect(lock2.lock).to be false }
|
109
|
+
expect(out).to include('due to timeout')
|
110
|
+
Process.wait(pid)
|
111
|
+
end
|
112
|
+
|
113
|
+
it 'should terminate the old lockholder after timeout' do
|
114
|
+
pid = Process.fork do
|
115
|
+
lock1 = PEROBS::LockFile.new(@file)
|
116
|
+
expect(lock1.lock).to be true
|
117
|
+
# This sleep will be killed
|
118
|
+
sleep 1000
|
119
|
+
end
|
120
|
+
|
121
|
+
while !File.exist?(@file)
|
122
|
+
sleep 1
|
123
|
+
end
|
124
|
+
|
125
|
+
lock2 = PEROBS::LockFile.new(@file, { :timeout_secs => 1 })
|
126
|
+
out = capture_io { expect(lock2.lock).to be true }
|
127
|
+
expect(out).to include('Old lock file found for PID')
|
128
|
+
expect(lock2.unlock).to be true
|
129
|
+
Process.wait(pid)
|
130
|
+
end
|
131
|
+
|
132
|
+
end
|
133
|
+
|
@@ -0,0 +1,245 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
#
|
3
|
+
# Copyright (c) 2016 by Chris Schlaeger <chris@taskjuggler.org>
|
4
|
+
#
|
5
|
+
# MIT License
|
6
|
+
#
|
7
|
+
# Permission is hereby granted, free of charge, to any person obtaining
|
8
|
+
# a copy of this software and associated documentation files (the
|
9
|
+
# "Software"), to deal in the Software without restriction, including
|
10
|
+
# without limitation the rights to use, copy, modify, merge, publish,
|
11
|
+
# distribute, sublicense, and/or sell copies of the Software, and to
|
12
|
+
# permit persons to whom the Software is furnished to do so, subject to
|
13
|
+
# the following conditions:
|
14
|
+
#
|
15
|
+
# The above copyright notice and this permission notice shall be
|
16
|
+
# included in all copies or substantial portions of the Software.
|
17
|
+
#
|
18
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
19
|
+
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
20
|
+
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
21
|
+
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
22
|
+
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
23
|
+
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
24
|
+
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
25
|
+
|
26
|
+
require 'fileutils'
|
27
|
+
|
28
|
+
require 'spec_helper'
|
29
|
+
require 'perobs/SpaceTree'
|
30
|
+
|
31
|
+
describe PEROBS::SpaceTree do
|
32
|
+
|
33
|
+
before(:all) do
|
34
|
+
@db_dir = generate_db_name('SpaceTree')
|
35
|
+
FileUtils.mkdir_p(@db_dir)
|
36
|
+
@m = PEROBS::SpaceTree.new(@db_dir)
|
37
|
+
end
|
38
|
+
|
39
|
+
after(:all) do
|
40
|
+
FileUtils.rm_rf(@db_dir)
|
41
|
+
end
|
42
|
+
|
43
|
+
it 'should open the space tree' do
|
44
|
+
@m.open
|
45
|
+
expect(@m.to_a).to eql([])
|
46
|
+
end
|
47
|
+
|
48
|
+
it 'should support adding spaces' do
|
49
|
+
@m.add_space(80, 8)
|
50
|
+
expect(@m.has_space?(80, 8)).to be true
|
51
|
+
expect(@m.to_a).to eql([[80, 8]])
|
52
|
+
expect(@m.check).to be true
|
53
|
+
@m.add_space(40, 4)
|
54
|
+
expect(@m.has_space?(40, 4)).to be true
|
55
|
+
expect(@m.to_a).to eql([[80, 8], [40, 4]])
|
56
|
+
@m.add_space(20, 2)
|
57
|
+
expect(@m.has_space?(20, 2)).to be true
|
58
|
+
expect(@m.to_a).to eql([[80, 8], [40, 4], [20, 2]])
|
59
|
+
@m.add_space(160, 16)
|
60
|
+
expect(@m.has_space?(160, 16)).to be true
|
61
|
+
expect(@m.to_a).to eql([[80, 8], [40, 4], [20, 2], [160, 16]])
|
62
|
+
@m.add_space(320, 32)
|
63
|
+
expect(@m.has_space?(320, 32)).to be true
|
64
|
+
expect(@m.to_a).to eql([[80, 8], [40, 4], [20, 2], [160, 16], [320, 32]])
|
65
|
+
@m.add_space(100, 10)
|
66
|
+
expect(@m.has_space?(100, 10)).to be true
|
67
|
+
expect(@m.to_a).to eql([[80, 8], [40, 4], [20, 2], [160, 16], [100, 10], [320, 32]])
|
68
|
+
@m.add_space(81, 8)
|
69
|
+
expect(@m.has_space?(81, 8)).to be true
|
70
|
+
expect(@m.to_a).to eql([[80, 8], [40, 4], [20, 2], [81, 8], [160, 16], [100, 10], [320, 32]])
|
71
|
+
expect(@m.check).to be true
|
72
|
+
end
|
73
|
+
|
74
|
+
it 'should convert the tree into a human readable string' do
|
75
|
+
out = <<EOT
|
76
|
+
o-v-1:[80, 8] <2 =7 >4
|
77
|
+
+<-v-2:[40, 4] ^1 <3
|
78
|
+
| +<---3:[20, 2] ^2
|
79
|
+
+=---7:[81, 8] ^1
|
80
|
+
+>-v-4:[160, 16] ^1 <6 >5
|
81
|
+
+<---6:[100, 10] ^4
|
82
|
+
+>---5:[320, 32] ^4
|
83
|
+
EOT
|
84
|
+
expect(@m.to_s).to eql out
|
85
|
+
end
|
86
|
+
|
87
|
+
it 'should keep values over an close/open' do
|
88
|
+
@m.add_space(1, 15)
|
89
|
+
expect(@m.check).to be true
|
90
|
+
@m.close
|
91
|
+
@m.open
|
92
|
+
expect(@m.check).to be true
|
93
|
+
expect(@m.to_a).to eql([[80, 8], [40, 4], [20, 2], [81, 8], [160, 16], [100, 10], [1, 15], [320, 32]])
|
94
|
+
end
|
95
|
+
|
96
|
+
it 'should find the smallest node' do
|
97
|
+
node = @m.instance_variable_get('@root').find_smallest_node
|
98
|
+
expect(node.size).to eql(2)
|
99
|
+
end
|
100
|
+
|
101
|
+
it 'should find the largest node' do
|
102
|
+
node = @m.instance_variable_get('@root').find_largest_node
|
103
|
+
expect(node.size).to eql(32)
|
104
|
+
expect(node.node_address).to eql(5)
|
105
|
+
expect(node.parent.size).to eql(16)
|
106
|
+
end
|
107
|
+
|
108
|
+
it 'should support clearing the data' do
|
109
|
+
@m.clear
|
110
|
+
expect(@m.to_a).to eql([])
|
111
|
+
@m.add_space(1, 1)
|
112
|
+
@m.add_space(2, 2)
|
113
|
+
@m.clear
|
114
|
+
expect(@m.to_a).to eql([])
|
115
|
+
end
|
116
|
+
|
117
|
+
it 'should delete an equal node' do
|
118
|
+
@m.clear
|
119
|
+
add_sizes([ 10, 5, 15, 10 ])
|
120
|
+
expect(@m.to_a).to eql([[0, 10], [1, 5], [3, 10], [2, 15]])
|
121
|
+
expect(@m.get_space(10)).to eql([0, 10])
|
122
|
+
|
123
|
+
@m.clear
|
124
|
+
add_sizes([ 10, 5, 15, 10, 10 ])
|
125
|
+
expect(@m.to_a).to eql([[0, 10], [1, 5], [4, 10], [3, 10], [2, 15]])
|
126
|
+
expect(@m.get_space(10)).to eql([0, 10])
|
127
|
+
expect(@m.get_space(10)).to eql([4, 10])
|
128
|
+
end
|
129
|
+
|
130
|
+
it 'should delete a smaller node' do
|
131
|
+
@m.clear
|
132
|
+
add_sizes([ 10, 5, 3, 7 ])
|
133
|
+
expect(@m.to_a).to eql([[0, 10], [1, 5], [2, 3], [3, 7]])
|
134
|
+
expect(@m.get_space(10)).to eql([0, 10])
|
135
|
+
expect(@m.to_a).to eql([[1, 5], [2, 3], [3, 7]])
|
136
|
+
|
137
|
+
@m.clear
|
138
|
+
add_sizes([ 10, 5, 3 ])
|
139
|
+
expect(@m.to_a).to eql([[0, 10], [1, 5], [2, 3]])
|
140
|
+
expect(@m.get_space(5)).to eql([1, 5])
|
141
|
+
expect(@m.to_a).to eql([[0, 10], [2, 3]])
|
142
|
+
|
143
|
+
@m.clear
|
144
|
+
add_sizes([ 10, 5 ])
|
145
|
+
expect(@m.to_a).to eql([[0, 10], [1, 5]])
|
146
|
+
expect(@m.get_space(5)).to eql([1, 5])
|
147
|
+
expect(@m.to_a).to eql([[0, 10]])
|
148
|
+
|
149
|
+
@m.clear
|
150
|
+
add_sizes([ 10, 5, 3, 7, 5 ])
|
151
|
+
expect(@m.to_a).to eql([[0, 10], [1, 5], [2, 3], [4, 5], [3, 7]])
|
152
|
+
expect(@m.get_space(3)).to eql([2, 3])
|
153
|
+
expect(@m.to_a).to eql([[0, 10], [1, 5], [4, 5], [3, 7]])
|
154
|
+
end
|
155
|
+
|
156
|
+
it 'should delete an larger node' do
|
157
|
+
@m.clear
|
158
|
+
add_sizes([ 10, 15, 13, 17 ])
|
159
|
+
expect(@m.to_a).to eql([[0, 10], [1, 15], [2, 13], [3, 17]])
|
160
|
+
expect(@m.get_space(10)).to eql([0, 10])
|
161
|
+
expect(@m.to_a).to eql([[1, 15], [2, 13], [3, 17]])
|
162
|
+
|
163
|
+
@m.clear
|
164
|
+
add_sizes([ 10, 15, 13 ])
|
165
|
+
expect(@m.to_a).to eql([[0, 10], [1, 15], [2, 13]])
|
166
|
+
expect(@m.get_space(15)).to eql([1, 15])
|
167
|
+
expect(@m.to_a).to eql([[0, 10], [2, 13]])
|
168
|
+
|
169
|
+
@m.clear
|
170
|
+
add_sizes([ 10, 15 ])
|
171
|
+
expect(@m.to_a).to eql([[0, 10], [1, 15]])
|
172
|
+
expect(@m.get_space(15)).to eql([1, 15])
|
173
|
+
expect(@m.to_a).to eql([[0, 10]])
|
174
|
+
|
175
|
+
@m.clear
|
176
|
+
add_sizes([ 10, 5, 15, 20, 17, 22 ])
|
177
|
+
expect(@m.to_a).to eql([[0, 10], [1, 5], [2, 15], [3, 20], [4, 17], [5, 22]])
|
178
|
+
expect(@m.get_space(15)).to eql([2, 15])
|
179
|
+
expect(@m.to_a).to eql([[0, 10], [1, 5], [3, 20], [4, 17], [5, 22]])
|
180
|
+
end
|
181
|
+
|
182
|
+
it 'should move largest of small tree' do
|
183
|
+
@m.clear
|
184
|
+
add_sizes([ 5, 3, 7 ])
|
185
|
+
expect(@m.get_space(5)).to eql([0, 5])
|
186
|
+
expect(@m.check).to be true
|
187
|
+
expect(@m.to_a).to eql([[1, 3], [2, 7]])
|
188
|
+
|
189
|
+
@m.clear
|
190
|
+
add_sizes([ 10, 5, 3, 7, 15, 7 ])
|
191
|
+
expect(@m.to_a).to eql([[0, 10], [1, 5], [2, 3], [3, 7], [5, 7], [4, 15]])
|
192
|
+
expect(@m.get_space(10)).to eql([0, 10])
|
193
|
+
expect(@m.check).to be true
|
194
|
+
expect(@m.to_a).to eql([[3, 7], [1, 5], [2, 3], [5, 7], [4, 15]])
|
195
|
+
|
196
|
+
@m.clear
|
197
|
+
add_sizes([ 10, 5, 3, 7, 15, 7, 6 ])
|
198
|
+
expect(@m.to_a).to eql([[0, 10], [1, 5], [2, 3], [3, 7], [6, 6], [5, 7], [4, 15]])
|
199
|
+
expect(@m.get_space(10)).to eql([0, 10])
|
200
|
+
expect(@m.to_a).to eql([[3, 7], [1, 5], [2, 3], [6, 6], [5, 7], [4, 15]])
|
201
|
+
|
202
|
+
@m.clear
|
203
|
+
add_sizes([ 10, 5, 3, 15, 13, 17 ])
|
204
|
+
expect(@m.get_space(10)).to eql([0, 10])
|
205
|
+
expect(@m.to_a).to eql([[1, 5], [2, 3], [3, 15], [4, 13], [5, 17]])
|
206
|
+
end
|
207
|
+
|
208
|
+
it 'should support a real-world traffic pattern' do
|
209
|
+
address = 0
|
210
|
+
spaces = []
|
211
|
+
0.upto(500) do
|
212
|
+
case rand(3)
|
213
|
+
when 0
|
214
|
+
# Insert new values
|
215
|
+
rand(9).times do
|
216
|
+
size = 20 + rand(5000)
|
217
|
+
@m.add_space(address, size)
|
218
|
+
spaces << [address, size]
|
219
|
+
address += size
|
220
|
+
end
|
221
|
+
when 1
|
222
|
+
# Remove some values
|
223
|
+
rand(7).times do
|
224
|
+
size = 20 + rand(6000)
|
225
|
+
if (space = @m.get_space(size))
|
226
|
+
expect(spaces.include?(space)).to be true
|
227
|
+
spaces.delete(space)
|
228
|
+
end
|
229
|
+
end
|
230
|
+
when 2
|
231
|
+
expect(@m.check).to be true
|
232
|
+
spaces.each do |address, size|
|
233
|
+
expect(@m.has_space?(address, size)).to be true
|
234
|
+
end
|
235
|
+
end
|
236
|
+
end
|
237
|
+
end
|
238
|
+
|
239
|
+
def add_sizes(sizes)
|
240
|
+
sizes.each_with_index do |size, i|
|
241
|
+
@m.add_space(i, size)
|
242
|
+
end
|
243
|
+
end
|
244
|
+
|
245
|
+
end
|
data/test/Store_spec.rb
CHANGED
data/test/spec_helper.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: perobs
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 3.0.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Chris Schlaeger
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-
|
11
|
+
date: 2017-08-10 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -66,25 +66,31 @@ files:
|
|
66
66
|
- Rakefile
|
67
67
|
- lib/perobs.rb
|
68
68
|
- lib/perobs/Array.rb
|
69
|
+
- lib/perobs/BTree.rb
|
69
70
|
- lib/perobs/BTreeBlob.rb
|
70
71
|
- lib/perobs/BTreeDB.rb
|
72
|
+
- lib/perobs/BTreeNode.rb
|
73
|
+
- lib/perobs/BTreeNodeCache.rb
|
74
|
+
- lib/perobs/BTreeNodeLink.rb
|
71
75
|
- lib/perobs/Cache.rb
|
72
76
|
- lib/perobs/ClassMap.rb
|
73
77
|
- lib/perobs/DataBase.rb
|
74
78
|
- lib/perobs/DynamoDB.rb
|
75
|
-
- lib/perobs/
|
79
|
+
- lib/perobs/EquiBlobsFile.rb
|
76
80
|
- lib/perobs/FlatFile.rb
|
77
81
|
- lib/perobs/FlatFileBlobHeader.rb
|
78
82
|
- lib/perobs/FlatFileDB.rb
|
79
|
-
- lib/perobs/FreeSpaceManager.rb
|
80
83
|
- lib/perobs/Handle.rb
|
81
84
|
- lib/perobs/Hash.rb
|
82
|
-
- lib/perobs/
|
83
|
-
- lib/perobs/IndexTreeNode.rb
|
85
|
+
- lib/perobs/LockFile.rb
|
84
86
|
- lib/perobs/Log.rb
|
85
87
|
- lib/perobs/Object.rb
|
86
88
|
- lib/perobs/ObjectBase.rb
|
87
89
|
- lib/perobs/RobustFile.rb
|
90
|
+
- lib/perobs/SpaceTree.rb
|
91
|
+
- lib/perobs/SpaceTreeNode.rb
|
92
|
+
- lib/perobs/SpaceTreeNodeCache.rb
|
93
|
+
- lib/perobs/SpaceTreeNodeLink.rb
|
88
94
|
- lib/perobs/StackFile.rb
|
89
95
|
- lib/perobs/Store.rb
|
90
96
|
- lib/perobs/TreeDB.rb
|
@@ -96,13 +102,14 @@ files:
|
|
96
102
|
- tasks/test.rake
|
97
103
|
- test/Array_spec.rb
|
98
104
|
- test/BTreeDB_spec.rb
|
105
|
+
- test/BTree_spec.rb
|
99
106
|
- test/ClassMap_spec.rb
|
100
|
-
- test/
|
107
|
+
- test/EquiBlobsFile_spec.rb
|
101
108
|
- test/FlatFileDB_spec.rb
|
102
|
-
- test/FreeSpaceManager_spec.rb
|
103
109
|
- test/Hash_spec.rb
|
104
|
-
- test/
|
110
|
+
- test/LockFile_spec.rb
|
105
111
|
- test/Object_spec.rb
|
112
|
+
- test/SpaceTree_spec.rb
|
106
113
|
- test/StackFile_spec.rb
|
107
114
|
- test/Store_spec.rb
|
108
115
|
- test/perobs_spec.rb
|
@@ -127,20 +134,21 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
127
134
|
version: '0'
|
128
135
|
requirements: []
|
129
136
|
rubyforge_project:
|
130
|
-
rubygems_version: 2.2.
|
137
|
+
rubygems_version: 2.2.5
|
131
138
|
signing_key:
|
132
139
|
specification_version: 4
|
133
140
|
summary: Persistent Ruby Object Store
|
134
141
|
test_files:
|
135
142
|
- test/Array_spec.rb
|
136
143
|
- test/BTreeDB_spec.rb
|
144
|
+
- test/BTree_spec.rb
|
137
145
|
- test/ClassMap_spec.rb
|
138
|
-
- test/
|
146
|
+
- test/EquiBlobsFile_spec.rb
|
139
147
|
- test/FlatFileDB_spec.rb
|
140
|
-
- test/FreeSpaceManager_spec.rb
|
141
148
|
- test/Hash_spec.rb
|
142
|
-
- test/
|
149
|
+
- test/LockFile_spec.rb
|
143
150
|
- test/Object_spec.rb
|
151
|
+
- test/SpaceTree_spec.rb
|
144
152
|
- test/StackFile_spec.rb
|
145
153
|
- test/Store_spec.rb
|
146
154
|
- test/perobs_spec.rb
|
@@ -1,193 +0,0 @@
|
|
1
|
-
# encoding: UTF-8
|
2
|
-
#
|
3
|
-
# = FixedSizeBlobFile.rb -- Persistent Ruby Object Store
|
4
|
-
#
|
5
|
-
# Copyright (c) 2016 by Chris Schlaeger <chris@taskjuggler.org>
|
6
|
-
#
|
7
|
-
# MIT License
|
8
|
-
#
|
9
|
-
# Permission is hereby granted, free of charge, to any person obtaining
|
10
|
-
# a copy of this software and associated documentation files (the
|
11
|
-
# "Software"), to deal in the Software without restriction, including
|
12
|
-
# without limitation the rights to use, copy, modify, merge, publish,
|
13
|
-
# distribute, sublicense, and/or sell copies of the Software, and to
|
14
|
-
# permit persons to whom the Software is furnished to do so, subject to
|
15
|
-
# the following conditions:
|
16
|
-
#
|
17
|
-
# The above copyright notice and this permission notice shall be
|
18
|
-
# included in all copies or substantial portions of the Software.
|
19
|
-
#
|
20
|
-
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
21
|
-
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
22
|
-
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
23
|
-
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
24
|
-
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
25
|
-
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
26
|
-
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
27
|
-
|
28
|
-
require 'perobs/Log'
|
29
|
-
require 'perobs/StackFile'
|
30
|
-
|
31
|
-
module PEROBS
|
32
|
-
|
33
|
-
# This class implements persistent storage space for fixed size data blobs.
|
34
|
-
# The blobs can be stored and retrieved and can be deleted again. The
|
35
|
-
# FixedSizeBlobFile manages the storage of the blobs and free storage
|
36
|
-
# spaces. The files grows and shrinks as needed. A blob is referenced by its
|
37
|
-
# address.
|
38
|
-
class FixedSizeBlobFile
|
39
|
-
|
40
|
-
# Create a new stack file in the given directory with the given file name.
|
41
|
-
# @param dir [String] Directory
|
42
|
-
# @param name [String] File name
|
43
|
-
# @param entry_bytes [Fixnum] Number of bytes each entry must have
|
44
|
-
def initialize(dir, name, entry_bytes)
|
45
|
-
@file_name = File.join(dir, name + '.blobs')
|
46
|
-
@entry_bytes = entry_bytes
|
47
|
-
@free_list = StackFile.new(dir, name + '-freelist', 8)
|
48
|
-
@f = nil
|
49
|
-
end
|
50
|
-
|
51
|
-
# Open the blob file.
|
52
|
-
def open
|
53
|
-
begin
|
54
|
-
if File.exist?(@file_name)
|
55
|
-
@f = File.open(@file_name, 'rb+')
|
56
|
-
else
|
57
|
-
@f = File.open(@file_name, 'wb+')
|
58
|
-
end
|
59
|
-
rescue IOError => e
|
60
|
-
PEROBS.log.fatal "Cannot open blob file #{@file_name}: #{e.message}"
|
61
|
-
end
|
62
|
-
unless @f.flock(File::LOCK_NB | File::LOCK_EX)
|
63
|
-
PEROBS.log.fatal 'Database blob file is locked by another process'
|
64
|
-
end
|
65
|
-
@free_list.open
|
66
|
-
end
|
67
|
-
|
68
|
-
# Close the blob file. This method must be called before the program is
|
69
|
-
# terminated to avoid data loss.
|
70
|
-
def close
|
71
|
-
@free_list.close
|
72
|
-
begin
|
73
|
-
@f.flush
|
74
|
-
@f.flock(File::LOCK_UN)
|
75
|
-
@f.close
|
76
|
-
rescue IOError => e
|
77
|
-
PEROBS.log.fatal "Cannot close blob file #{@file_name}: #{e.message}"
|
78
|
-
end
|
79
|
-
end
|
80
|
-
|
81
|
-
# Flush out all unwritten data.
|
82
|
-
def sync
|
83
|
-
@free_list.sync
|
84
|
-
begin
|
85
|
-
@f.sync
|
86
|
-
rescue IOError => e
|
87
|
-
PEROBS.log.fatal "Cannot sync blob file #{@file_name}: #{e.message}"
|
88
|
-
end
|
89
|
-
end
|
90
|
-
|
91
|
-
# Delete all data.
|
92
|
-
def clear
|
93
|
-
@f.truncate(0)
|
94
|
-
@f.flush
|
95
|
-
@free_list.clear
|
96
|
-
end
|
97
|
-
|
98
|
-
def empty?
|
99
|
-
sync
|
100
|
-
@f.size == 0
|
101
|
-
end
|
102
|
-
|
103
|
-
# Return the address of a free blob storage space. Addresses start at 0
|
104
|
-
# and increase linearly.
|
105
|
-
# @return [Fixnum] address of a free blob space
|
106
|
-
def free_address
|
107
|
-
if (bytes = @free_list.pop)
|
108
|
-
# Return an entry from the free list.
|
109
|
-
return bytes.unpack('Q')[0]
|
110
|
-
else
|
111
|
-
# There is currently no free entry. Return the address at the end of
|
112
|
-
# the file.
|
113
|
-
offset_to_address(@f.size)
|
114
|
-
end
|
115
|
-
end
|
116
|
-
|
117
|
-
# Store the given byte blob at the specified address. If the blob space is
|
118
|
-
# already in use the content will be overwritten.
|
119
|
-
# @param address [Fixnum] Address to store the blob
|
120
|
-
# @param bytes [String] bytes to store
|
121
|
-
def store_blob(address, bytes)
|
122
|
-
if bytes.length != @entry_bytes
|
123
|
-
PEROBS.log.fatal "All stack entries must be #{@entry_bytes} " +
|
124
|
-
"long. This entry is #{bytes.length} bytes long."
|
125
|
-
end
|
126
|
-
begin
|
127
|
-
@f.seek(address_to_offset(address))
|
128
|
-
# The first byte is tha flag byte. It's set to 1 for cells that hold a
|
129
|
-
# blob. 0 for empty cells.
|
130
|
-
@f.write([ 1 ].pack('C'))
|
131
|
-
@f.write(bytes)
|
132
|
-
@f.flush
|
133
|
-
rescue IOError => e
|
134
|
-
PEROBS.log.fatal "Cannot store blob at address #{address}: #{e.message}"
|
135
|
-
end
|
136
|
-
end
|
137
|
-
|
138
|
-
# Retrieve a blob from the given address.
|
139
|
-
# @param address [Fixnum] Address to store the blob
|
140
|
-
# @return [String] blob bytes
|
141
|
-
def retrieve_blob(address)
|
142
|
-
begin
|
143
|
-
if (offset = address_to_offset(address)) >= @f.size
|
144
|
-
return nil
|
145
|
-
end
|
146
|
-
|
147
|
-
@f.seek(address_to_offset(address))
|
148
|
-
if (@f.read(1).unpack('C')[0] != 1)
|
149
|
-
return nil
|
150
|
-
end
|
151
|
-
bytes = @f.read(@entry_bytes)
|
152
|
-
rescue IOError => e
|
153
|
-
PEROBS.log.fatal "Cannot retrieve blob at adress #{address}: " +
|
154
|
-
e.message
|
155
|
-
end
|
156
|
-
|
157
|
-
bytes
|
158
|
-
end
|
159
|
-
|
160
|
-
# Delete the blob at the given address.
|
161
|
-
# @param address [Fixnum] Address of blob to delete
|
162
|
-
def delete_blob(address)
|
163
|
-
begin
|
164
|
-
@f.seek(address_to_offset(address))
|
165
|
-
if (@f.read(1).unpack('C')[0] != 1)
|
166
|
-
PEROBS.log.fatal "There is no blob stored at address #{address}"
|
167
|
-
end
|
168
|
-
@f.seek(address_to_offset(address))
|
169
|
-
@f.write([ 0 ].pack('C'))
|
170
|
-
rescue IOError => e
|
171
|
-
PEROBS.log.fatal "Cannot delete blob at address #{address}: " +
|
172
|
-
e.message
|
173
|
-
end
|
174
|
-
# Add the address to the free list.
|
175
|
-
@free_list.push([ address ].pack('Q'))
|
176
|
-
end
|
177
|
-
|
178
|
-
private
|
179
|
-
|
180
|
-
# Translate a blob address to the actual offset in the file.
|
181
|
-
def address_to_offset(address)
|
182
|
-
address * (1 + @entry_bytes)
|
183
|
-
end
|
184
|
-
|
185
|
-
# Translate the file offset to the address of a blob.
|
186
|
-
def offset_to_address(offset)
|
187
|
-
offset / (1 + @entry_bytes)
|
188
|
-
end
|
189
|
-
|
190
|
-
end
|
191
|
-
|
192
|
-
end
|
193
|
-
|