perobs 2.5.0 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/perobs/Array.rb +1 -1
- data/lib/perobs/BTree.rb +233 -0
- data/lib/perobs/BTreeDB.rb +2 -0
- data/lib/perobs/BTreeNode.rb +706 -0
- data/lib/perobs/BTreeNodeCache.rb +107 -0
- data/lib/perobs/BTreeNodeLink.rb +141 -0
- data/lib/perobs/EquiBlobsFile.rb +570 -0
- data/lib/perobs/FlatFile.rb +179 -78
- data/lib/perobs/FlatFileBlobHeader.rb +92 -17
- data/lib/perobs/FlatFileDB.rb +16 -7
- data/lib/perobs/LockFile.rb +181 -0
- data/lib/perobs/Object.rb +2 -1
- data/lib/perobs/SpaceTree.rb +181 -0
- data/lib/perobs/SpaceTreeNode.rb +672 -0
- data/lib/perobs/SpaceTreeNodeCache.rb +76 -0
- data/lib/perobs/SpaceTreeNodeLink.rb +103 -0
- data/lib/perobs/Store.rb +27 -13
- data/lib/perobs/version.rb +1 -1
- data/test/BTree_spec.rb +128 -0
- data/test/EquiBlobsFile_spec.rb +199 -0
- data/test/FlatFileDB_spec.rb +63 -9
- data/test/LockFile_spec.rb +133 -0
- data/test/SpaceTree_spec.rb +245 -0
- data/test/Store_spec.rb +3 -0
- data/test/spec_helper.rb +13 -0
- metadata +21 -13
- data/lib/perobs/FixedSizeBlobFile.rb +0 -193
- data/lib/perobs/FreeSpaceManager.rb +0 -204
- data/lib/perobs/IndexTree.rb +0 -145
- data/lib/perobs/IndexTreeNode.rb +0 -316
- data/test/FixedSizeBlobFile_spec.rb +0 -91
- data/test/FreeSpaceManager_spec.rb +0 -91
- data/test/IndexTree_spec.rb +0 -118
@@ -0,0 +1,133 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
#
|
3
|
+
# Copyright (c) 2017 by Chris Schlaeger <chris@taskjuggler.org>
|
4
|
+
#
|
5
|
+
# MIT License
|
6
|
+
#
|
7
|
+
# Permission is hereby granted, free of charge, to any person obtaining
|
8
|
+
# a copy of this software and associated documentation files (the
|
9
|
+
# "Software"), to deal in the Software without restriction, including
|
10
|
+
# without limitation the rights to use, copy, modify, merge, publish,
|
11
|
+
# distribute, sublicense, and/or sell copies of the Software, and to
|
12
|
+
# permit persons to whom the Software is furnished to do so, subject to
|
13
|
+
# the following conditions:
|
14
|
+
#
|
15
|
+
# The above copyright notice and this permission notice shall be
|
16
|
+
# included in all copies or substantial portions of the Software.
|
17
|
+
#
|
18
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
19
|
+
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
20
|
+
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
21
|
+
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
22
|
+
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
23
|
+
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
24
|
+
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
25
|
+
|
26
|
+
require 'spec_helper'
|
27
|
+
|
28
|
+
require 'perobs/LockFile'
|
29
|
+
|
30
|
+
describe PEROBS::LockFile do
|
31
|
+
|
32
|
+
before(:each) do
|
33
|
+
@dir = Dir.mktmpdir('LockFile')
|
34
|
+
@file = File.join(@dir, 'LockFile.lock')
|
35
|
+
end
|
36
|
+
|
37
|
+
after(:each) do
|
38
|
+
FileUtils.rm_rf(@dir)
|
39
|
+
end
|
40
|
+
|
41
|
+
it 'should raise an error if the lock file directory does not exist' do
|
42
|
+
capture_io do
|
43
|
+
expect(PEROBS::LockFile.new('/foo/bar/foobar').lock).to be false
|
44
|
+
end
|
45
|
+
PEROBS.log.open($stderr)
|
46
|
+
end
|
47
|
+
|
48
|
+
it 'should support taking and releasing the lock' do
|
49
|
+
lock = PEROBS::LockFile.new(@file)
|
50
|
+
expect(lock.is_locked?).to be false
|
51
|
+
expect(lock.lock).to be true
|
52
|
+
expect(lock.is_locked?).to be true
|
53
|
+
expect(lock.unlock).to be true
|
54
|
+
end
|
55
|
+
|
56
|
+
it 'should fail the unlock after a forced unlock' do
|
57
|
+
lock = PEROBS::LockFile.new(@file)
|
58
|
+
expect(lock.lock).to be true
|
59
|
+
expect(lock.is_locked?).to be true
|
60
|
+
lock.forced_unlock
|
61
|
+
expect(lock.is_locked?).to be false
|
62
|
+
out = capture_io{ expect(lock.unlock).to be false }
|
63
|
+
expect(out).to include('There is no current lock to release')
|
64
|
+
end
|
65
|
+
|
66
|
+
it 'should fail if the lock is already taken' do
|
67
|
+
lock1 = PEROBS::LockFile.new(@file)
|
68
|
+
expect(lock1.lock).to be true
|
69
|
+
lock2 = PEROBS::LockFile.new(@file)
|
70
|
+
out = capture_io { expect(lock2.lock).to be false }
|
71
|
+
expect(out).to include('due to timeout')
|
72
|
+
expect(lock1.unlock).to be true
|
73
|
+
expect(lock2.lock).to be true
|
74
|
+
expect(lock2.unlock).to be true
|
75
|
+
end
|
76
|
+
|
77
|
+
it 'should wait for the old lockholder' do
|
78
|
+
pid = Process.fork do
|
79
|
+
lock1 = PEROBS::LockFile.new(@file)
|
80
|
+
expect(lock1.lock).to be true
|
81
|
+
sleep 5
|
82
|
+
expect(lock1.unlock).to be true
|
83
|
+
end
|
84
|
+
|
85
|
+
while !File.exist?(@file)
|
86
|
+
sleep 1
|
87
|
+
end
|
88
|
+
lock2 = PEROBS::LockFile.new(@file,
|
89
|
+
{ :max_retries => 100, :pause_secs => 0.5 })
|
90
|
+
expect(lock2.lock).to be true
|
91
|
+
expect(lock2.unlock).to be true
|
92
|
+
Process.wait(pid)
|
93
|
+
end
|
94
|
+
|
95
|
+
it 'should timeout waiting for the old lockholder' do
|
96
|
+
pid = Process.fork do
|
97
|
+
lock1 = PEROBS::LockFile.new(@file)
|
98
|
+
expect(lock1.lock).to be true
|
99
|
+
sleep 3
|
100
|
+
expect(lock1.unlock).to be true
|
101
|
+
end
|
102
|
+
|
103
|
+
while !File.exist?(@file)
|
104
|
+
sleep 1
|
105
|
+
end
|
106
|
+
lock2 = PEROBS::LockFile.new(@file,
|
107
|
+
{ :max_retries => 2, :pause_secs => 0.5 })
|
108
|
+
out = capture_io { expect(lock2.lock).to be false }
|
109
|
+
expect(out).to include('due to timeout')
|
110
|
+
Process.wait(pid)
|
111
|
+
end
|
112
|
+
|
113
|
+
it 'should terminate the old lockholder after timeout' do
|
114
|
+
pid = Process.fork do
|
115
|
+
lock1 = PEROBS::LockFile.new(@file)
|
116
|
+
expect(lock1.lock).to be true
|
117
|
+
# This sleep will be killed
|
118
|
+
sleep 1000
|
119
|
+
end
|
120
|
+
|
121
|
+
while !File.exist?(@file)
|
122
|
+
sleep 1
|
123
|
+
end
|
124
|
+
|
125
|
+
lock2 = PEROBS::LockFile.new(@file, { :timeout_secs => 1 })
|
126
|
+
out = capture_io { expect(lock2.lock).to be true }
|
127
|
+
expect(out).to include('Old lock file found for PID')
|
128
|
+
expect(lock2.unlock).to be true
|
129
|
+
Process.wait(pid)
|
130
|
+
end
|
131
|
+
|
132
|
+
end
|
133
|
+
|
@@ -0,0 +1,245 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
#
|
3
|
+
# Copyright (c) 2016 by Chris Schlaeger <chris@taskjuggler.org>
|
4
|
+
#
|
5
|
+
# MIT License
|
6
|
+
#
|
7
|
+
# Permission is hereby granted, free of charge, to any person obtaining
|
8
|
+
# a copy of this software and associated documentation files (the
|
9
|
+
# "Software"), to deal in the Software without restriction, including
|
10
|
+
# without limitation the rights to use, copy, modify, merge, publish,
|
11
|
+
# distribute, sublicense, and/or sell copies of the Software, and to
|
12
|
+
# permit persons to whom the Software is furnished to do so, subject to
|
13
|
+
# the following conditions:
|
14
|
+
#
|
15
|
+
# The above copyright notice and this permission notice shall be
|
16
|
+
# included in all copies or substantial portions of the Software.
|
17
|
+
#
|
18
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
19
|
+
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
20
|
+
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
21
|
+
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
22
|
+
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
23
|
+
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
24
|
+
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
25
|
+
|
26
|
+
require 'fileutils'
|
27
|
+
|
28
|
+
require 'spec_helper'
|
29
|
+
require 'perobs/SpaceTree'
|
30
|
+
|
31
|
+
describe PEROBS::SpaceTree do
|
32
|
+
|
33
|
+
before(:all) do
|
34
|
+
@db_dir = generate_db_name('SpaceTree')
|
35
|
+
FileUtils.mkdir_p(@db_dir)
|
36
|
+
@m = PEROBS::SpaceTree.new(@db_dir)
|
37
|
+
end
|
38
|
+
|
39
|
+
after(:all) do
|
40
|
+
FileUtils.rm_rf(@db_dir)
|
41
|
+
end
|
42
|
+
|
43
|
+
it 'should open the space tree' do
|
44
|
+
@m.open
|
45
|
+
expect(@m.to_a).to eql([])
|
46
|
+
end
|
47
|
+
|
48
|
+
it 'should support adding spaces' do
|
49
|
+
@m.add_space(80, 8)
|
50
|
+
expect(@m.has_space?(80, 8)).to be true
|
51
|
+
expect(@m.to_a).to eql([[80, 8]])
|
52
|
+
expect(@m.check).to be true
|
53
|
+
@m.add_space(40, 4)
|
54
|
+
expect(@m.has_space?(40, 4)).to be true
|
55
|
+
expect(@m.to_a).to eql([[80, 8], [40, 4]])
|
56
|
+
@m.add_space(20, 2)
|
57
|
+
expect(@m.has_space?(20, 2)).to be true
|
58
|
+
expect(@m.to_a).to eql([[80, 8], [40, 4], [20, 2]])
|
59
|
+
@m.add_space(160, 16)
|
60
|
+
expect(@m.has_space?(160, 16)).to be true
|
61
|
+
expect(@m.to_a).to eql([[80, 8], [40, 4], [20, 2], [160, 16]])
|
62
|
+
@m.add_space(320, 32)
|
63
|
+
expect(@m.has_space?(320, 32)).to be true
|
64
|
+
expect(@m.to_a).to eql([[80, 8], [40, 4], [20, 2], [160, 16], [320, 32]])
|
65
|
+
@m.add_space(100, 10)
|
66
|
+
expect(@m.has_space?(100, 10)).to be true
|
67
|
+
expect(@m.to_a).to eql([[80, 8], [40, 4], [20, 2], [160, 16], [100, 10], [320, 32]])
|
68
|
+
@m.add_space(81, 8)
|
69
|
+
expect(@m.has_space?(81, 8)).to be true
|
70
|
+
expect(@m.to_a).to eql([[80, 8], [40, 4], [20, 2], [81, 8], [160, 16], [100, 10], [320, 32]])
|
71
|
+
expect(@m.check).to be true
|
72
|
+
end
|
73
|
+
|
74
|
+
it 'should convert the tree into a human readable string' do
|
75
|
+
out = <<EOT
|
76
|
+
o-v-1:[80, 8] <2 =7 >4
|
77
|
+
+<-v-2:[40, 4] ^1 <3
|
78
|
+
| +<---3:[20, 2] ^2
|
79
|
+
+=---7:[81, 8] ^1
|
80
|
+
+>-v-4:[160, 16] ^1 <6 >5
|
81
|
+
+<---6:[100, 10] ^4
|
82
|
+
+>---5:[320, 32] ^4
|
83
|
+
EOT
|
84
|
+
expect(@m.to_s).to eql out
|
85
|
+
end
|
86
|
+
|
87
|
+
it 'should keep values over an close/open' do
|
88
|
+
@m.add_space(1, 15)
|
89
|
+
expect(@m.check).to be true
|
90
|
+
@m.close
|
91
|
+
@m.open
|
92
|
+
expect(@m.check).to be true
|
93
|
+
expect(@m.to_a).to eql([[80, 8], [40, 4], [20, 2], [81, 8], [160, 16], [100, 10], [1, 15], [320, 32]])
|
94
|
+
end
|
95
|
+
|
96
|
+
it 'should find the smallest node' do
|
97
|
+
node = @m.instance_variable_get('@root').find_smallest_node
|
98
|
+
expect(node.size).to eql(2)
|
99
|
+
end
|
100
|
+
|
101
|
+
it 'should find the largest node' do
|
102
|
+
node = @m.instance_variable_get('@root').find_largest_node
|
103
|
+
expect(node.size).to eql(32)
|
104
|
+
expect(node.node_address).to eql(5)
|
105
|
+
expect(node.parent.size).to eql(16)
|
106
|
+
end
|
107
|
+
|
108
|
+
it 'should support clearing the data' do
|
109
|
+
@m.clear
|
110
|
+
expect(@m.to_a).to eql([])
|
111
|
+
@m.add_space(1, 1)
|
112
|
+
@m.add_space(2, 2)
|
113
|
+
@m.clear
|
114
|
+
expect(@m.to_a).to eql([])
|
115
|
+
end
|
116
|
+
|
117
|
+
it 'should delete an equal node' do
|
118
|
+
@m.clear
|
119
|
+
add_sizes([ 10, 5, 15, 10 ])
|
120
|
+
expect(@m.to_a).to eql([[0, 10], [1, 5], [3, 10], [2, 15]])
|
121
|
+
expect(@m.get_space(10)).to eql([0, 10])
|
122
|
+
|
123
|
+
@m.clear
|
124
|
+
add_sizes([ 10, 5, 15, 10, 10 ])
|
125
|
+
expect(@m.to_a).to eql([[0, 10], [1, 5], [4, 10], [3, 10], [2, 15]])
|
126
|
+
expect(@m.get_space(10)).to eql([0, 10])
|
127
|
+
expect(@m.get_space(10)).to eql([4, 10])
|
128
|
+
end
|
129
|
+
|
130
|
+
it 'should delete a smaller node' do
|
131
|
+
@m.clear
|
132
|
+
add_sizes([ 10, 5, 3, 7 ])
|
133
|
+
expect(@m.to_a).to eql([[0, 10], [1, 5], [2, 3], [3, 7]])
|
134
|
+
expect(@m.get_space(10)).to eql([0, 10])
|
135
|
+
expect(@m.to_a).to eql([[1, 5], [2, 3], [3, 7]])
|
136
|
+
|
137
|
+
@m.clear
|
138
|
+
add_sizes([ 10, 5, 3 ])
|
139
|
+
expect(@m.to_a).to eql([[0, 10], [1, 5], [2, 3]])
|
140
|
+
expect(@m.get_space(5)).to eql([1, 5])
|
141
|
+
expect(@m.to_a).to eql([[0, 10], [2, 3]])
|
142
|
+
|
143
|
+
@m.clear
|
144
|
+
add_sizes([ 10, 5 ])
|
145
|
+
expect(@m.to_a).to eql([[0, 10], [1, 5]])
|
146
|
+
expect(@m.get_space(5)).to eql([1, 5])
|
147
|
+
expect(@m.to_a).to eql([[0, 10]])
|
148
|
+
|
149
|
+
@m.clear
|
150
|
+
add_sizes([ 10, 5, 3, 7, 5 ])
|
151
|
+
expect(@m.to_a).to eql([[0, 10], [1, 5], [2, 3], [4, 5], [3, 7]])
|
152
|
+
expect(@m.get_space(3)).to eql([2, 3])
|
153
|
+
expect(@m.to_a).to eql([[0, 10], [1, 5], [4, 5], [3, 7]])
|
154
|
+
end
|
155
|
+
|
156
|
+
it 'should delete an larger node' do
|
157
|
+
@m.clear
|
158
|
+
add_sizes([ 10, 15, 13, 17 ])
|
159
|
+
expect(@m.to_a).to eql([[0, 10], [1, 15], [2, 13], [3, 17]])
|
160
|
+
expect(@m.get_space(10)).to eql([0, 10])
|
161
|
+
expect(@m.to_a).to eql([[1, 15], [2, 13], [3, 17]])
|
162
|
+
|
163
|
+
@m.clear
|
164
|
+
add_sizes([ 10, 15, 13 ])
|
165
|
+
expect(@m.to_a).to eql([[0, 10], [1, 15], [2, 13]])
|
166
|
+
expect(@m.get_space(15)).to eql([1, 15])
|
167
|
+
expect(@m.to_a).to eql([[0, 10], [2, 13]])
|
168
|
+
|
169
|
+
@m.clear
|
170
|
+
add_sizes([ 10, 15 ])
|
171
|
+
expect(@m.to_a).to eql([[0, 10], [1, 15]])
|
172
|
+
expect(@m.get_space(15)).to eql([1, 15])
|
173
|
+
expect(@m.to_a).to eql([[0, 10]])
|
174
|
+
|
175
|
+
@m.clear
|
176
|
+
add_sizes([ 10, 5, 15, 20, 17, 22 ])
|
177
|
+
expect(@m.to_a).to eql([[0, 10], [1, 5], [2, 15], [3, 20], [4, 17], [5, 22]])
|
178
|
+
expect(@m.get_space(15)).to eql([2, 15])
|
179
|
+
expect(@m.to_a).to eql([[0, 10], [1, 5], [3, 20], [4, 17], [5, 22]])
|
180
|
+
end
|
181
|
+
|
182
|
+
it 'should move largest of small tree' do
|
183
|
+
@m.clear
|
184
|
+
add_sizes([ 5, 3, 7 ])
|
185
|
+
expect(@m.get_space(5)).to eql([0, 5])
|
186
|
+
expect(@m.check).to be true
|
187
|
+
expect(@m.to_a).to eql([[1, 3], [2, 7]])
|
188
|
+
|
189
|
+
@m.clear
|
190
|
+
add_sizes([ 10, 5, 3, 7, 15, 7 ])
|
191
|
+
expect(@m.to_a).to eql([[0, 10], [1, 5], [2, 3], [3, 7], [5, 7], [4, 15]])
|
192
|
+
expect(@m.get_space(10)).to eql([0, 10])
|
193
|
+
expect(@m.check).to be true
|
194
|
+
expect(@m.to_a).to eql([[3, 7], [1, 5], [2, 3], [5, 7], [4, 15]])
|
195
|
+
|
196
|
+
@m.clear
|
197
|
+
add_sizes([ 10, 5, 3, 7, 15, 7, 6 ])
|
198
|
+
expect(@m.to_a).to eql([[0, 10], [1, 5], [2, 3], [3, 7], [6, 6], [5, 7], [4, 15]])
|
199
|
+
expect(@m.get_space(10)).to eql([0, 10])
|
200
|
+
expect(@m.to_a).to eql([[3, 7], [1, 5], [2, 3], [6, 6], [5, 7], [4, 15]])
|
201
|
+
|
202
|
+
@m.clear
|
203
|
+
add_sizes([ 10, 5, 3, 15, 13, 17 ])
|
204
|
+
expect(@m.get_space(10)).to eql([0, 10])
|
205
|
+
expect(@m.to_a).to eql([[1, 5], [2, 3], [3, 15], [4, 13], [5, 17]])
|
206
|
+
end
|
207
|
+
|
208
|
+
it 'should support a real-world traffic pattern' do
|
209
|
+
address = 0
|
210
|
+
spaces = []
|
211
|
+
0.upto(500) do
|
212
|
+
case rand(3)
|
213
|
+
when 0
|
214
|
+
# Insert new values
|
215
|
+
rand(9).times do
|
216
|
+
size = 20 + rand(5000)
|
217
|
+
@m.add_space(address, size)
|
218
|
+
spaces << [address, size]
|
219
|
+
address += size
|
220
|
+
end
|
221
|
+
when 1
|
222
|
+
# Remove some values
|
223
|
+
rand(7).times do
|
224
|
+
size = 20 + rand(6000)
|
225
|
+
if (space = @m.get_space(size))
|
226
|
+
expect(spaces.include?(space)).to be true
|
227
|
+
spaces.delete(space)
|
228
|
+
end
|
229
|
+
end
|
230
|
+
when 2
|
231
|
+
expect(@m.check).to be true
|
232
|
+
spaces.each do |address, size|
|
233
|
+
expect(@m.has_space?(address, size)).to be true
|
234
|
+
end
|
235
|
+
end
|
236
|
+
end
|
237
|
+
end
|
238
|
+
|
239
|
+
def add_sizes(sizes)
|
240
|
+
sizes.each_with_index do |size, i|
|
241
|
+
@m.add_space(i, size)
|
242
|
+
end
|
243
|
+
end
|
244
|
+
|
245
|
+
end
|
data/test/Store_spec.rb
CHANGED
data/test/spec_helper.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: perobs
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 3.0.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Chris Schlaeger
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-
|
11
|
+
date: 2017-08-10 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -66,25 +66,31 @@ files:
|
|
66
66
|
- Rakefile
|
67
67
|
- lib/perobs.rb
|
68
68
|
- lib/perobs/Array.rb
|
69
|
+
- lib/perobs/BTree.rb
|
69
70
|
- lib/perobs/BTreeBlob.rb
|
70
71
|
- lib/perobs/BTreeDB.rb
|
72
|
+
- lib/perobs/BTreeNode.rb
|
73
|
+
- lib/perobs/BTreeNodeCache.rb
|
74
|
+
- lib/perobs/BTreeNodeLink.rb
|
71
75
|
- lib/perobs/Cache.rb
|
72
76
|
- lib/perobs/ClassMap.rb
|
73
77
|
- lib/perobs/DataBase.rb
|
74
78
|
- lib/perobs/DynamoDB.rb
|
75
|
-
- lib/perobs/
|
79
|
+
- lib/perobs/EquiBlobsFile.rb
|
76
80
|
- lib/perobs/FlatFile.rb
|
77
81
|
- lib/perobs/FlatFileBlobHeader.rb
|
78
82
|
- lib/perobs/FlatFileDB.rb
|
79
|
-
- lib/perobs/FreeSpaceManager.rb
|
80
83
|
- lib/perobs/Handle.rb
|
81
84
|
- lib/perobs/Hash.rb
|
82
|
-
- lib/perobs/
|
83
|
-
- lib/perobs/IndexTreeNode.rb
|
85
|
+
- lib/perobs/LockFile.rb
|
84
86
|
- lib/perobs/Log.rb
|
85
87
|
- lib/perobs/Object.rb
|
86
88
|
- lib/perobs/ObjectBase.rb
|
87
89
|
- lib/perobs/RobustFile.rb
|
90
|
+
- lib/perobs/SpaceTree.rb
|
91
|
+
- lib/perobs/SpaceTreeNode.rb
|
92
|
+
- lib/perobs/SpaceTreeNodeCache.rb
|
93
|
+
- lib/perobs/SpaceTreeNodeLink.rb
|
88
94
|
- lib/perobs/StackFile.rb
|
89
95
|
- lib/perobs/Store.rb
|
90
96
|
- lib/perobs/TreeDB.rb
|
@@ -96,13 +102,14 @@ files:
|
|
96
102
|
- tasks/test.rake
|
97
103
|
- test/Array_spec.rb
|
98
104
|
- test/BTreeDB_spec.rb
|
105
|
+
- test/BTree_spec.rb
|
99
106
|
- test/ClassMap_spec.rb
|
100
|
-
- test/
|
107
|
+
- test/EquiBlobsFile_spec.rb
|
101
108
|
- test/FlatFileDB_spec.rb
|
102
|
-
- test/FreeSpaceManager_spec.rb
|
103
109
|
- test/Hash_spec.rb
|
104
|
-
- test/
|
110
|
+
- test/LockFile_spec.rb
|
105
111
|
- test/Object_spec.rb
|
112
|
+
- test/SpaceTree_spec.rb
|
106
113
|
- test/StackFile_spec.rb
|
107
114
|
- test/Store_spec.rb
|
108
115
|
- test/perobs_spec.rb
|
@@ -127,20 +134,21 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
127
134
|
version: '0'
|
128
135
|
requirements: []
|
129
136
|
rubyforge_project:
|
130
|
-
rubygems_version: 2.2.
|
137
|
+
rubygems_version: 2.2.5
|
131
138
|
signing_key:
|
132
139
|
specification_version: 4
|
133
140
|
summary: Persistent Ruby Object Store
|
134
141
|
test_files:
|
135
142
|
- test/Array_spec.rb
|
136
143
|
- test/BTreeDB_spec.rb
|
144
|
+
- test/BTree_spec.rb
|
137
145
|
- test/ClassMap_spec.rb
|
138
|
-
- test/
|
146
|
+
- test/EquiBlobsFile_spec.rb
|
139
147
|
- test/FlatFileDB_spec.rb
|
140
|
-
- test/FreeSpaceManager_spec.rb
|
141
148
|
- test/Hash_spec.rb
|
142
|
-
- test/
|
149
|
+
- test/LockFile_spec.rb
|
143
150
|
- test/Object_spec.rb
|
151
|
+
- test/SpaceTree_spec.rb
|
144
152
|
- test/StackFile_spec.rb
|
145
153
|
- test/Store_spec.rb
|
146
154
|
- test/perobs_spec.rb
|
@@ -1,193 +0,0 @@
|
|
1
|
-
# encoding: UTF-8
|
2
|
-
#
|
3
|
-
# = FixedSizeBlobFile.rb -- Persistent Ruby Object Store
|
4
|
-
#
|
5
|
-
# Copyright (c) 2016 by Chris Schlaeger <chris@taskjuggler.org>
|
6
|
-
#
|
7
|
-
# MIT License
|
8
|
-
#
|
9
|
-
# Permission is hereby granted, free of charge, to any person obtaining
|
10
|
-
# a copy of this software and associated documentation files (the
|
11
|
-
# "Software"), to deal in the Software without restriction, including
|
12
|
-
# without limitation the rights to use, copy, modify, merge, publish,
|
13
|
-
# distribute, sublicense, and/or sell copies of the Software, and to
|
14
|
-
# permit persons to whom the Software is furnished to do so, subject to
|
15
|
-
# the following conditions:
|
16
|
-
#
|
17
|
-
# The above copyright notice and this permission notice shall be
|
18
|
-
# included in all copies or substantial portions of the Software.
|
19
|
-
#
|
20
|
-
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
21
|
-
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
22
|
-
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
23
|
-
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
24
|
-
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
25
|
-
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
26
|
-
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
27
|
-
|
28
|
-
require 'perobs/Log'
|
29
|
-
require 'perobs/StackFile'
|
30
|
-
|
31
|
-
module PEROBS
|
32
|
-
|
33
|
-
# This class implements persistent storage space for fixed size data blobs.
|
34
|
-
# The blobs can be stored and retrieved and can be deleted again. The
|
35
|
-
# FixedSizeBlobFile manages the storage of the blobs and free storage
|
36
|
-
# spaces. The files grows and shrinks as needed. A blob is referenced by its
|
37
|
-
# address.
|
38
|
-
class FixedSizeBlobFile
|
39
|
-
|
40
|
-
# Create a new stack file in the given directory with the given file name.
|
41
|
-
# @param dir [String] Directory
|
42
|
-
# @param name [String] File name
|
43
|
-
# @param entry_bytes [Fixnum] Number of bytes each entry must have
|
44
|
-
def initialize(dir, name, entry_bytes)
|
45
|
-
@file_name = File.join(dir, name + '.blobs')
|
46
|
-
@entry_bytes = entry_bytes
|
47
|
-
@free_list = StackFile.new(dir, name + '-freelist', 8)
|
48
|
-
@f = nil
|
49
|
-
end
|
50
|
-
|
51
|
-
# Open the blob file.
|
52
|
-
def open
|
53
|
-
begin
|
54
|
-
if File.exist?(@file_name)
|
55
|
-
@f = File.open(@file_name, 'rb+')
|
56
|
-
else
|
57
|
-
@f = File.open(@file_name, 'wb+')
|
58
|
-
end
|
59
|
-
rescue IOError => e
|
60
|
-
PEROBS.log.fatal "Cannot open blob file #{@file_name}: #{e.message}"
|
61
|
-
end
|
62
|
-
unless @f.flock(File::LOCK_NB | File::LOCK_EX)
|
63
|
-
PEROBS.log.fatal 'Database blob file is locked by another process'
|
64
|
-
end
|
65
|
-
@free_list.open
|
66
|
-
end
|
67
|
-
|
68
|
-
# Close the blob file. This method must be called before the program is
|
69
|
-
# terminated to avoid data loss.
|
70
|
-
def close
|
71
|
-
@free_list.close
|
72
|
-
begin
|
73
|
-
@f.flush
|
74
|
-
@f.flock(File::LOCK_UN)
|
75
|
-
@f.close
|
76
|
-
rescue IOError => e
|
77
|
-
PEROBS.log.fatal "Cannot close blob file #{@file_name}: #{e.message}"
|
78
|
-
end
|
79
|
-
end
|
80
|
-
|
81
|
-
# Flush out all unwritten data.
|
82
|
-
def sync
|
83
|
-
@free_list.sync
|
84
|
-
begin
|
85
|
-
@f.sync
|
86
|
-
rescue IOError => e
|
87
|
-
PEROBS.log.fatal "Cannot sync blob file #{@file_name}: #{e.message}"
|
88
|
-
end
|
89
|
-
end
|
90
|
-
|
91
|
-
# Delete all data.
|
92
|
-
def clear
|
93
|
-
@f.truncate(0)
|
94
|
-
@f.flush
|
95
|
-
@free_list.clear
|
96
|
-
end
|
97
|
-
|
98
|
-
def empty?
|
99
|
-
sync
|
100
|
-
@f.size == 0
|
101
|
-
end
|
102
|
-
|
103
|
-
# Return the address of a free blob storage space. Addresses start at 0
|
104
|
-
# and increase linearly.
|
105
|
-
# @return [Fixnum] address of a free blob space
|
106
|
-
def free_address
|
107
|
-
if (bytes = @free_list.pop)
|
108
|
-
# Return an entry from the free list.
|
109
|
-
return bytes.unpack('Q')[0]
|
110
|
-
else
|
111
|
-
# There is currently no free entry. Return the address at the end of
|
112
|
-
# the file.
|
113
|
-
offset_to_address(@f.size)
|
114
|
-
end
|
115
|
-
end
|
116
|
-
|
117
|
-
# Store the given byte blob at the specified address. If the blob space is
|
118
|
-
# already in use the content will be overwritten.
|
119
|
-
# @param address [Fixnum] Address to store the blob
|
120
|
-
# @param bytes [String] bytes to store
|
121
|
-
def store_blob(address, bytes)
|
122
|
-
if bytes.length != @entry_bytes
|
123
|
-
PEROBS.log.fatal "All stack entries must be #{@entry_bytes} " +
|
124
|
-
"long. This entry is #{bytes.length} bytes long."
|
125
|
-
end
|
126
|
-
begin
|
127
|
-
@f.seek(address_to_offset(address))
|
128
|
-
# The first byte is tha flag byte. It's set to 1 for cells that hold a
|
129
|
-
# blob. 0 for empty cells.
|
130
|
-
@f.write([ 1 ].pack('C'))
|
131
|
-
@f.write(bytes)
|
132
|
-
@f.flush
|
133
|
-
rescue IOError => e
|
134
|
-
PEROBS.log.fatal "Cannot store blob at address #{address}: #{e.message}"
|
135
|
-
end
|
136
|
-
end
|
137
|
-
|
138
|
-
# Retrieve a blob from the given address.
|
139
|
-
# @param address [Fixnum] Address to store the blob
|
140
|
-
# @return [String] blob bytes
|
141
|
-
def retrieve_blob(address)
|
142
|
-
begin
|
143
|
-
if (offset = address_to_offset(address)) >= @f.size
|
144
|
-
return nil
|
145
|
-
end
|
146
|
-
|
147
|
-
@f.seek(address_to_offset(address))
|
148
|
-
if (@f.read(1).unpack('C')[0] != 1)
|
149
|
-
return nil
|
150
|
-
end
|
151
|
-
bytes = @f.read(@entry_bytes)
|
152
|
-
rescue IOError => e
|
153
|
-
PEROBS.log.fatal "Cannot retrieve blob at adress #{address}: " +
|
154
|
-
e.message
|
155
|
-
end
|
156
|
-
|
157
|
-
bytes
|
158
|
-
end
|
159
|
-
|
160
|
-
# Delete the blob at the given address.
|
161
|
-
# @param address [Fixnum] Address of blob to delete
|
162
|
-
def delete_blob(address)
|
163
|
-
begin
|
164
|
-
@f.seek(address_to_offset(address))
|
165
|
-
if (@f.read(1).unpack('C')[0] != 1)
|
166
|
-
PEROBS.log.fatal "There is no blob stored at address #{address}"
|
167
|
-
end
|
168
|
-
@f.seek(address_to_offset(address))
|
169
|
-
@f.write([ 0 ].pack('C'))
|
170
|
-
rescue IOError => e
|
171
|
-
PEROBS.log.fatal "Cannot delete blob at address #{address}: " +
|
172
|
-
e.message
|
173
|
-
end
|
174
|
-
# Add the address to the free list.
|
175
|
-
@free_list.push([ address ].pack('Q'))
|
176
|
-
end
|
177
|
-
|
178
|
-
private
|
179
|
-
|
180
|
-
# Translate a blob address to the actual offset in the file.
|
181
|
-
def address_to_offset(address)
|
182
|
-
address * (1 + @entry_bytes)
|
183
|
-
end
|
184
|
-
|
185
|
-
# Translate the file offset to the address of a blob.
|
186
|
-
def offset_to_address(offset)
|
187
|
-
offset / (1 + @entry_bytes)
|
188
|
-
end
|
189
|
-
|
190
|
-
end
|
191
|
-
|
192
|
-
end
|
193
|
-
|