perobs 4.0.0 → 4.4.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/README.md +27 -16
- data/lib/perobs/Array.rb +66 -19
- data/lib/perobs/BTree.rb +106 -15
- data/lib/perobs/BTreeBlob.rb +4 -3
- data/lib/perobs/BTreeDB.rb +5 -4
- data/lib/perobs/BTreeNode.rb +482 -156
- data/lib/perobs/BTreeNodeLink.rb +10 -0
- data/lib/perobs/BigArray.rb +285 -0
- data/lib/perobs/BigArrayNode.rb +1002 -0
- data/lib/perobs/BigHash.rb +246 -0
- data/lib/perobs/BigTree.rb +197 -0
- data/lib/perobs/BigTreeNode.rb +873 -0
- data/lib/perobs/Cache.rb +48 -10
- data/lib/perobs/ConsoleProgressMeter.rb +61 -0
- data/lib/perobs/DataBase.rb +4 -3
- data/lib/perobs/DynamoDB.rb +57 -15
- data/lib/perobs/EquiBlobsFile.rb +155 -50
- data/lib/perobs/FNV_Hash_1a_64.rb +54 -0
- data/lib/perobs/FlatFile.rb +519 -227
- data/lib/perobs/FlatFileBlobHeader.rb +113 -54
- data/lib/perobs/FlatFileDB.rb +49 -23
- data/lib/perobs/FuzzyStringMatcher.rb +175 -0
- data/lib/perobs/Hash.rb +127 -33
- data/lib/perobs/IDList.rb +144 -0
- data/lib/perobs/IDListPage.rb +107 -0
- data/lib/perobs/IDListPageFile.rb +180 -0
- data/lib/perobs/IDListPageRecord.rb +142 -0
- data/lib/perobs/Object.rb +18 -15
- data/lib/perobs/ObjectBase.rb +46 -5
- data/lib/perobs/PersistentObjectCache.rb +57 -68
- data/lib/perobs/PersistentObjectCacheLine.rb +24 -12
- data/lib/perobs/ProgressMeter.rb +97 -0
- data/lib/perobs/SpaceManager.rb +273 -0
- data/lib/perobs/SpaceTree.rb +21 -12
- data/lib/perobs/SpaceTreeNode.rb +53 -61
- data/lib/perobs/Store.rb +264 -145
- data/lib/perobs/version.rb +1 -1
- data/lib/perobs.rb +2 -0
- data/perobs.gemspec +4 -4
- data/test/Array_spec.rb +15 -6
- data/test/BTree_spec.rb +6 -2
- data/test/BigArray_spec.rb +261 -0
- data/test/BigHash_spec.rb +152 -0
- data/test/BigTreeNode_spec.rb +153 -0
- data/test/BigTree_spec.rb +259 -0
- data/test/EquiBlobsFile_spec.rb +105 -1
- data/test/FNV_Hash_1a_64_spec.rb +59 -0
- data/test/FlatFileDB_spec.rb +198 -14
- data/test/FuzzyStringMatcher_spec.rb +261 -0
- data/test/Hash_spec.rb +13 -3
- data/test/IDList_spec.rb +77 -0
- data/test/LegacyDBs/LegacyDB.rb +155 -0
- data/test/LegacyDBs/version_3/class_map.json +1 -0
- data/test/LegacyDBs/version_3/config.json +1 -0
- data/test/LegacyDBs/version_3/database.blobs +0 -0
- data/test/LegacyDBs/version_3/database_spaces.blobs +0 -0
- data/test/LegacyDBs/version_3/index.blobs +0 -0
- data/test/LegacyDBs/version_3/version +1 -0
- data/test/LockFile_spec.rb +9 -6
- data/test/SpaceManager_spec.rb +176 -0
- data/test/SpaceTree_spec.rb +4 -1
- data/test/Store_spec.rb +305 -203
- data/test/spec_helper.rb +9 -4
- metadata +57 -16
- data/lib/perobs/BTreeNodeCache.rb +0 -109
- data/lib/perobs/TreeDB.rb +0 -277
@@ -0,0 +1,259 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
#
|
3
|
+
# Copyright (c) 2016, 2017 by Chris Schlaeger <chris@taskjuggler.org>
|
4
|
+
#
|
5
|
+
# MIT License
|
6
|
+
#
|
7
|
+
# Permission is hereby granted, free of charge, to any person obtaining
|
8
|
+
# a copy of this software and associated documentation files (the
|
9
|
+
# "Software"), to deal in the Software without restriction, including
|
10
|
+
# without limitation the rights to use, copy, modify, merge, publish,
|
11
|
+
# distribute, sublicense, and/or sell copies of the Software, and to
|
12
|
+
# permit persons to whom the Software is furnished to do so, subject to
|
13
|
+
# the following conditions:
|
14
|
+
#
|
15
|
+
# The above copyright notice and this permission notice shall be
|
16
|
+
# included in all copies or substantial portions of the Software.
|
17
|
+
#
|
18
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
19
|
+
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
20
|
+
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
21
|
+
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
22
|
+
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
23
|
+
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
24
|
+
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
25
|
+
|
26
|
+
require 'fileutils'
|
27
|
+
|
28
|
+
require 'spec_helper'
|
29
|
+
require 'perobs/Store'
|
30
|
+
require 'perobs/BigTree'
|
31
|
+
|
32
|
+
describe PEROBS::BigTree do
|
33
|
+
|
34
|
+
ORDER = 7
|
35
|
+
|
36
|
+
before(:all) do
|
37
|
+
@db_name = generate_db_name(__FILE__)
|
38
|
+
@store = PEROBS::Store.new(@db_name)
|
39
|
+
@t = @store.new(PEROBS::BigTree, ORDER)
|
40
|
+
end
|
41
|
+
|
42
|
+
after(:all) do
|
43
|
+
@store.delete_store
|
44
|
+
end
|
45
|
+
|
46
|
+
it 'should be empty' do
|
47
|
+
expect(@t.empty?).to be true
|
48
|
+
expect(@t.length).to eql(0)
|
49
|
+
s = @t.statistics
|
50
|
+
expect(s.leaf_nodes).to eql(1)
|
51
|
+
expect(s.branch_nodes).to eql(0)
|
52
|
+
expect(s.min_depth).to eql(1)
|
53
|
+
expect(s.max_depth).to eql(1)
|
54
|
+
end
|
55
|
+
|
56
|
+
it 'should deal with requests for unknown keys' do
|
57
|
+
expect(@t.has_key?(42)).to be false
|
58
|
+
expect(@t.get(42)).to be_nil
|
59
|
+
end
|
60
|
+
|
61
|
+
it 'should support adding sequential key/value pairs' do
|
62
|
+
0.upto(ORDER ** 3) do |i|
|
63
|
+
@t.insert(i, 3 * i)
|
64
|
+
expect(@t.check).to be true
|
65
|
+
expect(@t.length).to eql(i + 1)
|
66
|
+
expect(@t.has_key?(i)).to be true
|
67
|
+
expect(@t.get(i)).to eql(3 * i)
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
it 'should iterate over the stored key and value pairs' do
|
72
|
+
i = 0
|
73
|
+
@t.each do |k, v|
|
74
|
+
expect(k).to eql(i)
|
75
|
+
expect(v).to eql(3 * i)
|
76
|
+
i += 1
|
77
|
+
end
|
78
|
+
expect(i).to eql(ORDER ** 3 + 1)
|
79
|
+
end
|
80
|
+
|
81
|
+
it 'should iterate in reverse order over the stored key and value pairs' do
|
82
|
+
i = ORDER ** 3
|
83
|
+
@t.reverse_each do |k, v|
|
84
|
+
expect(k).to eql(i)
|
85
|
+
expect(v).to eql(3 * i)
|
86
|
+
i -= 1
|
87
|
+
end
|
88
|
+
expect(i).to eql(-1)
|
89
|
+
end
|
90
|
+
|
91
|
+
it 'should yield the key/value pairs on check' do
|
92
|
+
i = 0
|
93
|
+
@t.check do |k, v|
|
94
|
+
expect(k).to eql(k)
|
95
|
+
expect(v).to eql(3 * k)
|
96
|
+
i += 1
|
97
|
+
end
|
98
|
+
expect(i).to eql(ORDER ** 3 + 1)
|
99
|
+
end
|
100
|
+
|
101
|
+
it 'should support overwriting existing entries' do
|
102
|
+
0.upto(ORDER ** 3) do |i|
|
103
|
+
@t.insert(i, 7 * i)
|
104
|
+
expect(@t.check).to be true
|
105
|
+
expect(@t.length).to eql(ORDER ** 3 + 1)
|
106
|
+
expect(@t.has_key?(i)).to be true
|
107
|
+
expect(@t.get(i)).to eql(7 * i)
|
108
|
+
end
|
109
|
+
end
|
110
|
+
|
111
|
+
it 'should support clearing the tree' do
|
112
|
+
@t.clear
|
113
|
+
expect(@t.check).to be true
|
114
|
+
expect(@t.empty?).to be true
|
115
|
+
expect(@t.length).to eql(0)
|
116
|
+
i = 0
|
117
|
+
@t.each { |k, v| i += 1 }
|
118
|
+
expect(i).to eql(0)
|
119
|
+
end
|
120
|
+
|
121
|
+
it 'should support adding random key/value pairs' do
|
122
|
+
(1..ORDER ** 3).to_a.shuffle.each do |i|
|
123
|
+
@t.insert(i, i * 100)
|
124
|
+
end
|
125
|
+
expect(@t.check).to be true
|
126
|
+
(1..ORDER ** 3).to_a.shuffle.each do |i|
|
127
|
+
expect(@t.get(i)).to eql(i * 100)
|
128
|
+
end
|
129
|
+
end
|
130
|
+
|
131
|
+
it 'should support removing keys in random order' do
|
132
|
+
@t.clear
|
133
|
+
(1..ORDER ** 3).to_a.shuffle.each do |i|
|
134
|
+
@t.insert(i, i * 100)
|
135
|
+
end
|
136
|
+
expect(@t.length).to eql(ORDER ** 3)
|
137
|
+
(1..ORDER ** 3).to_a.shuffle.each do |i|
|
138
|
+
expect(@t.remove(i)).to eql(i * 100)
|
139
|
+
expect(@t.check).to be true
|
140
|
+
end
|
141
|
+
expect(@t.length).to eql(0)
|
142
|
+
end
|
143
|
+
|
144
|
+
it 'should support removing keys in increasing order' do
|
145
|
+
@t.clear
|
146
|
+
(1..ORDER ** 3).to_a.shuffle.each do |i|
|
147
|
+
@t.insert(i, i * 100)
|
148
|
+
end
|
149
|
+
expect(@t.length).to eql(ORDER ** 3)
|
150
|
+
(1..ORDER ** 3).to_a.each do |i|
|
151
|
+
expect(@t.remove(i)).to eql(i * 100)
|
152
|
+
expect(@t.check).to be true
|
153
|
+
end
|
154
|
+
expect(@t.length).to eql(0)
|
155
|
+
end
|
156
|
+
|
157
|
+
it 'should support removing keys in reverse order' do
|
158
|
+
@t.clear
|
159
|
+
(1..ORDER ** 3).to_a.shuffle.each do |i|
|
160
|
+
@t.insert(i, i * 100)
|
161
|
+
end
|
162
|
+
expect(@t.length).to eql(ORDER ** 3)
|
163
|
+
(1..ORDER ** 3).to_a.reverse_each do |i|
|
164
|
+
expect(@t.remove(i)).to eql(i * 100)
|
165
|
+
expect(@t.check).to be true
|
166
|
+
end
|
167
|
+
expect(@t.length).to eql(0)
|
168
|
+
end
|
169
|
+
|
170
|
+
it 'should persist the data' do
|
171
|
+
db_name = generate_db_name(__FILE__ + '_persist')
|
172
|
+
store = PEROBS::Store.new(db_name)
|
173
|
+
store['bigtree'] = t = store.new(PEROBS::BigTree, 4)
|
174
|
+
10.times do |i|
|
175
|
+
t.insert(i, i)
|
176
|
+
end
|
177
|
+
10.times do |i|
|
178
|
+
expect(t.get(i)).to eql(i)
|
179
|
+
end
|
180
|
+
store.exit
|
181
|
+
|
182
|
+
store = PEROBS::Store.new(db_name)
|
183
|
+
t = store['bigtree']
|
184
|
+
10.times do |i|
|
185
|
+
expect(t.get(i)).to eql(i)
|
186
|
+
end
|
187
|
+
store.delete_store
|
188
|
+
end
|
189
|
+
|
190
|
+
it 'should delete all entries matching a condition' do
|
191
|
+
@t.clear
|
192
|
+
(1..50).to_a.shuffle.each do |i|
|
193
|
+
@t.insert(i, i)
|
194
|
+
end
|
195
|
+
@t.delete_if { |k, v| v % 7 == 0 }
|
196
|
+
expect(@t.check).to be true
|
197
|
+
@t.each do |k, v|
|
198
|
+
expect(v % 7).to be > 0, "failed for #{v}"
|
199
|
+
end
|
200
|
+
expect(@t.length).to eql(43)
|
201
|
+
@t.delete_if { |k, v| v % 2 == 0 }
|
202
|
+
expect(@t.check).to be true
|
203
|
+
@t.each do |k, v|
|
204
|
+
expect(v % 2).to be > 0
|
205
|
+
end
|
206
|
+
expect(@t.length).to eql(21)
|
207
|
+
@t.delete_if { |k, v| true }
|
208
|
+
expect(@t.check).to be true
|
209
|
+
expect(@t.empty?).to be true
|
210
|
+
end
|
211
|
+
|
212
|
+
it 'should survive a real-world usage test' do
|
213
|
+
@t.clear
|
214
|
+
ref = {}
|
215
|
+
0.upto(1000) do
|
216
|
+
case rand(5)
|
217
|
+
when 0
|
218
|
+
0.upto(2) do
|
219
|
+
key = rand(100000)
|
220
|
+
value = key * 10
|
221
|
+
@t.insert(key, value)
|
222
|
+
ref[key] = value
|
223
|
+
end
|
224
|
+
when 1
|
225
|
+
if ref.length > 0
|
226
|
+
key = ref.keys[rand(ref.keys.length)]
|
227
|
+
expect(@t.remove(key)).to eql(ref[key])
|
228
|
+
ref.delete(key)
|
229
|
+
end
|
230
|
+
when 2
|
231
|
+
if ref.length > 0
|
232
|
+
0.upto(3) do
|
233
|
+
key = ref.keys[rand(ref.keys.length)]
|
234
|
+
expect(@t.get(key)).to eql(ref[key])
|
235
|
+
end
|
236
|
+
end
|
237
|
+
when 3
|
238
|
+
if ref.length > 0
|
239
|
+
key = ref.keys[rand(ref.keys.length)]
|
240
|
+
value = ref[key] + 1
|
241
|
+
@t.insert(key, value)
|
242
|
+
ref[key] = value
|
243
|
+
end
|
244
|
+
when 4
|
245
|
+
if rand(50) == 0
|
246
|
+
expect(@t.check).to be true
|
247
|
+
end
|
248
|
+
end
|
249
|
+
end
|
250
|
+
|
251
|
+
i = 0
|
252
|
+
@t.each do |k, v|
|
253
|
+
expect(ref[k]).to eql(v)
|
254
|
+
i += 1
|
255
|
+
end
|
256
|
+
expect(i).to eql(ref.length)
|
257
|
+
end
|
258
|
+
|
259
|
+
end
|
data/test/EquiBlobsFile_spec.rb
CHANGED
@@ -27,13 +27,17 @@ require 'fileutils'
|
|
27
27
|
|
28
28
|
require 'spec_helper'
|
29
29
|
require 'perobs/EquiBlobsFile'
|
30
|
+
require 'perobs/ProgressMeter'
|
30
31
|
|
31
32
|
describe PEROBS::EquiBlobsFile do
|
32
33
|
|
33
34
|
before(:all) do
|
35
|
+
PEROBS.log.level = Logger::ERROR
|
36
|
+
PEROBS.log.open($stderr)
|
34
37
|
@db_dir = generate_db_name('EquiBlobsFile')
|
35
38
|
FileUtils.mkdir_p(@db_dir)
|
36
|
-
@bf = PEROBS::EquiBlobsFile.new(@db_dir, 'EquiBlobsFile',
|
39
|
+
@bf = PEROBS::EquiBlobsFile.new(@db_dir, 'EquiBlobsFile',
|
40
|
+
PEROBS::ProgressMeter.new, 8)
|
37
41
|
end
|
38
42
|
|
39
43
|
after(:all) do
|
@@ -134,6 +138,12 @@ describe PEROBS::EquiBlobsFile do
|
|
134
138
|
expect(@bf.check).to be true
|
135
139
|
end
|
136
140
|
|
141
|
+
it 'should support deleting reserved blobs' do
|
142
|
+
adr = @bf.free_address
|
143
|
+
@bf.delete_blob(adr)
|
144
|
+
expect(@bf.check).to be true
|
145
|
+
end
|
146
|
+
|
137
147
|
it 'should support clearing the file' do
|
138
148
|
@bf.clear
|
139
149
|
expect(@bf.total_entries).to eql(0)
|
@@ -191,5 +201,99 @@ describe PEROBS::EquiBlobsFile do
|
|
191
201
|
expect(@bf.check).to be true
|
192
202
|
end
|
193
203
|
|
204
|
+
it 'should support custom offsets' do
|
205
|
+
@bf.close
|
206
|
+
@bf.erase
|
207
|
+
@bf.clear_custom_data
|
208
|
+
@bf.register_custom_data('foo', 42)
|
209
|
+
@bf.register_custom_data('bar', 43)
|
210
|
+
@bf.open
|
211
|
+
expect(@bf.total_entries).to eql(0)
|
212
|
+
expect(@bf.total_spaces).to eql(0)
|
213
|
+
expect(@bf.check).to be true
|
214
|
+
expect(@bf.free_address).to eql(1)
|
215
|
+
@bf.store_blob(1,'11111111')
|
216
|
+
expect(@bf.free_address).to eql(2)
|
217
|
+
@bf.store_blob(2,'22222222')
|
218
|
+
expect(@bf.free_address).to eql(3)
|
219
|
+
@bf.store_blob(3,'33333333')
|
220
|
+
expect(@bf.check).to be true
|
221
|
+
expect(@bf.total_entries).to eql(3)
|
222
|
+
expect(@bf.total_spaces).to eql(0)
|
223
|
+
@bf.delete_blob(2)
|
224
|
+
expect(@bf.check).to be true
|
225
|
+
expect(@bf.total_entries).to eql(2)
|
226
|
+
expect(@bf.total_spaces).to eql(1)
|
227
|
+
expect(@bf.free_address).to eql(2)
|
228
|
+
@bf.store_blob(2,'44444444')
|
229
|
+
expect(@bf.total_entries).to eql(3)
|
230
|
+
expect(@bf.total_spaces).to eql(0)
|
231
|
+
expect(@bf.check).to be true
|
232
|
+
end
|
233
|
+
|
234
|
+
it 'should support a mix of adds and deletes' do
|
235
|
+
@bf.close
|
236
|
+
@bf.erase
|
237
|
+
@bf.clear_custom_data
|
238
|
+
@bf.register_custom_data('foo', 42)
|
239
|
+
@bf.register_custom_data('bar', 43)
|
240
|
+
@bf.open
|
241
|
+
|
242
|
+
entries = {}
|
243
|
+
1000.times do |i|
|
244
|
+
rand(30).times do
|
245
|
+
adr = @bf.free_address
|
246
|
+
expect(entries[adr]).to be nil
|
247
|
+
val = rand(2 ** 64)
|
248
|
+
@bf.store_blob(adr, [ val ].pack('Q'))
|
249
|
+
entries[adr] = val
|
250
|
+
#expect(@bf.check).to be true
|
251
|
+
end
|
252
|
+
|
253
|
+
rand(15).times do
|
254
|
+
unless entries.empty?
|
255
|
+
addresses = entries.keys
|
256
|
+
adr = addresses[rand(addresses.length)]
|
257
|
+
expect(@bf.retrieve_blob(adr).unpack('Q').first).to eql(entries[adr])
|
258
|
+
@bf.delete_blob(adr)
|
259
|
+
entries.delete(adr)
|
260
|
+
#expect(@bf.check).to be true
|
261
|
+
end
|
262
|
+
end
|
263
|
+
|
264
|
+
rand(5).times do
|
265
|
+
unless entries.empty?
|
266
|
+
addresses = entries.keys
|
267
|
+
adr = addresses[rand(addresses.length)]
|
268
|
+
val = rand(2 ** 64)
|
269
|
+
@bf.store_blob(adr, [ val ].pack('Q'))
|
270
|
+
entries[adr] = val
|
271
|
+
#expect(@bf.check).to be true
|
272
|
+
end
|
273
|
+
end
|
274
|
+
|
275
|
+
if rand(100) == 0
|
276
|
+
expect(@bf.check).to be true
|
277
|
+
end
|
278
|
+
|
279
|
+
if rand(100) == 0
|
280
|
+
@bf.first_entry = i
|
281
|
+
@bf.set_custom_data('foo', i)
|
282
|
+
end
|
283
|
+
|
284
|
+
if rand(50) == 0
|
285
|
+
@bf.close
|
286
|
+
@bf.open
|
287
|
+
end
|
288
|
+
|
289
|
+
if rand(500) == 0
|
290
|
+
@bf.clear
|
291
|
+
entries = {}
|
292
|
+
end
|
293
|
+
end
|
294
|
+
|
295
|
+
expect(@bf.check).to be true
|
296
|
+
end
|
297
|
+
|
194
298
|
end
|
195
299
|
|
@@ -0,0 +1,59 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
#
|
3
|
+
# Copyright (c) 2019 by Chris Schlaeger <chris@taskjuggler.org>
|
4
|
+
#
|
5
|
+
# This file contains tests for Array that are similar to the tests for the
|
6
|
+
# Array implementation in MRI. The ideas of these tests were replicated in
|
7
|
+
# this code.
|
8
|
+
#
|
9
|
+
# MIT License
|
10
|
+
#
|
11
|
+
# Permission is hereby granted, free of charge, to any person obtaining
|
12
|
+
# a copy of this software and associated documentation files (the
|
13
|
+
# "Software"), to deal in the Software without restriction, including
|
14
|
+
# without limitation the rights to use, copy, modify, merge, publish,
|
15
|
+
# distribute, sublicense, and/or sell copies of the Software, and to
|
16
|
+
# permit persons to whom the Software is furnished to do so, subject to
|
17
|
+
# the following conditions:
|
18
|
+
#
|
19
|
+
# The above copyright notice and this permission notice shall be
|
20
|
+
# included in all copies or substantial portions of the Software.
|
21
|
+
#
|
22
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
23
|
+
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
24
|
+
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
25
|
+
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
26
|
+
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
27
|
+
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
28
|
+
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
29
|
+
|
30
|
+
require 'spec_helper'
|
31
|
+
|
32
|
+
require 'perobs'
|
33
|
+
|
34
|
+
describe PEROBS::FNV_Hash_1a_64 do
|
35
|
+
|
36
|
+
it 'should generate stable hashes for Strings' do
|
37
|
+
refs = [
|
38
|
+
[ 'foo', 15902901984413996407 ],
|
39
|
+
[ 'foo', 15902901984413996407 ],
|
40
|
+
[ 'bar', 16101355973854746 ],
|
41
|
+
[ 'foobar', 9625390261332436968 ],
|
42
|
+
[ 'PEROBS rocks your application!', 4089220442501866848 ],
|
43
|
+
[ 'Permission is hereby granted, free of charge, to any person ' +
|
44
|
+
'obtaining a copy of this software and associated documentation ' +
|
45
|
+
'files (the "Software"), to deal in the Software without ' +
|
46
|
+
'restriction, including without limitation the rights to use, ' +
|
47
|
+
'copy, modify, merge, publish, distribute, sublicense, and/or ' +
|
48
|
+
'sell copies of the Software, and to permit persons to whom the ' +
|
49
|
+
'Software is furnished to do so, subject to the following conditions:',
|
50
|
+
17637146001033534275 ]
|
51
|
+
]
|
52
|
+
|
53
|
+
refs.each do |v|
|
54
|
+
expect(PEROBS::FNV_Hash_1a_64::digest(v[0])).to eql(v[1])
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
end
|
59
|
+
|
data/test/FlatFileDB_spec.rb
CHANGED
@@ -28,6 +28,7 @@ require 'fileutils'
|
|
28
28
|
require 'spec_helper'
|
29
29
|
require 'perobs/FlatFileDB'
|
30
30
|
require 'perobs/Store'
|
31
|
+
require 'LegacyDBs/LegacyDB'
|
31
32
|
|
32
33
|
class FlatFileDB_O < PEROBS::Object
|
33
34
|
|
@@ -47,7 +48,12 @@ describe PEROBS::FlatFileDB do
|
|
47
48
|
before(:each) do
|
48
49
|
@db_dir = generate_db_name(__FILE__)
|
49
50
|
FileUtils.mkdir_p(@db_dir)
|
50
|
-
@
|
51
|
+
@store_options = {
|
52
|
+
:engine => PEROBS::FlatFileDB,
|
53
|
+
:log => $stderr,
|
54
|
+
:log_level => Logger::ERROR
|
55
|
+
}
|
56
|
+
@store = PEROBS::Store.new(@db_dir, @store_options)
|
51
57
|
end
|
52
58
|
|
53
59
|
after(:each) do
|
@@ -65,19 +71,30 @@ describe PEROBS::FlatFileDB do
|
|
65
71
|
expect { db2.open }.to raise_error(PEROBS::FatalError)
|
66
72
|
end
|
67
73
|
|
68
|
-
it 'should do a version upgrade' do
|
74
|
+
it 'should do a version upgrade from version 3' do
|
69
75
|
# Close the store
|
70
|
-
@store['o'] = @store.new(FlatFileDB_O)
|
71
76
|
@store.exit
|
77
|
+
src_dir = File.join(File.dirname(__FILE__), 'LegacyDBs', 'version_3')
|
78
|
+
FileUtils.cp_r(Dir.glob(src_dir + '/*'), @db_dir)
|
72
79
|
|
73
|
-
|
74
|
-
|
75
|
-
|
80
|
+
db = LegacyDB.new(@db_dir)
|
81
|
+
capture_io { db.open }
|
82
|
+
capture_io { expect(db.check).to be true }
|
83
|
+
end
|
76
84
|
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
85
|
+
it 'should do a version upgrade from version 4.1' do
|
86
|
+
# Close the store
|
87
|
+
@store.exit
|
88
|
+
src_dir = File.join(File.dirname(__FILE__), 'LegacyDBs', 'version_4.1')
|
89
|
+
FileUtils.cp_r(Dir.glob(src_dir + '/*'), @db_dir)
|
90
|
+
|
91
|
+
db = LegacyDB.new(@db_dir)
|
92
|
+
capture_io { db.open }
|
93
|
+
capture_io { expect(db.repair).to eql(0) }
|
94
|
+
expect(File.exist?(File.join(@db_dir, 'space_index.blobs'))).to be true
|
95
|
+
expect(File.exist?(File.join(@db_dir, 'space_list.blobs'))).to be true
|
96
|
+
expect(File.exist?(File.join(@db_dir, 'database_spaces.blobs'))).to be false
|
97
|
+
capture_io { expect(db.check).to be true }
|
81
98
|
end
|
82
99
|
|
83
100
|
it 'should refuse a version downgrade' do
|
@@ -89,7 +106,7 @@ describe PEROBS::FlatFileDB do
|
|
89
106
|
File.write(version_file, '1000000')
|
90
107
|
|
91
108
|
# Open the store again
|
92
|
-
expect { PEROBS::Store.new(@db_dir
|
109
|
+
expect { PEROBS::Store.new(@db_dir) }.to raise_error(PEROBS::FatalError)
|
93
110
|
end
|
94
111
|
|
95
112
|
it 'should recover from a lost index file' do
|
@@ -97,7 +114,10 @@ describe PEROBS::FlatFileDB do
|
|
97
114
|
@store.exit
|
98
115
|
|
99
116
|
File.delete(File.join(@db_dir, 'index.blobs'))
|
100
|
-
store =
|
117
|
+
store = nil
|
118
|
+
capture_io do
|
119
|
+
store = PEROBS::Store.new(@db_dir)
|
120
|
+
end
|
101
121
|
expect(store['o'].b).to eql(42)
|
102
122
|
end
|
103
123
|
|
@@ -106,10 +126,174 @@ describe PEROBS::FlatFileDB do
|
|
106
126
|
@store.exit
|
107
127
|
|
108
128
|
File.write(File.join(@db_dir, 'index.blobs'), '*' * 500)
|
109
|
-
store =
|
110
|
-
|
129
|
+
store = nil
|
130
|
+
capture_io do
|
131
|
+
store = PEROBS::Store.new(@db_dir)
|
132
|
+
end
|
133
|
+
capture_io { store.check(true) }
|
111
134
|
expect(store['o'].b).to eql(42)
|
112
135
|
end
|
113
136
|
|
137
|
+
it 'should discard a corrupted blob inside the database.blobs file' do
|
138
|
+
@store.exit
|
139
|
+
|
140
|
+
db = PEROBS::FlatFileDB.new(@db_dir)
|
141
|
+
db_file = File.join(@db_dir, 'database.blobs')
|
142
|
+
db.open
|
143
|
+
0.upto(5) do |i|
|
144
|
+
db.put_object("#{i + 1}:#{'X' * (i + 1) * 30}", i + 1)
|
145
|
+
end
|
146
|
+
db.close
|
147
|
+
db.open
|
148
|
+
0.upto(5) do |i|
|
149
|
+
db.put_object("#{i + 10}:#{'Y' * (i + 1) * 25}", i + 10)
|
150
|
+
end
|
151
|
+
pos = db.instance_variable_get('@flat_file').find_obj_addr_by_id(10)
|
152
|
+
db.close
|
153
|
+
|
154
|
+
f = File.open(db_file, 'rb+')
|
155
|
+
f.seek(pos)
|
156
|
+
f.write('ZZZZZ')
|
157
|
+
f.close
|
158
|
+
|
159
|
+
db.open
|
160
|
+
expect(db.check_db).to eql(1)
|
161
|
+
expect(db.check_db(true)).to eql(1)
|
162
|
+
db.close
|
163
|
+
db = PEROBS::FlatFileDB.new(@db_dir, { :log => $stderr,
|
164
|
+
:log_level => Logger::ERROR })
|
165
|
+
db.open
|
166
|
+
expect(db.check_db).to eql(0)
|
167
|
+
|
168
|
+
0.upto(5) do |i|
|
169
|
+
expect(db.get_object(i + 1)).to eql("#{i + 1}:#{'X' * (i + 1) * 30}")
|
170
|
+
end
|
171
|
+
expect(db.get_object(10)).to be_nil
|
172
|
+
1.upto(5) do |i|
|
173
|
+
expect(db.get_object(i + 10)).to eql("#{i + 10}:#{'Y' * (i + 1) * 25}")
|
174
|
+
end
|
175
|
+
db.close
|
176
|
+
end
|
177
|
+
|
178
|
+
it 'should discard a corrupted blob at the end of the database.blobs file' do
|
179
|
+
@store.exit
|
180
|
+
|
181
|
+
db = PEROBS::FlatFileDB.new(@db_dir)
|
182
|
+
db_file = File.join(@db_dir, 'database.blobs')
|
183
|
+
db.open
|
184
|
+
0.upto(5) do |i|
|
185
|
+
db.put_object("#{i + 1}:#{'X' * (i + 1) * 30}$", i + 1)
|
186
|
+
end
|
187
|
+
db.close
|
188
|
+
|
189
|
+
f = File.truncate(db_file, File.size(db_file) - 20)
|
190
|
+
|
191
|
+
db.open
|
192
|
+
expect(db.check_db).to eql(2)
|
193
|
+
expect(db.check_db(true)).to eql(2)
|
194
|
+
db.close
|
195
|
+
db = PEROBS::FlatFileDB.new(@db_dir, { :log => $stderr,
|
196
|
+
:log_level => Logger::ERROR })
|
197
|
+
db.open
|
198
|
+
expect(db.check_db).to eql(0)
|
199
|
+
|
200
|
+
0.upto(4) do |i|
|
201
|
+
expect(db.get_object(i + 1)).to eql("#{i + 1}:#{'X' * (i + 1) * 30}$")
|
202
|
+
end
|
203
|
+
expect(db.get_object(6)).to be_nil
|
204
|
+
db.close
|
205
|
+
end
|
206
|
+
|
207
|
+
it 'should discard a corrupted header at the end of the database.blobs file' do
|
208
|
+
@store.exit
|
209
|
+
|
210
|
+
db = PEROBS::FlatFileDB.new(@db_dir)
|
211
|
+
db_file = File.join(@db_dir, 'database.blobs')
|
212
|
+
db.open
|
213
|
+
0.upto(5) do |i|
|
214
|
+
db.put_object("#{i + 1}:#{'X' * (i + 1) * 30}$", i + 1)
|
215
|
+
end
|
216
|
+
db.close
|
217
|
+
|
218
|
+
f = File.truncate(db_file, File.size(db_file) - 200)
|
219
|
+
|
220
|
+
db.open
|
221
|
+
expect(db.check_db).to eql(1)
|
222
|
+
expect(db.check_db(true)).to eql(1)
|
223
|
+
db.close
|
224
|
+
db = PEROBS::FlatFileDB.new(@db_dir, { :log => $stderr,
|
225
|
+
:log_level => Logger::ERROR })
|
226
|
+
db.open
|
227
|
+
expect(db.check_db).to eql(0)
|
228
|
+
|
229
|
+
0.upto(4) do |i|
|
230
|
+
expect(db.get_object(i + 1)).to eql("#{i + 1}:#{'X' * (i + 1) * 30}$")
|
231
|
+
end
|
232
|
+
expect(db.get_object(6)).to be_nil
|
233
|
+
db.close
|
234
|
+
end
|
235
|
+
|
236
|
+
it 'should handle a lost blob at the end of the database.blobs file' do
|
237
|
+
@store.exit
|
238
|
+
|
239
|
+
db = PEROBS::FlatFileDB.new(@db_dir)
|
240
|
+
db_file = File.join(@db_dir, 'database.blobs')
|
241
|
+
db.open
|
242
|
+
0.upto(5) do |i|
|
243
|
+
db.put_object("#{i + 1}:#{'X' * (i + 1) * 30}$", i + 1)
|
244
|
+
end
|
245
|
+
db.close
|
246
|
+
|
247
|
+
# This exactly removes the last blob (#6)
|
248
|
+
f = File.truncate(db_file, File.size(db_file) - 210)
|
249
|
+
|
250
|
+
db.open
|
251
|
+
expect(db.check_db).to eql(1)
|
252
|
+
# The repair won't find the missing blob since the blob file is without
|
253
|
+
# errors.
|
254
|
+
expect(db.check_db(true)).to eql(0)
|
255
|
+
db.close
|
256
|
+
db = PEROBS::FlatFileDB.new(@db_dir, { :log => $stderr,
|
257
|
+
:log_level => Logger::ERROR })
|
258
|
+
db.open
|
259
|
+
expect(db.check_db).to eql(0)
|
260
|
+
|
261
|
+
0.upto(4) do |i|
|
262
|
+
expect(db.get_object(i + 1)).to eql("#{i + 1}:#{'X' * (i + 1) * 30}$")
|
263
|
+
end
|
264
|
+
expect(db.get_object(6)).to be_nil
|
265
|
+
db.close
|
266
|
+
end
|
267
|
+
|
268
|
+
it 'should handle duplicate entries for the same ID in database.blobs file' do
|
269
|
+
@store.exit
|
270
|
+
|
271
|
+
db = PEROBS::FlatFileDB.new(@db_dir)
|
272
|
+
db_file = File.join(@db_dir, 'database.blobs')
|
273
|
+
db.open
|
274
|
+
0.upto(5) do |i|
|
275
|
+
db.put_object("#{i + 1}:#{'X' * (i + 1) * 30}$", i + 1)
|
276
|
+
end
|
277
|
+
db.close
|
278
|
+
|
279
|
+
# This appends the entry 2 again
|
280
|
+
blob2 = File.read(db_file, 319 - 199, 199)
|
281
|
+
File.write(db_file, blob2, File.size(db_file))
|
282
|
+
|
283
|
+
db.open
|
284
|
+
expect(db.check_db).to eql(2)
|
285
|
+
expect(db.check_db(true)).to eql(1)
|
286
|
+
db.close
|
287
|
+
db = PEROBS::FlatFileDB.new(@db_dir, { :log => $stderr,
|
288
|
+
:log_level => Logger::WARN })
|
289
|
+
db.open
|
290
|
+
expect(db.check_db).to eql(0)
|
291
|
+
|
292
|
+
0.upto(5) do |i|
|
293
|
+
expect(db.get_object(i + 1)).to eql("#{i + 1}:#{'X' * (i + 1) * 30}$")
|
294
|
+
end
|
295
|
+
db.close
|
296
|
+
end
|
297
|
+
|
114
298
|
end
|
115
299
|
|