perobs 4.0.0 → 4.4.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (67) hide show
  1. checksums.yaml +5 -5
  2. data/README.md +27 -16
  3. data/lib/perobs/Array.rb +66 -19
  4. data/lib/perobs/BTree.rb +106 -15
  5. data/lib/perobs/BTreeBlob.rb +4 -3
  6. data/lib/perobs/BTreeDB.rb +5 -4
  7. data/lib/perobs/BTreeNode.rb +482 -156
  8. data/lib/perobs/BTreeNodeLink.rb +10 -0
  9. data/lib/perobs/BigArray.rb +285 -0
  10. data/lib/perobs/BigArrayNode.rb +1002 -0
  11. data/lib/perobs/BigHash.rb +246 -0
  12. data/lib/perobs/BigTree.rb +197 -0
  13. data/lib/perobs/BigTreeNode.rb +873 -0
  14. data/lib/perobs/Cache.rb +48 -10
  15. data/lib/perobs/ConsoleProgressMeter.rb +61 -0
  16. data/lib/perobs/DataBase.rb +4 -3
  17. data/lib/perobs/DynamoDB.rb +57 -15
  18. data/lib/perobs/EquiBlobsFile.rb +155 -50
  19. data/lib/perobs/FNV_Hash_1a_64.rb +54 -0
  20. data/lib/perobs/FlatFile.rb +519 -227
  21. data/lib/perobs/FlatFileBlobHeader.rb +113 -54
  22. data/lib/perobs/FlatFileDB.rb +49 -23
  23. data/lib/perobs/FuzzyStringMatcher.rb +175 -0
  24. data/lib/perobs/Hash.rb +127 -33
  25. data/lib/perobs/IDList.rb +144 -0
  26. data/lib/perobs/IDListPage.rb +107 -0
  27. data/lib/perobs/IDListPageFile.rb +180 -0
  28. data/lib/perobs/IDListPageRecord.rb +142 -0
  29. data/lib/perobs/Object.rb +18 -15
  30. data/lib/perobs/ObjectBase.rb +46 -5
  31. data/lib/perobs/PersistentObjectCache.rb +57 -68
  32. data/lib/perobs/PersistentObjectCacheLine.rb +24 -12
  33. data/lib/perobs/ProgressMeter.rb +97 -0
  34. data/lib/perobs/SpaceManager.rb +273 -0
  35. data/lib/perobs/SpaceTree.rb +21 -12
  36. data/lib/perobs/SpaceTreeNode.rb +53 -61
  37. data/lib/perobs/Store.rb +264 -145
  38. data/lib/perobs/version.rb +1 -1
  39. data/lib/perobs.rb +2 -0
  40. data/perobs.gemspec +4 -4
  41. data/test/Array_spec.rb +15 -6
  42. data/test/BTree_spec.rb +6 -2
  43. data/test/BigArray_spec.rb +261 -0
  44. data/test/BigHash_spec.rb +152 -0
  45. data/test/BigTreeNode_spec.rb +153 -0
  46. data/test/BigTree_spec.rb +259 -0
  47. data/test/EquiBlobsFile_spec.rb +105 -1
  48. data/test/FNV_Hash_1a_64_spec.rb +59 -0
  49. data/test/FlatFileDB_spec.rb +198 -14
  50. data/test/FuzzyStringMatcher_spec.rb +261 -0
  51. data/test/Hash_spec.rb +13 -3
  52. data/test/IDList_spec.rb +77 -0
  53. data/test/LegacyDBs/LegacyDB.rb +155 -0
  54. data/test/LegacyDBs/version_3/class_map.json +1 -0
  55. data/test/LegacyDBs/version_3/config.json +1 -0
  56. data/test/LegacyDBs/version_3/database.blobs +0 -0
  57. data/test/LegacyDBs/version_3/database_spaces.blobs +0 -0
  58. data/test/LegacyDBs/version_3/index.blobs +0 -0
  59. data/test/LegacyDBs/version_3/version +1 -0
  60. data/test/LockFile_spec.rb +9 -6
  61. data/test/SpaceManager_spec.rb +176 -0
  62. data/test/SpaceTree_spec.rb +4 -1
  63. data/test/Store_spec.rb +305 -203
  64. data/test/spec_helper.rb +9 -4
  65. metadata +57 -16
  66. data/lib/perobs/BTreeNodeCache.rb +0 -109
  67. data/lib/perobs/TreeDB.rb +0 -277
@@ -0,0 +1,259 @@
1
+ # encoding: UTF-8
2
+ #
3
+ # Copyright (c) 2016, 2017 by Chris Schlaeger <chris@taskjuggler.org>
4
+ #
5
+ # MIT License
6
+ #
7
+ # Permission is hereby granted, free of charge, to any person obtaining
8
+ # a copy of this software and associated documentation files (the
9
+ # "Software"), to deal in the Software without restriction, including
10
+ # without limitation the rights to use, copy, modify, merge, publish,
11
+ # distribute, sublicense, and/or sell copies of the Software, and to
12
+ # permit persons to whom the Software is furnished to do so, subject to
13
+ # the following conditions:
14
+ #
15
+ # The above copyright notice and this permission notice shall be
16
+ # included in all copies or substantial portions of the Software.
17
+ #
18
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
19
+ # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
20
+ # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
21
+ # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
22
+ # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
23
+ # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
24
+ # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
25
+
26
+ require 'fileutils'
27
+
28
+ require 'spec_helper'
29
+ require 'perobs/Store'
30
+ require 'perobs/BigTree'
31
+
32
+ describe PEROBS::BigTree do
33
+
34
+ ORDER = 7
35
+
36
+ before(:all) do
37
+ @db_name = generate_db_name(__FILE__)
38
+ @store = PEROBS::Store.new(@db_name)
39
+ @t = @store.new(PEROBS::BigTree, ORDER)
40
+ end
41
+
42
+ after(:all) do
43
+ @store.delete_store
44
+ end
45
+
46
+ it 'should be empty' do
47
+ expect(@t.empty?).to be true
48
+ expect(@t.length).to eql(0)
49
+ s = @t.statistics
50
+ expect(s.leaf_nodes).to eql(1)
51
+ expect(s.branch_nodes).to eql(0)
52
+ expect(s.min_depth).to eql(1)
53
+ expect(s.max_depth).to eql(1)
54
+ end
55
+
56
+ it 'should deal with requests for unknown keys' do
57
+ expect(@t.has_key?(42)).to be false
58
+ expect(@t.get(42)).to be_nil
59
+ end
60
+
61
+ it 'should support adding sequential key/value pairs' do
62
+ 0.upto(ORDER ** 3) do |i|
63
+ @t.insert(i, 3 * i)
64
+ expect(@t.check).to be true
65
+ expect(@t.length).to eql(i + 1)
66
+ expect(@t.has_key?(i)).to be true
67
+ expect(@t.get(i)).to eql(3 * i)
68
+ end
69
+ end
70
+
71
+ it 'should iterate over the stored key and value pairs' do
72
+ i = 0
73
+ @t.each do |k, v|
74
+ expect(k).to eql(i)
75
+ expect(v).to eql(3 * i)
76
+ i += 1
77
+ end
78
+ expect(i).to eql(ORDER ** 3 + 1)
79
+ end
80
+
81
+ it 'should iterate in reverse order over the stored key and value pairs' do
82
+ i = ORDER ** 3
83
+ @t.reverse_each do |k, v|
84
+ expect(k).to eql(i)
85
+ expect(v).to eql(3 * i)
86
+ i -= 1
87
+ end
88
+ expect(i).to eql(-1)
89
+ end
90
+
91
+ it 'should yield the key/value pairs on check' do
92
+ i = 0
93
+ @t.check do |k, v|
94
+ expect(k).to eql(k)
95
+ expect(v).to eql(3 * k)
96
+ i += 1
97
+ end
98
+ expect(i).to eql(ORDER ** 3 + 1)
99
+ end
100
+
101
+ it 'should support overwriting existing entries' do
102
+ 0.upto(ORDER ** 3) do |i|
103
+ @t.insert(i, 7 * i)
104
+ expect(@t.check).to be true
105
+ expect(@t.length).to eql(ORDER ** 3 + 1)
106
+ expect(@t.has_key?(i)).to be true
107
+ expect(@t.get(i)).to eql(7 * i)
108
+ end
109
+ end
110
+
111
+ it 'should support clearing the tree' do
112
+ @t.clear
113
+ expect(@t.check).to be true
114
+ expect(@t.empty?).to be true
115
+ expect(@t.length).to eql(0)
116
+ i = 0
117
+ @t.each { |k, v| i += 1 }
118
+ expect(i).to eql(0)
119
+ end
120
+
121
+ it 'should support adding random key/value pairs' do
122
+ (1..ORDER ** 3).to_a.shuffle.each do |i|
123
+ @t.insert(i, i * 100)
124
+ end
125
+ expect(@t.check).to be true
126
+ (1..ORDER ** 3).to_a.shuffle.each do |i|
127
+ expect(@t.get(i)).to eql(i * 100)
128
+ end
129
+ end
130
+
131
+ it 'should support removing keys in random order' do
132
+ @t.clear
133
+ (1..ORDER ** 3).to_a.shuffle.each do |i|
134
+ @t.insert(i, i * 100)
135
+ end
136
+ expect(@t.length).to eql(ORDER ** 3)
137
+ (1..ORDER ** 3).to_a.shuffle.each do |i|
138
+ expect(@t.remove(i)).to eql(i * 100)
139
+ expect(@t.check).to be true
140
+ end
141
+ expect(@t.length).to eql(0)
142
+ end
143
+
144
+ it 'should support removing keys in increasing order' do
145
+ @t.clear
146
+ (1..ORDER ** 3).to_a.shuffle.each do |i|
147
+ @t.insert(i, i * 100)
148
+ end
149
+ expect(@t.length).to eql(ORDER ** 3)
150
+ (1..ORDER ** 3).to_a.each do |i|
151
+ expect(@t.remove(i)).to eql(i * 100)
152
+ expect(@t.check).to be true
153
+ end
154
+ expect(@t.length).to eql(0)
155
+ end
156
+
157
+ it 'should support removing keys in reverse order' do
158
+ @t.clear
159
+ (1..ORDER ** 3).to_a.shuffle.each do |i|
160
+ @t.insert(i, i * 100)
161
+ end
162
+ expect(@t.length).to eql(ORDER ** 3)
163
+ (1..ORDER ** 3).to_a.reverse_each do |i|
164
+ expect(@t.remove(i)).to eql(i * 100)
165
+ expect(@t.check).to be true
166
+ end
167
+ expect(@t.length).to eql(0)
168
+ end
169
+
170
+ it 'should persist the data' do
171
+ db_name = generate_db_name(__FILE__ + '_persist')
172
+ store = PEROBS::Store.new(db_name)
173
+ store['bigtree'] = t = store.new(PEROBS::BigTree, 4)
174
+ 10.times do |i|
175
+ t.insert(i, i)
176
+ end
177
+ 10.times do |i|
178
+ expect(t.get(i)).to eql(i)
179
+ end
180
+ store.exit
181
+
182
+ store = PEROBS::Store.new(db_name)
183
+ t = store['bigtree']
184
+ 10.times do |i|
185
+ expect(t.get(i)).to eql(i)
186
+ end
187
+ store.delete_store
188
+ end
189
+
190
+ it 'should delete all entries matching a condition' do
191
+ @t.clear
192
+ (1..50).to_a.shuffle.each do |i|
193
+ @t.insert(i, i)
194
+ end
195
+ @t.delete_if { |k, v| v % 7 == 0 }
196
+ expect(@t.check).to be true
197
+ @t.each do |k, v|
198
+ expect(v % 7).to be > 0, "failed for #{v}"
199
+ end
200
+ expect(@t.length).to eql(43)
201
+ @t.delete_if { |k, v| v % 2 == 0 }
202
+ expect(@t.check).to be true
203
+ @t.each do |k, v|
204
+ expect(v % 2).to be > 0
205
+ end
206
+ expect(@t.length).to eql(21)
207
+ @t.delete_if { |k, v| true }
208
+ expect(@t.check).to be true
209
+ expect(@t.empty?).to be true
210
+ end
211
+
212
+ it 'should survive a real-world usage test' do
213
+ @t.clear
214
+ ref = {}
215
+ 0.upto(1000) do
216
+ case rand(5)
217
+ when 0
218
+ 0.upto(2) do
219
+ key = rand(100000)
220
+ value = key * 10
221
+ @t.insert(key, value)
222
+ ref[key] = value
223
+ end
224
+ when 1
225
+ if ref.length > 0
226
+ key = ref.keys[rand(ref.keys.length)]
227
+ expect(@t.remove(key)).to eql(ref[key])
228
+ ref.delete(key)
229
+ end
230
+ when 2
231
+ if ref.length > 0
232
+ 0.upto(3) do
233
+ key = ref.keys[rand(ref.keys.length)]
234
+ expect(@t.get(key)).to eql(ref[key])
235
+ end
236
+ end
237
+ when 3
238
+ if ref.length > 0
239
+ key = ref.keys[rand(ref.keys.length)]
240
+ value = ref[key] + 1
241
+ @t.insert(key, value)
242
+ ref[key] = value
243
+ end
244
+ when 4
245
+ if rand(50) == 0
246
+ expect(@t.check).to be true
247
+ end
248
+ end
249
+ end
250
+
251
+ i = 0
252
+ @t.each do |k, v|
253
+ expect(ref[k]).to eql(v)
254
+ i += 1
255
+ end
256
+ expect(i).to eql(ref.length)
257
+ end
258
+
259
+ end
@@ -27,13 +27,17 @@ require 'fileutils'
27
27
 
28
28
  require 'spec_helper'
29
29
  require 'perobs/EquiBlobsFile'
30
+ require 'perobs/ProgressMeter'
30
31
 
31
32
  describe PEROBS::EquiBlobsFile do
32
33
 
33
34
  before(:all) do
35
+ PEROBS.log.level = Logger::ERROR
36
+ PEROBS.log.open($stderr)
34
37
  @db_dir = generate_db_name('EquiBlobsFile')
35
38
  FileUtils.mkdir_p(@db_dir)
36
- @bf = PEROBS::EquiBlobsFile.new(@db_dir, 'EquiBlobsFile', 8)
39
+ @bf = PEROBS::EquiBlobsFile.new(@db_dir, 'EquiBlobsFile',
40
+ PEROBS::ProgressMeter.new, 8)
37
41
  end
38
42
 
39
43
  after(:all) do
@@ -134,6 +138,12 @@ describe PEROBS::EquiBlobsFile do
134
138
  expect(@bf.check).to be true
135
139
  end
136
140
 
141
+ it 'should support deleting reserved blobs' do
142
+ adr = @bf.free_address
143
+ @bf.delete_blob(adr)
144
+ expect(@bf.check).to be true
145
+ end
146
+
137
147
  it 'should support clearing the file' do
138
148
  @bf.clear
139
149
  expect(@bf.total_entries).to eql(0)
@@ -191,5 +201,99 @@ describe PEROBS::EquiBlobsFile do
191
201
  expect(@bf.check).to be true
192
202
  end
193
203
 
204
+ it 'should support custom offsets' do
205
+ @bf.close
206
+ @bf.erase
207
+ @bf.clear_custom_data
208
+ @bf.register_custom_data('foo', 42)
209
+ @bf.register_custom_data('bar', 43)
210
+ @bf.open
211
+ expect(@bf.total_entries).to eql(0)
212
+ expect(@bf.total_spaces).to eql(0)
213
+ expect(@bf.check).to be true
214
+ expect(@bf.free_address).to eql(1)
215
+ @bf.store_blob(1,'11111111')
216
+ expect(@bf.free_address).to eql(2)
217
+ @bf.store_blob(2,'22222222')
218
+ expect(@bf.free_address).to eql(3)
219
+ @bf.store_blob(3,'33333333')
220
+ expect(@bf.check).to be true
221
+ expect(@bf.total_entries).to eql(3)
222
+ expect(@bf.total_spaces).to eql(0)
223
+ @bf.delete_blob(2)
224
+ expect(@bf.check).to be true
225
+ expect(@bf.total_entries).to eql(2)
226
+ expect(@bf.total_spaces).to eql(1)
227
+ expect(@bf.free_address).to eql(2)
228
+ @bf.store_blob(2,'44444444')
229
+ expect(@bf.total_entries).to eql(3)
230
+ expect(@bf.total_spaces).to eql(0)
231
+ expect(@bf.check).to be true
232
+ end
233
+
234
+ it 'should support a mix of adds and deletes' do
235
+ @bf.close
236
+ @bf.erase
237
+ @bf.clear_custom_data
238
+ @bf.register_custom_data('foo', 42)
239
+ @bf.register_custom_data('bar', 43)
240
+ @bf.open
241
+
242
+ entries = {}
243
+ 1000.times do |i|
244
+ rand(30).times do
245
+ adr = @bf.free_address
246
+ expect(entries[adr]).to be nil
247
+ val = rand(2 ** 64)
248
+ @bf.store_blob(adr, [ val ].pack('Q'))
249
+ entries[adr] = val
250
+ #expect(@bf.check).to be true
251
+ end
252
+
253
+ rand(15).times do
254
+ unless entries.empty?
255
+ addresses = entries.keys
256
+ adr = addresses[rand(addresses.length)]
257
+ expect(@bf.retrieve_blob(adr).unpack('Q').first).to eql(entries[adr])
258
+ @bf.delete_blob(adr)
259
+ entries.delete(adr)
260
+ #expect(@bf.check).to be true
261
+ end
262
+ end
263
+
264
+ rand(5).times do
265
+ unless entries.empty?
266
+ addresses = entries.keys
267
+ adr = addresses[rand(addresses.length)]
268
+ val = rand(2 ** 64)
269
+ @bf.store_blob(adr, [ val ].pack('Q'))
270
+ entries[adr] = val
271
+ #expect(@bf.check).to be true
272
+ end
273
+ end
274
+
275
+ if rand(100) == 0
276
+ expect(@bf.check).to be true
277
+ end
278
+
279
+ if rand(100) == 0
280
+ @bf.first_entry = i
281
+ @bf.set_custom_data('foo', i)
282
+ end
283
+
284
+ if rand(50) == 0
285
+ @bf.close
286
+ @bf.open
287
+ end
288
+
289
+ if rand(500) == 0
290
+ @bf.clear
291
+ entries = {}
292
+ end
293
+ end
294
+
295
+ expect(@bf.check).to be true
296
+ end
297
+
194
298
  end
195
299
 
@@ -0,0 +1,59 @@
1
+ # encoding: UTF-8
2
+ #
3
+ # Copyright (c) 2019 by Chris Schlaeger <chris@taskjuggler.org>
4
+ #
5
+ # This file contains tests for Array that are similar to the tests for the
6
+ # Array implementation in MRI. The ideas of these tests were replicated in
7
+ # this code.
8
+ #
9
+ # MIT License
10
+ #
11
+ # Permission is hereby granted, free of charge, to any person obtaining
12
+ # a copy of this software and associated documentation files (the
13
+ # "Software"), to deal in the Software without restriction, including
14
+ # without limitation the rights to use, copy, modify, merge, publish,
15
+ # distribute, sublicense, and/or sell copies of the Software, and to
16
+ # permit persons to whom the Software is furnished to do so, subject to
17
+ # the following conditions:
18
+ #
19
+ # The above copyright notice and this permission notice shall be
20
+ # included in all copies or substantial portions of the Software.
21
+ #
22
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
23
+ # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
24
+ # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
25
+ # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
26
+ # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
27
+ # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
28
+ # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
29
+
30
+ require 'spec_helper'
31
+
32
+ require 'perobs'
33
+
34
+ describe PEROBS::FNV_Hash_1a_64 do
35
+
36
+ it 'should generate stable hashes for Strings' do
37
+ refs = [
38
+ [ 'foo', 15902901984413996407 ],
39
+ [ 'foo', 15902901984413996407 ],
40
+ [ 'bar', 16101355973854746 ],
41
+ [ 'foobar', 9625390261332436968 ],
42
+ [ 'PEROBS rocks your application!', 4089220442501866848 ],
43
+ [ 'Permission is hereby granted, free of charge, to any person ' +
44
+ 'obtaining a copy of this software and associated documentation ' +
45
+ 'files (the "Software"), to deal in the Software without ' +
46
+ 'restriction, including without limitation the rights to use, ' +
47
+ 'copy, modify, merge, publish, distribute, sublicense, and/or ' +
48
+ 'sell copies of the Software, and to permit persons to whom the ' +
49
+ 'Software is furnished to do so, subject to the following conditions:',
50
+ 17637146001033534275 ]
51
+ ]
52
+
53
+ refs.each do |v|
54
+ expect(PEROBS::FNV_Hash_1a_64::digest(v[0])).to eql(v[1])
55
+ end
56
+ end
57
+
58
+ end
59
+
@@ -28,6 +28,7 @@ require 'fileutils'
28
28
  require 'spec_helper'
29
29
  require 'perobs/FlatFileDB'
30
30
  require 'perobs/Store'
31
+ require 'LegacyDBs/LegacyDB'
31
32
 
32
33
  class FlatFileDB_O < PEROBS::Object
33
34
 
@@ -47,7 +48,12 @@ describe PEROBS::FlatFileDB do
47
48
  before(:each) do
48
49
  @db_dir = generate_db_name(__FILE__)
49
50
  FileUtils.mkdir_p(@db_dir)
50
- @store = PEROBS::Store.new(@db_dir, :engine => PEROBS::FlatFileDB)
51
+ @store_options = {
52
+ :engine => PEROBS::FlatFileDB,
53
+ :log => $stderr,
54
+ :log_level => Logger::ERROR
55
+ }
56
+ @store = PEROBS::Store.new(@db_dir, @store_options)
51
57
  end
52
58
 
53
59
  after(:each) do
@@ -65,19 +71,30 @@ describe PEROBS::FlatFileDB do
65
71
  expect { db2.open }.to raise_error(PEROBS::FatalError)
66
72
  end
67
73
 
68
- it 'should do a version upgrade' do
74
+ it 'should do a version upgrade from version 3' do
69
75
  # Close the store
70
- @store['o'] = @store.new(FlatFileDB_O)
71
76
  @store.exit
77
+ src_dir = File.join(File.dirname(__FILE__), 'LegacyDBs', 'version_3')
78
+ FileUtils.cp_r(Dir.glob(src_dir + '/*'), @db_dir)
72
79
 
73
- # Manually downgrade the version file to version 1
74
- version_file = File.join(@db_dir, 'version')
75
- File.write(version_file, '1')
80
+ db = LegacyDB.new(@db_dir)
81
+ capture_io { db.open }
82
+ capture_io { expect(db.check).to be true }
83
+ end
76
84
 
77
- # Open the store again
78
- store = PEROBS::Store.new(@db_dir, :engine => PEROBS::FlatFileDB)
79
- expect(File.read(version_file).to_i).to eql(PEROBS::FlatFileDB::VERSION)
80
- expect(store['o'].b).to eql(42)
85
+ it 'should do a version upgrade from version 4.1' do
86
+ # Close the store
87
+ @store.exit
88
+ src_dir = File.join(File.dirname(__FILE__), 'LegacyDBs', 'version_4.1')
89
+ FileUtils.cp_r(Dir.glob(src_dir + '/*'), @db_dir)
90
+
91
+ db = LegacyDB.new(@db_dir)
92
+ capture_io { db.open }
93
+ capture_io { expect(db.repair).to eql(0) }
94
+ expect(File.exist?(File.join(@db_dir, 'space_index.blobs'))).to be true
95
+ expect(File.exist?(File.join(@db_dir, 'space_list.blobs'))).to be true
96
+ expect(File.exist?(File.join(@db_dir, 'database_spaces.blobs'))).to be false
97
+ capture_io { expect(db.check).to be true }
81
98
  end
82
99
 
83
100
  it 'should refuse a version downgrade' do
@@ -89,7 +106,7 @@ describe PEROBS::FlatFileDB do
89
106
  File.write(version_file, '1000000')
90
107
 
91
108
  # Open the store again
92
- expect { PEROBS::Store.new(@db_dir, :engine => PEROBS::FlatFileDB) }.to raise_error(PEROBS::FatalError)
109
+ expect { PEROBS::Store.new(@db_dir) }.to raise_error(PEROBS::FatalError)
93
110
  end
94
111
 
95
112
  it 'should recover from a lost index file' do
@@ -97,7 +114,10 @@ describe PEROBS::FlatFileDB do
97
114
  @store.exit
98
115
 
99
116
  File.delete(File.join(@db_dir, 'index.blobs'))
100
- store = PEROBS::Store.new(@db_dir, :engine => PEROBS::FlatFileDB)
117
+ store = nil
118
+ capture_io do
119
+ store = PEROBS::Store.new(@db_dir)
120
+ end
101
121
  expect(store['o'].b).to eql(42)
102
122
  end
103
123
 
@@ -106,10 +126,174 @@ describe PEROBS::FlatFileDB do
106
126
  @store.exit
107
127
 
108
128
  File.write(File.join(@db_dir, 'index.blobs'), '*' * 500)
109
- store = PEROBS::Store.new(@db_dir, :engine => PEROBS::FlatFileDB)
110
- store.check(true)
129
+ store = nil
130
+ capture_io do
131
+ store = PEROBS::Store.new(@db_dir)
132
+ end
133
+ capture_io { store.check(true) }
111
134
  expect(store['o'].b).to eql(42)
112
135
  end
113
136
 
137
+ it 'should discard a corrupted blob inside the database.blobs file' do
138
+ @store.exit
139
+
140
+ db = PEROBS::FlatFileDB.new(@db_dir)
141
+ db_file = File.join(@db_dir, 'database.blobs')
142
+ db.open
143
+ 0.upto(5) do |i|
144
+ db.put_object("#{i + 1}:#{'X' * (i + 1) * 30}", i + 1)
145
+ end
146
+ db.close
147
+ db.open
148
+ 0.upto(5) do |i|
149
+ db.put_object("#{i + 10}:#{'Y' * (i + 1) * 25}", i + 10)
150
+ end
151
+ pos = db.instance_variable_get('@flat_file').find_obj_addr_by_id(10)
152
+ db.close
153
+
154
+ f = File.open(db_file, 'rb+')
155
+ f.seek(pos)
156
+ f.write('ZZZZZ')
157
+ f.close
158
+
159
+ db.open
160
+ expect(db.check_db).to eql(1)
161
+ expect(db.check_db(true)).to eql(1)
162
+ db.close
163
+ db = PEROBS::FlatFileDB.new(@db_dir, { :log => $stderr,
164
+ :log_level => Logger::ERROR })
165
+ db.open
166
+ expect(db.check_db).to eql(0)
167
+
168
+ 0.upto(5) do |i|
169
+ expect(db.get_object(i + 1)).to eql("#{i + 1}:#{'X' * (i + 1) * 30}")
170
+ end
171
+ expect(db.get_object(10)).to be_nil
172
+ 1.upto(5) do |i|
173
+ expect(db.get_object(i + 10)).to eql("#{i + 10}:#{'Y' * (i + 1) * 25}")
174
+ end
175
+ db.close
176
+ end
177
+
178
+ it 'should discard a corrupted blob at the end of the database.blobs file' do
179
+ @store.exit
180
+
181
+ db = PEROBS::FlatFileDB.new(@db_dir)
182
+ db_file = File.join(@db_dir, 'database.blobs')
183
+ db.open
184
+ 0.upto(5) do |i|
185
+ db.put_object("#{i + 1}:#{'X' * (i + 1) * 30}$", i + 1)
186
+ end
187
+ db.close
188
+
189
+ f = File.truncate(db_file, File.size(db_file) - 20)
190
+
191
+ db.open
192
+ expect(db.check_db).to eql(2)
193
+ expect(db.check_db(true)).to eql(2)
194
+ db.close
195
+ db = PEROBS::FlatFileDB.new(@db_dir, { :log => $stderr,
196
+ :log_level => Logger::ERROR })
197
+ db.open
198
+ expect(db.check_db).to eql(0)
199
+
200
+ 0.upto(4) do |i|
201
+ expect(db.get_object(i + 1)).to eql("#{i + 1}:#{'X' * (i + 1) * 30}$")
202
+ end
203
+ expect(db.get_object(6)).to be_nil
204
+ db.close
205
+ end
206
+
207
+ it 'should discard a corrupted header at the end of the database.blobs file' do
208
+ @store.exit
209
+
210
+ db = PEROBS::FlatFileDB.new(@db_dir)
211
+ db_file = File.join(@db_dir, 'database.blobs')
212
+ db.open
213
+ 0.upto(5) do |i|
214
+ db.put_object("#{i + 1}:#{'X' * (i + 1) * 30}$", i + 1)
215
+ end
216
+ db.close
217
+
218
+ f = File.truncate(db_file, File.size(db_file) - 200)
219
+
220
+ db.open
221
+ expect(db.check_db).to eql(1)
222
+ expect(db.check_db(true)).to eql(1)
223
+ db.close
224
+ db = PEROBS::FlatFileDB.new(@db_dir, { :log => $stderr,
225
+ :log_level => Logger::ERROR })
226
+ db.open
227
+ expect(db.check_db).to eql(0)
228
+
229
+ 0.upto(4) do |i|
230
+ expect(db.get_object(i + 1)).to eql("#{i + 1}:#{'X' * (i + 1) * 30}$")
231
+ end
232
+ expect(db.get_object(6)).to be_nil
233
+ db.close
234
+ end
235
+
236
+ it 'should handle a lost blob at the end of the database.blobs file' do
237
+ @store.exit
238
+
239
+ db = PEROBS::FlatFileDB.new(@db_dir)
240
+ db_file = File.join(@db_dir, 'database.blobs')
241
+ db.open
242
+ 0.upto(5) do |i|
243
+ db.put_object("#{i + 1}:#{'X' * (i + 1) * 30}$", i + 1)
244
+ end
245
+ db.close
246
+
247
+ # This exactly removes the last blob (#6)
248
+ f = File.truncate(db_file, File.size(db_file) - 210)
249
+
250
+ db.open
251
+ expect(db.check_db).to eql(1)
252
+ # The repair won't find the missing blob since the blob file is without
253
+ # errors.
254
+ expect(db.check_db(true)).to eql(0)
255
+ db.close
256
+ db = PEROBS::FlatFileDB.new(@db_dir, { :log => $stderr,
257
+ :log_level => Logger::ERROR })
258
+ db.open
259
+ expect(db.check_db).to eql(0)
260
+
261
+ 0.upto(4) do |i|
262
+ expect(db.get_object(i + 1)).to eql("#{i + 1}:#{'X' * (i + 1) * 30}$")
263
+ end
264
+ expect(db.get_object(6)).to be_nil
265
+ db.close
266
+ end
267
+
268
+ it 'should handle duplicate entries for the same ID in database.blobs file' do
269
+ @store.exit
270
+
271
+ db = PEROBS::FlatFileDB.new(@db_dir)
272
+ db_file = File.join(@db_dir, 'database.blobs')
273
+ db.open
274
+ 0.upto(5) do |i|
275
+ db.put_object("#{i + 1}:#{'X' * (i + 1) * 30}$", i + 1)
276
+ end
277
+ db.close
278
+
279
+ # This appends the entry 2 again
280
+ blob2 = File.read(db_file, 319 - 199, 199)
281
+ File.write(db_file, blob2, File.size(db_file))
282
+
283
+ db.open
284
+ expect(db.check_db).to eql(2)
285
+ expect(db.check_db(true)).to eql(1)
286
+ db.close
287
+ db = PEROBS::FlatFileDB.new(@db_dir, { :log => $stderr,
288
+ :log_level => Logger::WARN })
289
+ db.open
290
+ expect(db.check_db).to eql(0)
291
+
292
+ 0.upto(5) do |i|
293
+ expect(db.get_object(i + 1)).to eql("#{i + 1}:#{'X' * (i + 1) * 30}$")
294
+ end
295
+ db.close
296
+ end
297
+
114
298
  end
115
299