perobs 4.0.0 → 4.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. checksums.yaml +5 -5
  2. data/README.md +27 -16
  3. data/lib/perobs/Array.rb +66 -19
  4. data/lib/perobs/BTree.rb +106 -15
  5. data/lib/perobs/BTreeBlob.rb +4 -3
  6. data/lib/perobs/BTreeDB.rb +5 -4
  7. data/lib/perobs/BTreeNode.rb +482 -156
  8. data/lib/perobs/BTreeNodeLink.rb +10 -0
  9. data/lib/perobs/BigArray.rb +285 -0
  10. data/lib/perobs/BigArrayNode.rb +1002 -0
  11. data/lib/perobs/BigHash.rb +246 -0
  12. data/lib/perobs/BigTree.rb +197 -0
  13. data/lib/perobs/BigTreeNode.rb +873 -0
  14. data/lib/perobs/Cache.rb +48 -10
  15. data/lib/perobs/ConsoleProgressMeter.rb +61 -0
  16. data/lib/perobs/DataBase.rb +4 -3
  17. data/lib/perobs/DynamoDB.rb +57 -15
  18. data/lib/perobs/EquiBlobsFile.rb +155 -50
  19. data/lib/perobs/FNV_Hash_1a_64.rb +54 -0
  20. data/lib/perobs/FlatFile.rb +519 -227
  21. data/lib/perobs/FlatFileBlobHeader.rb +113 -54
  22. data/lib/perobs/FlatFileDB.rb +49 -23
  23. data/lib/perobs/FuzzyStringMatcher.rb +175 -0
  24. data/lib/perobs/Hash.rb +127 -33
  25. data/lib/perobs/IDList.rb +144 -0
  26. data/lib/perobs/IDListPage.rb +107 -0
  27. data/lib/perobs/IDListPageFile.rb +180 -0
  28. data/lib/perobs/IDListPageRecord.rb +142 -0
  29. data/lib/perobs/Object.rb +18 -15
  30. data/lib/perobs/ObjectBase.rb +46 -5
  31. data/lib/perobs/PersistentObjectCache.rb +57 -68
  32. data/lib/perobs/PersistentObjectCacheLine.rb +24 -12
  33. data/lib/perobs/ProgressMeter.rb +97 -0
  34. data/lib/perobs/SpaceManager.rb +273 -0
  35. data/lib/perobs/SpaceTree.rb +21 -12
  36. data/lib/perobs/SpaceTreeNode.rb +53 -61
  37. data/lib/perobs/Store.rb +264 -145
  38. data/lib/perobs/version.rb +1 -1
  39. data/lib/perobs.rb +2 -0
  40. data/perobs.gemspec +4 -4
  41. data/test/Array_spec.rb +15 -6
  42. data/test/BTree_spec.rb +6 -2
  43. data/test/BigArray_spec.rb +261 -0
  44. data/test/BigHash_spec.rb +152 -0
  45. data/test/BigTreeNode_spec.rb +153 -0
  46. data/test/BigTree_spec.rb +259 -0
  47. data/test/EquiBlobsFile_spec.rb +105 -1
  48. data/test/FNV_Hash_1a_64_spec.rb +59 -0
  49. data/test/FlatFileDB_spec.rb +198 -14
  50. data/test/FuzzyStringMatcher_spec.rb +261 -0
  51. data/test/Hash_spec.rb +13 -3
  52. data/test/IDList_spec.rb +77 -0
  53. data/test/LegacyDBs/LegacyDB.rb +155 -0
  54. data/test/LegacyDBs/version_3/class_map.json +1 -0
  55. data/test/LegacyDBs/version_3/config.json +1 -0
  56. data/test/LegacyDBs/version_3/database.blobs +0 -0
  57. data/test/LegacyDBs/version_3/database_spaces.blobs +0 -0
  58. data/test/LegacyDBs/version_3/index.blobs +0 -0
  59. data/test/LegacyDBs/version_3/version +1 -0
  60. data/test/LockFile_spec.rb +9 -6
  61. data/test/SpaceManager_spec.rb +176 -0
  62. data/test/SpaceTree_spec.rb +4 -1
  63. data/test/Store_spec.rb +305 -203
  64. data/test/spec_helper.rb +9 -4
  65. metadata +57 -16
  66. data/lib/perobs/BTreeNodeCache.rb +0 -109
  67. data/lib/perobs/TreeDB.rb +0 -277
@@ -0,0 +1,259 @@
1
+ # encoding: UTF-8
2
+ #
3
+ # Copyright (c) 2016, 2017 by Chris Schlaeger <chris@taskjuggler.org>
4
+ #
5
+ # MIT License
6
+ #
7
+ # Permission is hereby granted, free of charge, to any person obtaining
8
+ # a copy of this software and associated documentation files (the
9
+ # "Software"), to deal in the Software without restriction, including
10
+ # without limitation the rights to use, copy, modify, merge, publish,
11
+ # distribute, sublicense, and/or sell copies of the Software, and to
12
+ # permit persons to whom the Software is furnished to do so, subject to
13
+ # the following conditions:
14
+ #
15
+ # The above copyright notice and this permission notice shall be
16
+ # included in all copies or substantial portions of the Software.
17
+ #
18
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
19
+ # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
20
+ # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
21
+ # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
22
+ # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
23
+ # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
24
+ # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
25
+
26
+ require 'fileutils'
27
+
28
+ require 'spec_helper'
29
+ require 'perobs/Store'
30
+ require 'perobs/BigTree'
31
+
32
+ describe PEROBS::BigTree do
33
+
34
+ ORDER = 7
35
+
36
+ before(:all) do
37
+ @db_name = generate_db_name(__FILE__)
38
+ @store = PEROBS::Store.new(@db_name)
39
+ @t = @store.new(PEROBS::BigTree, ORDER)
40
+ end
41
+
42
+ after(:all) do
43
+ @store.delete_store
44
+ end
45
+
46
+ it 'should be empty' do
47
+ expect(@t.empty?).to be true
48
+ expect(@t.length).to eql(0)
49
+ s = @t.statistics
50
+ expect(s.leaf_nodes).to eql(1)
51
+ expect(s.branch_nodes).to eql(0)
52
+ expect(s.min_depth).to eql(1)
53
+ expect(s.max_depth).to eql(1)
54
+ end
55
+
56
+ it 'should deal with requests for unknown keys' do
57
+ expect(@t.has_key?(42)).to be false
58
+ expect(@t.get(42)).to be_nil
59
+ end
60
+
61
+ it 'should support adding sequential key/value pairs' do
62
+ 0.upto(ORDER ** 3) do |i|
63
+ @t.insert(i, 3 * i)
64
+ expect(@t.check).to be true
65
+ expect(@t.length).to eql(i + 1)
66
+ expect(@t.has_key?(i)).to be true
67
+ expect(@t.get(i)).to eql(3 * i)
68
+ end
69
+ end
70
+
71
+ it 'should iterate over the stored key and value pairs' do
72
+ i = 0
73
+ @t.each do |k, v|
74
+ expect(k).to eql(i)
75
+ expect(v).to eql(3 * i)
76
+ i += 1
77
+ end
78
+ expect(i).to eql(ORDER ** 3 + 1)
79
+ end
80
+
81
+ it 'should iterate in reverse order over the stored key and value pairs' do
82
+ i = ORDER ** 3
83
+ @t.reverse_each do |k, v|
84
+ expect(k).to eql(i)
85
+ expect(v).to eql(3 * i)
86
+ i -= 1
87
+ end
88
+ expect(i).to eql(-1)
89
+ end
90
+
91
+ it 'should yield the key/value pairs on check' do
92
+ i = 0
93
+ @t.check do |k, v|
94
+ expect(k).to eql(k)
95
+ expect(v).to eql(3 * k)
96
+ i += 1
97
+ end
98
+ expect(i).to eql(ORDER ** 3 + 1)
99
+ end
100
+
101
+ it 'should support overwriting existing entries' do
102
+ 0.upto(ORDER ** 3) do |i|
103
+ @t.insert(i, 7 * i)
104
+ expect(@t.check).to be true
105
+ expect(@t.length).to eql(ORDER ** 3 + 1)
106
+ expect(@t.has_key?(i)).to be true
107
+ expect(@t.get(i)).to eql(7 * i)
108
+ end
109
+ end
110
+
111
+ it 'should support clearing the tree' do
112
+ @t.clear
113
+ expect(@t.check).to be true
114
+ expect(@t.empty?).to be true
115
+ expect(@t.length).to eql(0)
116
+ i = 0
117
+ @t.each { |k, v| i += 1 }
118
+ expect(i).to eql(0)
119
+ end
120
+
121
+ it 'should support adding random key/value pairs' do
122
+ (1..ORDER ** 3).to_a.shuffle.each do |i|
123
+ @t.insert(i, i * 100)
124
+ end
125
+ expect(@t.check).to be true
126
+ (1..ORDER ** 3).to_a.shuffle.each do |i|
127
+ expect(@t.get(i)).to eql(i * 100)
128
+ end
129
+ end
130
+
131
+ it 'should support removing keys in random order' do
132
+ @t.clear
133
+ (1..ORDER ** 3).to_a.shuffle.each do |i|
134
+ @t.insert(i, i * 100)
135
+ end
136
+ expect(@t.length).to eql(ORDER ** 3)
137
+ (1..ORDER ** 3).to_a.shuffle.each do |i|
138
+ expect(@t.remove(i)).to eql(i * 100)
139
+ expect(@t.check).to be true
140
+ end
141
+ expect(@t.length).to eql(0)
142
+ end
143
+
144
+ it 'should support removing keys in increasing order' do
145
+ @t.clear
146
+ (1..ORDER ** 3).to_a.shuffle.each do |i|
147
+ @t.insert(i, i * 100)
148
+ end
149
+ expect(@t.length).to eql(ORDER ** 3)
150
+ (1..ORDER ** 3).to_a.each do |i|
151
+ expect(@t.remove(i)).to eql(i * 100)
152
+ expect(@t.check).to be true
153
+ end
154
+ expect(@t.length).to eql(0)
155
+ end
156
+
157
+ it 'should support removing keys in reverse order' do
158
+ @t.clear
159
+ (1..ORDER ** 3).to_a.shuffle.each do |i|
160
+ @t.insert(i, i * 100)
161
+ end
162
+ expect(@t.length).to eql(ORDER ** 3)
163
+ (1..ORDER ** 3).to_a.reverse_each do |i|
164
+ expect(@t.remove(i)).to eql(i * 100)
165
+ expect(@t.check).to be true
166
+ end
167
+ expect(@t.length).to eql(0)
168
+ end
169
+
170
+ it 'should persist the data' do
171
+ db_name = generate_db_name(__FILE__ + '_persist')
172
+ store = PEROBS::Store.new(db_name)
173
+ store['bigtree'] = t = store.new(PEROBS::BigTree, 4)
174
+ 10.times do |i|
175
+ t.insert(i, i)
176
+ end
177
+ 10.times do |i|
178
+ expect(t.get(i)).to eql(i)
179
+ end
180
+ store.exit
181
+
182
+ store = PEROBS::Store.new(db_name)
183
+ t = store['bigtree']
184
+ 10.times do |i|
185
+ expect(t.get(i)).to eql(i)
186
+ end
187
+ store.delete_store
188
+ end
189
+
190
+ it 'should delete all entries matching a condition' do
191
+ @t.clear
192
+ (1..50).to_a.shuffle.each do |i|
193
+ @t.insert(i, i)
194
+ end
195
+ @t.delete_if { |k, v| v % 7 == 0 }
196
+ expect(@t.check).to be true
197
+ @t.each do |k, v|
198
+ expect(v % 7).to be > 0, "failed for #{v}"
199
+ end
200
+ expect(@t.length).to eql(43)
201
+ @t.delete_if { |k, v| v % 2 == 0 }
202
+ expect(@t.check).to be true
203
+ @t.each do |k, v|
204
+ expect(v % 2).to be > 0
205
+ end
206
+ expect(@t.length).to eql(21)
207
+ @t.delete_if { |k, v| true }
208
+ expect(@t.check).to be true
209
+ expect(@t.empty?).to be true
210
+ end
211
+
212
+ it 'should survive a real-world usage test' do
213
+ @t.clear
214
+ ref = {}
215
+ 0.upto(1000) do
216
+ case rand(5)
217
+ when 0
218
+ 0.upto(2) do
219
+ key = rand(100000)
220
+ value = key * 10
221
+ @t.insert(key, value)
222
+ ref[key] = value
223
+ end
224
+ when 1
225
+ if ref.length > 0
226
+ key = ref.keys[rand(ref.keys.length)]
227
+ expect(@t.remove(key)).to eql(ref[key])
228
+ ref.delete(key)
229
+ end
230
+ when 2
231
+ if ref.length > 0
232
+ 0.upto(3) do
233
+ key = ref.keys[rand(ref.keys.length)]
234
+ expect(@t.get(key)).to eql(ref[key])
235
+ end
236
+ end
237
+ when 3
238
+ if ref.length > 0
239
+ key = ref.keys[rand(ref.keys.length)]
240
+ value = ref[key] + 1
241
+ @t.insert(key, value)
242
+ ref[key] = value
243
+ end
244
+ when 4
245
+ if rand(50) == 0
246
+ expect(@t.check).to be true
247
+ end
248
+ end
249
+ end
250
+
251
+ i = 0
252
+ @t.each do |k, v|
253
+ expect(ref[k]).to eql(v)
254
+ i += 1
255
+ end
256
+ expect(i).to eql(ref.length)
257
+ end
258
+
259
+ end
@@ -27,13 +27,17 @@ require 'fileutils'
27
27
 
28
28
  require 'spec_helper'
29
29
  require 'perobs/EquiBlobsFile'
30
+ require 'perobs/ProgressMeter'
30
31
 
31
32
  describe PEROBS::EquiBlobsFile do
32
33
 
33
34
  before(:all) do
35
+ PEROBS.log.level = Logger::ERROR
36
+ PEROBS.log.open($stderr)
34
37
  @db_dir = generate_db_name('EquiBlobsFile')
35
38
  FileUtils.mkdir_p(@db_dir)
36
- @bf = PEROBS::EquiBlobsFile.new(@db_dir, 'EquiBlobsFile', 8)
39
+ @bf = PEROBS::EquiBlobsFile.new(@db_dir, 'EquiBlobsFile',
40
+ PEROBS::ProgressMeter.new, 8)
37
41
  end
38
42
 
39
43
  after(:all) do
@@ -134,6 +138,12 @@ describe PEROBS::EquiBlobsFile do
134
138
  expect(@bf.check).to be true
135
139
  end
136
140
 
141
+ it 'should support deleting reserved blobs' do
142
+ adr = @bf.free_address
143
+ @bf.delete_blob(adr)
144
+ expect(@bf.check).to be true
145
+ end
146
+
137
147
  it 'should support clearing the file' do
138
148
  @bf.clear
139
149
  expect(@bf.total_entries).to eql(0)
@@ -191,5 +201,99 @@ describe PEROBS::EquiBlobsFile do
191
201
  expect(@bf.check).to be true
192
202
  end
193
203
 
204
+ it 'should support custom offsets' do
205
+ @bf.close
206
+ @bf.erase
207
+ @bf.clear_custom_data
208
+ @bf.register_custom_data('foo', 42)
209
+ @bf.register_custom_data('bar', 43)
210
+ @bf.open
211
+ expect(@bf.total_entries).to eql(0)
212
+ expect(@bf.total_spaces).to eql(0)
213
+ expect(@bf.check).to be true
214
+ expect(@bf.free_address).to eql(1)
215
+ @bf.store_blob(1,'11111111')
216
+ expect(@bf.free_address).to eql(2)
217
+ @bf.store_blob(2,'22222222')
218
+ expect(@bf.free_address).to eql(3)
219
+ @bf.store_blob(3,'33333333')
220
+ expect(@bf.check).to be true
221
+ expect(@bf.total_entries).to eql(3)
222
+ expect(@bf.total_spaces).to eql(0)
223
+ @bf.delete_blob(2)
224
+ expect(@bf.check).to be true
225
+ expect(@bf.total_entries).to eql(2)
226
+ expect(@bf.total_spaces).to eql(1)
227
+ expect(@bf.free_address).to eql(2)
228
+ @bf.store_blob(2,'44444444')
229
+ expect(@bf.total_entries).to eql(3)
230
+ expect(@bf.total_spaces).to eql(0)
231
+ expect(@bf.check).to be true
232
+ end
233
+
234
+ it 'should support a mix of adds and deletes' do
235
+ @bf.close
236
+ @bf.erase
237
+ @bf.clear_custom_data
238
+ @bf.register_custom_data('foo', 42)
239
+ @bf.register_custom_data('bar', 43)
240
+ @bf.open
241
+
242
+ entries = {}
243
+ 1000.times do |i|
244
+ rand(30).times do
245
+ adr = @bf.free_address
246
+ expect(entries[adr]).to be nil
247
+ val = rand(2 ** 64)
248
+ @bf.store_blob(adr, [ val ].pack('Q'))
249
+ entries[adr] = val
250
+ #expect(@bf.check).to be true
251
+ end
252
+
253
+ rand(15).times do
254
+ unless entries.empty?
255
+ addresses = entries.keys
256
+ adr = addresses[rand(addresses.length)]
257
+ expect(@bf.retrieve_blob(adr).unpack('Q').first).to eql(entries[adr])
258
+ @bf.delete_blob(adr)
259
+ entries.delete(adr)
260
+ #expect(@bf.check).to be true
261
+ end
262
+ end
263
+
264
+ rand(5).times do
265
+ unless entries.empty?
266
+ addresses = entries.keys
267
+ adr = addresses[rand(addresses.length)]
268
+ val = rand(2 ** 64)
269
+ @bf.store_blob(adr, [ val ].pack('Q'))
270
+ entries[adr] = val
271
+ #expect(@bf.check).to be true
272
+ end
273
+ end
274
+
275
+ if rand(100) == 0
276
+ expect(@bf.check).to be true
277
+ end
278
+
279
+ if rand(100) == 0
280
+ @bf.first_entry = i
281
+ @bf.set_custom_data('foo', i)
282
+ end
283
+
284
+ if rand(50) == 0
285
+ @bf.close
286
+ @bf.open
287
+ end
288
+
289
+ if rand(500) == 0
290
+ @bf.clear
291
+ entries = {}
292
+ end
293
+ end
294
+
295
+ expect(@bf.check).to be true
296
+ end
297
+
194
298
  end
195
299
 
@@ -0,0 +1,59 @@
1
+ # encoding: UTF-8
2
+ #
3
+ # Copyright (c) 2019 by Chris Schlaeger <chris@taskjuggler.org>
4
+ #
5
+ # This file contains tests for Array that are similar to the tests for the
6
+ # Array implementation in MRI. The ideas of these tests were replicated in
7
+ # this code.
8
+ #
9
+ # MIT License
10
+ #
11
+ # Permission is hereby granted, free of charge, to any person obtaining
12
+ # a copy of this software and associated documentation files (the
13
+ # "Software"), to deal in the Software without restriction, including
14
+ # without limitation the rights to use, copy, modify, merge, publish,
15
+ # distribute, sublicense, and/or sell copies of the Software, and to
16
+ # permit persons to whom the Software is furnished to do so, subject to
17
+ # the following conditions:
18
+ #
19
+ # The above copyright notice and this permission notice shall be
20
+ # included in all copies or substantial portions of the Software.
21
+ #
22
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
23
+ # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
24
+ # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
25
+ # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
26
+ # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
27
+ # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
28
+ # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
29
+
30
+ require 'spec_helper'
31
+
32
+ require 'perobs'
33
+
34
+ describe PEROBS::FNV_Hash_1a_64 do
35
+
36
+ it 'should generate stable hashes for Strings' do
37
+ refs = [
38
+ [ 'foo', 15902901984413996407 ],
39
+ [ 'foo', 15902901984413996407 ],
40
+ [ 'bar', 16101355973854746 ],
41
+ [ 'foobar', 9625390261332436968 ],
42
+ [ 'PEROBS rocks your application!', 4089220442501866848 ],
43
+ [ 'Permission is hereby granted, free of charge, to any person ' +
44
+ 'obtaining a copy of this software and associated documentation ' +
45
+ 'files (the "Software"), to deal in the Software without ' +
46
+ 'restriction, including without limitation the rights to use, ' +
47
+ 'copy, modify, merge, publish, distribute, sublicense, and/or ' +
48
+ 'sell copies of the Software, and to permit persons to whom the ' +
49
+ 'Software is furnished to do so, subject to the following conditions:',
50
+ 17637146001033534275 ]
51
+ ]
52
+
53
+ refs.each do |v|
54
+ expect(PEROBS::FNV_Hash_1a_64::digest(v[0])).to eql(v[1])
55
+ end
56
+ end
57
+
58
+ end
59
+
@@ -28,6 +28,7 @@ require 'fileutils'
28
28
  require 'spec_helper'
29
29
  require 'perobs/FlatFileDB'
30
30
  require 'perobs/Store'
31
+ require 'LegacyDBs/LegacyDB'
31
32
 
32
33
  class FlatFileDB_O < PEROBS::Object
33
34
 
@@ -47,7 +48,12 @@ describe PEROBS::FlatFileDB do
47
48
  before(:each) do
48
49
  @db_dir = generate_db_name(__FILE__)
49
50
  FileUtils.mkdir_p(@db_dir)
50
- @store = PEROBS::Store.new(@db_dir, :engine => PEROBS::FlatFileDB)
51
+ @store_options = {
52
+ :engine => PEROBS::FlatFileDB,
53
+ :log => $stderr,
54
+ :log_level => Logger::ERROR
55
+ }
56
+ @store = PEROBS::Store.new(@db_dir, @store_options)
51
57
  end
52
58
 
53
59
  after(:each) do
@@ -65,19 +71,30 @@ describe PEROBS::FlatFileDB do
65
71
  expect { db2.open }.to raise_error(PEROBS::FatalError)
66
72
  end
67
73
 
68
- it 'should do a version upgrade' do
74
+ it 'should do a version upgrade from version 3' do
69
75
  # Close the store
70
- @store['o'] = @store.new(FlatFileDB_O)
71
76
  @store.exit
77
+ src_dir = File.join(File.dirname(__FILE__), 'LegacyDBs', 'version_3')
78
+ FileUtils.cp_r(Dir.glob(src_dir + '/*'), @db_dir)
72
79
 
73
- # Manually downgrade the version file to version 1
74
- version_file = File.join(@db_dir, 'version')
75
- File.write(version_file, '1')
80
+ db = LegacyDB.new(@db_dir)
81
+ capture_io { db.open }
82
+ capture_io { expect(db.check).to be true }
83
+ end
76
84
 
77
- # Open the store again
78
- store = PEROBS::Store.new(@db_dir, :engine => PEROBS::FlatFileDB)
79
- expect(File.read(version_file).to_i).to eql(PEROBS::FlatFileDB::VERSION)
80
- expect(store['o'].b).to eql(42)
85
+ it 'should do a version upgrade from version 4.1' do
86
+ # Close the store
87
+ @store.exit
88
+ src_dir = File.join(File.dirname(__FILE__), 'LegacyDBs', 'version_4.1')
89
+ FileUtils.cp_r(Dir.glob(src_dir + '/*'), @db_dir)
90
+
91
+ db = LegacyDB.new(@db_dir)
92
+ capture_io { db.open }
93
+ capture_io { expect(db.repair).to eql(0) }
94
+ expect(File.exist?(File.join(@db_dir, 'space_index.blobs'))).to be true
95
+ expect(File.exist?(File.join(@db_dir, 'space_list.blobs'))).to be true
96
+ expect(File.exist?(File.join(@db_dir, 'database_spaces.blobs'))).to be false
97
+ capture_io { expect(db.check).to be true }
81
98
  end
82
99
 
83
100
  it 'should refuse a version downgrade' do
@@ -89,7 +106,7 @@ describe PEROBS::FlatFileDB do
89
106
  File.write(version_file, '1000000')
90
107
 
91
108
  # Open the store again
92
- expect { PEROBS::Store.new(@db_dir, :engine => PEROBS::FlatFileDB) }.to raise_error(PEROBS::FatalError)
109
+ expect { PEROBS::Store.new(@db_dir) }.to raise_error(PEROBS::FatalError)
93
110
  end
94
111
 
95
112
  it 'should recover from a lost index file' do
@@ -97,7 +114,10 @@ describe PEROBS::FlatFileDB do
97
114
  @store.exit
98
115
 
99
116
  File.delete(File.join(@db_dir, 'index.blobs'))
100
- store = PEROBS::Store.new(@db_dir, :engine => PEROBS::FlatFileDB)
117
+ store = nil
118
+ capture_io do
119
+ store = PEROBS::Store.new(@db_dir)
120
+ end
101
121
  expect(store['o'].b).to eql(42)
102
122
  end
103
123
 
@@ -106,10 +126,174 @@ describe PEROBS::FlatFileDB do
106
126
  @store.exit
107
127
 
108
128
  File.write(File.join(@db_dir, 'index.blobs'), '*' * 500)
109
- store = PEROBS::Store.new(@db_dir, :engine => PEROBS::FlatFileDB)
110
- store.check(true)
129
+ store = nil
130
+ capture_io do
131
+ store = PEROBS::Store.new(@db_dir)
132
+ end
133
+ capture_io { store.check(true) }
111
134
  expect(store['o'].b).to eql(42)
112
135
  end
113
136
 
137
+ it 'should discard a corrupted blob inside the database.blobs file' do
138
+ @store.exit
139
+
140
+ db = PEROBS::FlatFileDB.new(@db_dir)
141
+ db_file = File.join(@db_dir, 'database.blobs')
142
+ db.open
143
+ 0.upto(5) do |i|
144
+ db.put_object("#{i + 1}:#{'X' * (i + 1) * 30}", i + 1)
145
+ end
146
+ db.close
147
+ db.open
148
+ 0.upto(5) do |i|
149
+ db.put_object("#{i + 10}:#{'Y' * (i + 1) * 25}", i + 10)
150
+ end
151
+ pos = db.instance_variable_get('@flat_file').find_obj_addr_by_id(10)
152
+ db.close
153
+
154
+ f = File.open(db_file, 'rb+')
155
+ f.seek(pos)
156
+ f.write('ZZZZZ')
157
+ f.close
158
+
159
+ db.open
160
+ expect(db.check_db).to eql(1)
161
+ expect(db.check_db(true)).to eql(1)
162
+ db.close
163
+ db = PEROBS::FlatFileDB.new(@db_dir, { :log => $stderr,
164
+ :log_level => Logger::ERROR })
165
+ db.open
166
+ expect(db.check_db).to eql(0)
167
+
168
+ 0.upto(5) do |i|
169
+ expect(db.get_object(i + 1)).to eql("#{i + 1}:#{'X' * (i + 1) * 30}")
170
+ end
171
+ expect(db.get_object(10)).to be_nil
172
+ 1.upto(5) do |i|
173
+ expect(db.get_object(i + 10)).to eql("#{i + 10}:#{'Y' * (i + 1) * 25}")
174
+ end
175
+ db.close
176
+ end
177
+
178
+ it 'should discard a corrupted blob at the end of the database.blobs file' do
179
+ @store.exit
180
+
181
+ db = PEROBS::FlatFileDB.new(@db_dir)
182
+ db_file = File.join(@db_dir, 'database.blobs')
183
+ db.open
184
+ 0.upto(5) do |i|
185
+ db.put_object("#{i + 1}:#{'X' * (i + 1) * 30}$", i + 1)
186
+ end
187
+ db.close
188
+
189
+ f = File.truncate(db_file, File.size(db_file) - 20)
190
+
191
+ db.open
192
+ expect(db.check_db).to eql(2)
193
+ expect(db.check_db(true)).to eql(2)
194
+ db.close
195
+ db = PEROBS::FlatFileDB.new(@db_dir, { :log => $stderr,
196
+ :log_level => Logger::ERROR })
197
+ db.open
198
+ expect(db.check_db).to eql(0)
199
+
200
+ 0.upto(4) do |i|
201
+ expect(db.get_object(i + 1)).to eql("#{i + 1}:#{'X' * (i + 1) * 30}$")
202
+ end
203
+ expect(db.get_object(6)).to be_nil
204
+ db.close
205
+ end
206
+
207
+ it 'should discard a corrupted header at the end of the database.blobs file' do
208
+ @store.exit
209
+
210
+ db = PEROBS::FlatFileDB.new(@db_dir)
211
+ db_file = File.join(@db_dir, 'database.blobs')
212
+ db.open
213
+ 0.upto(5) do |i|
214
+ db.put_object("#{i + 1}:#{'X' * (i + 1) * 30}$", i + 1)
215
+ end
216
+ db.close
217
+
218
+ f = File.truncate(db_file, File.size(db_file) - 200)
219
+
220
+ db.open
221
+ expect(db.check_db).to eql(1)
222
+ expect(db.check_db(true)).to eql(1)
223
+ db.close
224
+ db = PEROBS::FlatFileDB.new(@db_dir, { :log => $stderr,
225
+ :log_level => Logger::ERROR })
226
+ db.open
227
+ expect(db.check_db).to eql(0)
228
+
229
+ 0.upto(4) do |i|
230
+ expect(db.get_object(i + 1)).to eql("#{i + 1}:#{'X' * (i + 1) * 30}$")
231
+ end
232
+ expect(db.get_object(6)).to be_nil
233
+ db.close
234
+ end
235
+
236
+ it 'should handle a lost blob at the end of the database.blobs file' do
237
+ @store.exit
238
+
239
+ db = PEROBS::FlatFileDB.new(@db_dir)
240
+ db_file = File.join(@db_dir, 'database.blobs')
241
+ db.open
242
+ 0.upto(5) do |i|
243
+ db.put_object("#{i + 1}:#{'X' * (i + 1) * 30}$", i + 1)
244
+ end
245
+ db.close
246
+
247
+ # This exactly removes the last blob (#6)
248
+ f = File.truncate(db_file, File.size(db_file) - 210)
249
+
250
+ db.open
251
+ expect(db.check_db).to eql(1)
252
+ # The repair won't find the missing blob since the blob file is without
253
+ # errors.
254
+ expect(db.check_db(true)).to eql(0)
255
+ db.close
256
+ db = PEROBS::FlatFileDB.new(@db_dir, { :log => $stderr,
257
+ :log_level => Logger::ERROR })
258
+ db.open
259
+ expect(db.check_db).to eql(0)
260
+
261
+ 0.upto(4) do |i|
262
+ expect(db.get_object(i + 1)).to eql("#{i + 1}:#{'X' * (i + 1) * 30}$")
263
+ end
264
+ expect(db.get_object(6)).to be_nil
265
+ db.close
266
+ end
267
+
268
+ it 'should handle duplicate entries for the same ID in database.blobs file' do
269
+ @store.exit
270
+
271
+ db = PEROBS::FlatFileDB.new(@db_dir)
272
+ db_file = File.join(@db_dir, 'database.blobs')
273
+ db.open
274
+ 0.upto(5) do |i|
275
+ db.put_object("#{i + 1}:#{'X' * (i + 1) * 30}$", i + 1)
276
+ end
277
+ db.close
278
+
279
+ # This appends the entry 2 again
280
+ blob2 = File.read(db_file, 319 - 199, 199)
281
+ File.write(db_file, blob2, File.size(db_file))
282
+
283
+ db.open
284
+ expect(db.check_db).to eql(2)
285
+ expect(db.check_db(true)).to eql(1)
286
+ db.close
287
+ db = PEROBS::FlatFileDB.new(@db_dir, { :log => $stderr,
288
+ :log_level => Logger::WARN })
289
+ db.open
290
+ expect(db.check_db).to eql(0)
291
+
292
+ 0.upto(5) do |i|
293
+ expect(db.get_object(i + 1)).to eql("#{i + 1}:#{'X' * (i + 1) * 30}$")
294
+ end
295
+ db.close
296
+ end
297
+
114
298
  end
115
299