perobs 4.0.0 → 4.1.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (59) hide show
  1. checksums.yaml +4 -4
  2. data/lib/perobs.rb +1 -0
  3. data/lib/perobs/Array.rb +66 -19
  4. data/lib/perobs/BTree.rb +83 -12
  5. data/lib/perobs/BTreeBlob.rb +1 -1
  6. data/lib/perobs/BTreeDB.rb +2 -2
  7. data/lib/perobs/BTreeNode.rb +365 -85
  8. data/lib/perobs/BigArray.rb +267 -0
  9. data/lib/perobs/BigArrayNode.rb +998 -0
  10. data/lib/perobs/BigHash.rb +262 -0
  11. data/lib/perobs/BigTree.rb +184 -0
  12. data/lib/perobs/BigTreeNode.rb +873 -0
  13. data/lib/perobs/ConsoleProgressMeter.rb +61 -0
  14. data/lib/perobs/DataBase.rb +4 -3
  15. data/lib/perobs/DynamoDB.rb +57 -15
  16. data/lib/perobs/EquiBlobsFile.rb +143 -51
  17. data/lib/perobs/FNV_Hash_1a_64.rb +54 -0
  18. data/lib/perobs/FlatFile.rb +363 -203
  19. data/lib/perobs/FlatFileBlobHeader.rb +98 -54
  20. data/lib/perobs/FlatFileDB.rb +42 -20
  21. data/lib/perobs/Hash.rb +58 -13
  22. data/lib/perobs/IDList.rb +144 -0
  23. data/lib/perobs/IDListPage.rb +107 -0
  24. data/lib/perobs/IDListPageFile.rb +180 -0
  25. data/lib/perobs/IDListPageRecord.rb +142 -0
  26. data/lib/perobs/Object.rb +18 -15
  27. data/lib/perobs/ObjectBase.rb +38 -4
  28. data/lib/perobs/PersistentObjectCache.rb +53 -67
  29. data/lib/perobs/PersistentObjectCacheLine.rb +24 -12
  30. data/lib/perobs/ProgressMeter.rb +97 -0
  31. data/lib/perobs/SpaceTree.rb +21 -12
  32. data/lib/perobs/SpaceTreeNode.rb +53 -61
  33. data/lib/perobs/Store.rb +71 -32
  34. data/lib/perobs/version.rb +1 -1
  35. data/perobs.gemspec +4 -4
  36. data/test/Array_spec.rb +15 -6
  37. data/test/BTree_spec.rb +5 -2
  38. data/test/BigArray_spec.rb +214 -0
  39. data/test/BigHash_spec.rb +144 -0
  40. data/test/BigTreeNode_spec.rb +153 -0
  41. data/test/BigTree_spec.rb +259 -0
  42. data/test/EquiBlobsFile_spec.rb +105 -1
  43. data/test/FNV_Hash_1a_64_spec.rb +59 -0
  44. data/test/FlatFileDB_spec.rb +63 -14
  45. data/test/Hash_spec.rb +1 -2
  46. data/test/IDList_spec.rb +77 -0
  47. data/test/LegacyDBs/LegacyDB.rb +151 -0
  48. data/test/LegacyDBs/version_3/class_map.json +1 -0
  49. data/test/LegacyDBs/version_3/config.json +1 -0
  50. data/test/LegacyDBs/version_3/database.blobs +0 -0
  51. data/test/LegacyDBs/version_3/database_spaces.blobs +0 -0
  52. data/test/LegacyDBs/version_3/index.blobs +0 -0
  53. data/test/LegacyDBs/version_3/version +1 -0
  54. data/test/LockFile_spec.rb +9 -6
  55. data/test/SpaceTree_spec.rb +4 -1
  56. data/test/Store_spec.rb +290 -199
  57. data/test/spec_helper.rb +9 -4
  58. metadata +47 -10
  59. data/lib/perobs/TreeDB.rb +0 -277
@@ -1,4 +1,4 @@
1
1
  module PEROBS
2
2
  # The version number
3
- VERSION = "4.0.0"
3
+ VERSION = "4.1.0"
4
4
  end
@@ -16,9 +16,9 @@ GEM_SPEC = Gem::Specification.new do |spec|
16
16
  spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
17
17
  spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
18
18
  spec.require_paths = ["lib"]
19
- spec.required_ruby_version = '>=2.0'
19
+ spec.required_ruby_version = '>=2.3'
20
20
 
21
- spec.add_development_dependency 'bundler', '~> 1.6'
22
- spec.add_development_dependency 'yard', '~>0.8.7'
23
- spec.add_development_dependency 'rake', '~> 10.0'
21
+ spec.add_development_dependency 'bundler', '~> 2.3'
22
+ spec.add_development_dependency 'yard', '~>0.9.12'
23
+ spec.add_development_dependency 'rake', '~> 10.1'
24
24
  end
@@ -169,8 +169,13 @@ describe PEROBS::Array do
169
169
 
170
170
  it 'should support collect!()' do
171
171
  a = cpa([ 1, 'cat', 1..1 ])
172
- expect(a.collect! { |e| e.class.to_s }).to eq([ 'Integer', 'String', 'Range' ])
173
- pcheck { expect(@store['a'].to_a).to eq([ 'Integer', 'String', 'Range' ]) }
172
+ if RUBY_VERSION < '2.2'
173
+ expect(a.collect! { |e| e.class.to_s }.to_a).to eq([ 'Fixnum', 'String', 'Range' ])
174
+ pcheck { expect(@store['a'].to_a).to eq([ 'Fixnum', 'String', 'Range' ]) }
175
+ else
176
+ expect(a.collect! { |e| e.class.to_s }.to_a).to eq([ 'Integer', 'String', 'Range' ])
177
+ pcheck { expect(@store['a'].to_a).to eq([ 'Integer', 'String', 'Range' ]) }
178
+ end
174
179
 
175
180
  a = cpa([ 1, 'cat', 1..1 ])
176
181
  expect(a.collect! { 99 }).to eq([ 99, 99, 99])
@@ -179,8 +184,13 @@ describe PEROBS::Array do
179
184
 
180
185
  it 'should support map!()' do
181
186
  a = cpa([ 1, 'cat', 1..1 ])
182
- expect(a.map! { |e| e.class.to_s }).to eq([ 'Integer', 'String', 'Range' ])
183
- pcheck { expect(@store['a']).to eq([ 'Integer', 'String', 'Range' ]) }
187
+ if RUBY_VERSION < '2.2'
188
+ expect(a.map! { |e| e.class.to_s }.to_a).to eq([ 'Fixnum', 'String', 'Range' ])
189
+ pcheck { expect(@store['a'].to_a).to eq([ 'Fixnum', 'String', 'Range' ]) }
190
+ else
191
+ expect(a.map! { |e| e.class.to_s }.to_a).to eq([ 'Integer', 'String', 'Range' ])
192
+ pcheck { expect(@store['a'].to_a).to eq([ 'Integer', 'String', 'Range' ]) }
193
+ end
184
194
 
185
195
  a = cpa([ 1, 'cat', 1..1 ])
186
196
  expect(a.map! { 99 }).to eq([ 99, 99, 99])
@@ -242,9 +252,8 @@ describe PEROBS::Array do
242
252
  it 'should catch a leaked PEROBS::ObjectBase object' do
243
253
  @store['a'] = a = @store.new(PEROBS::Array)
244
254
  o = @store.new(PO)
245
- a[0] = o.get_self
246
255
  PEROBS.log.open(StringIO.new)
247
- expect { @store.exit }.to raise_error(PEROBS::FatalError)
256
+ expect { a[0] = o.get_self }.to raise_error(PEROBS::FatalError)
248
257
  PEROBS.log.open($stderr)
249
258
  end
250
259
 
@@ -27,13 +27,16 @@ require 'fileutils'
27
27
 
28
28
  require 'spec_helper'
29
29
  require 'perobs/BTree'
30
+ require 'perobs/ProgressMeter'
30
31
 
31
32
  describe PEROBS::BTree do
32
33
 
33
34
  before(:all) do
34
35
  @db_dir = generate_db_name('BTree')
35
36
  FileUtils.mkdir_p(@db_dir)
36
- @m = PEROBS::BTree.new(@db_dir, 'btree', 11)
37
+ @m = PEROBS::BTree.new(@db_dir, 'btree', 11, PEROBS::ProgressMeter.new)
38
+ PEROBS.log.level = Logger::ERROR
39
+ PEROBS.log.open($stderr)
37
40
  end
38
41
 
39
42
  after(:all) do
@@ -128,7 +131,7 @@ describe PEROBS::BTree do
128
131
  it 'should survive a real-world usage test' do
129
132
  @m.clear
130
133
  ref = {}
131
- 0.upto(50000) do
134
+ 0.upto(20000) do
132
135
  case rand(5)
133
136
  when 0
134
137
  0.upto(2) do
@@ -0,0 +1,214 @@
1
+ # encoding: UTF-8
2
+ #
3
+ # Copyright (c) 2015, 2016 by Chris Schlaeger <chris@taskjuggler.org>
4
+ #
5
+ # This file contains tests for Array that are similar to the tests for the
6
+ # Array implementation in MRI. The ideas of these tests were replicated in
7
+ # this code.
8
+ #
9
+ # MIT License
10
+ #
11
+ # Permission is hereby granted, free of charge, to any person obtaining
12
+ # a copy of this software and associated documentation files (the
13
+ # "Software"), to deal in the Software without restriction, including
14
+ # without limitation the rights to use, copy, modify, merge, publish,
15
+ # distribute, sublicense, and/or sell copies of the Software, and to
16
+ # permit persons to whom the Software is furnished to do so, subject to
17
+ # the following conditions:
18
+ #
19
+ # The above copyright notice and this permission notice shall be
20
+ # included in all copies or substantial portions of the Software.
21
+ #
22
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
23
+ # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
24
+ # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
25
+ # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
26
+ # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
27
+ # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
28
+ # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
29
+
30
+ require 'spec_helper'
31
+
32
+ require 'perobs'
33
+ require 'perobs/BigArray'
34
+
35
+ NODE_ENTRIES = 4
36
+
37
+ describe PEROBS::BigArray do
38
+
39
+ before(:all) do
40
+ @db_name = generate_db_name(__FILE__)
41
+ @store = PEROBS::Store.new(@db_name)
42
+ end
43
+
44
+ before(:each) do
45
+ @store['array'] = @a = @store.new(PEROBS::BigArray, NODE_ENTRIES)
46
+ end
47
+
48
+ after(:each) do
49
+ @store['array'] = @a = nil
50
+ @store.gc
51
+ end
52
+
53
+ after(:all) do
54
+ @store.delete_store
55
+ end
56
+
57
+ it 'should be empty on create' do
58
+ expect(@a.empty?).to be true
59
+ expect(@a.length).to eq(0)
60
+ expect(@a.check).to be true
61
+ expect(@a[0]).to be nil
62
+ end
63
+
64
+ it 'should append the first element' do
65
+ @a << 0
66
+ expect(@a.empty?).to be false
67
+ expect(@a[0]).to eq(0)
68
+ expect(@a.length).to eq(1)
69
+ expect(@a.check).to be true
70
+ end
71
+
72
+ it 'should fill 10 nodes with appends' do
73
+ (10 * NODE_ENTRIES).times do |i|
74
+ @a << i
75
+ expect(@a.check).to be true
76
+ expect(@a.length).to eq(i + 1)
77
+ end
78
+ end
79
+
80
+ it 'should insert at 0' do
81
+ @a.insert(0, 0)
82
+ expect(@a.empty?).to be false
83
+ end
84
+
85
+ it 'should insert at end' do
86
+ 0.upto(3 * NODE_ENTRIES) do |i|
87
+ @a.insert(i, i)
88
+ expect(@a.check).to be true
89
+ expect(@a.length).to eq(i + 1)
90
+ end
91
+ end
92
+
93
+ it 'should insert in the middle' do
94
+ 0.upto(NODE_ENTRIES - 1) do |i|
95
+ @a << 9999
96
+ end
97
+ 0.upto(3 * NODE_ENTRIES) do |i|
98
+ @a.insert(i, i)
99
+ expect(@a.check).to be true
100
+ expect(@a.length).to eq(i + 1 + NODE_ENTRIES)
101
+ end
102
+ end
103
+
104
+ it 'should convert to a Ruby Array' do
105
+ expect(@a.to_a).to eql([])
106
+ (3 * NODE_ENTRIES).times do |i|
107
+ @a << i
108
+ end
109
+ expect(@a.to_a).to eql((0..3 * NODE_ENTRIES - 1).to_a)
110
+ end
111
+
112
+ it 'should support the [] operator' do
113
+ expect(@a[0]).to be nil
114
+ expect{@a[-1]}.to raise_error(IndexError)
115
+ @a[0] = 0
116
+ expect(@a[0]).to eq(0)
117
+ 1.upto(3 * NODE_ENTRIES) do |i|
118
+ @a.insert(i, i)
119
+ end
120
+ 0.upto(3 * NODE_ENTRIES) do |i|
121
+ expect(@a[i]).to eq(i)
122
+ end
123
+ expect(@a[3 * NODE_ENTRIES + 1]).to be nil
124
+ 0.upto(3 * NODE_ENTRIES) do |i|
125
+ expect(@a[-3 * NODE_ENTRIES + i - 1]).to eq(i)
126
+ end
127
+ expect{@a[-3 * NODE_ENTRIES - 2]}.to raise_error(IndexError)
128
+ (3 * NODE_ENTRIES + 1).upto(4 * NODE_ENTRIES) do |i|
129
+ expect(@a[i]).to be nil
130
+ end
131
+ end
132
+
133
+ it 'should delete elements' do
134
+ expect(@a.delete_at(0)).to be nil
135
+ expect(@a.length).to eq(0)
136
+ expect(@a.check).to be true
137
+ expect(@a.delete_at(-1)).to be nil
138
+ expect(@a.length).to eq(0)
139
+ expect(@a.check).to be true
140
+ @a << 0
141
+ expect(@a.delete_at(0)).to eql(0)
142
+ expect(@a.length).to eq(0)
143
+ expect(@a.check).to be true
144
+
145
+ n = 3 * NODE_ENTRIES
146
+ 0.upto(n) { |i| @a.insert(i, i) }
147
+ 0.upto(n) do |i|
148
+ expect(@a.delete_at(0)).to eql(i)
149
+ expect(@a.check).to be true
150
+ end
151
+
152
+ 0.upto(n) { |i| @a.insert(i, i) }
153
+ n.downto(0) do |i|
154
+ expect(@a.delete_at(-1)).to eql(i)
155
+ expect(@a.check).to be true
156
+ end
157
+
158
+ n = 11 * NODE_ENTRIES
159
+ 0.upto(n - 1) { |i| @a.insert(i, i) }
160
+ expect(@a.delete_at(n + 2)).to be nil
161
+ expect(@a.delete_at(-(n + 2))).to be nil
162
+ expect(@a.size).to eql(n)
163
+
164
+ n.times do |i|
165
+ @a.delete_at(rand(@a.size))
166
+ expect(@a.size).to be (n - 1 - i)
167
+ expect(@a.check).to be true
168
+ end
169
+ expect(@a.size).to eql(0)
170
+ end
171
+
172
+ it 'should iterate over all values' do
173
+ n = 3 * NODE_ENTRIES
174
+ 0.upto(n) { |i| @a.insert(i, i) }
175
+
176
+ i = 0
177
+ @a.each do |v|
178
+ expect(v).to eql(i)
179
+ i += 1
180
+ end
181
+ end
182
+
183
+ it 'should iterate over all values in reverse order' do
184
+ n = 3 * NODE_ENTRIES
185
+ 0.upto(n) { |i| @a.insert(i, i) }
186
+
187
+ i = 0
188
+ @a.reverse_each do |v|
189
+ expect(v).to eql(n - i)
190
+ i += 1
191
+ end
192
+ end
193
+
194
+ it 'should persist the data' do
195
+ db_name = generate_db_name(__FILE__ + "_persist")
196
+ store = PEROBS::Store.new(db_name)
197
+ store['array'] = a = store.new(PEROBS::BigArray, NODE_ENTRIES)
198
+
199
+ (3 * NODE_ENTRIES).times do |i|
200
+ a.insert(i, i)
201
+ end
202
+ expect(a.length).to eq(3 * NODE_ENTRIES)
203
+ store.exit
204
+
205
+ store = PEROBS::Store.new(db_name)
206
+ a = store['array']
207
+ (3 * NODE_ENTRIES).times do |i|
208
+ expect(a[i]).to eql(i)
209
+ end
210
+ expect(a.length).to eq(3 * NODE_ENTRIES)
211
+ store.delete_store
212
+ end
213
+
214
+ end
@@ -0,0 +1,144 @@
1
+ # encoding: UTF-8
2
+ #
3
+ # Copyright (c) 2017, 2019 by Chris Schlaeger <chris@taskjuggler.org>
4
+ #
5
+ # This file contains tests for Hash that are similar to the tests for the
6
+ # Hash implementation in MRI. The ideas of these tests were replicated in
7
+ # this code.
8
+ #
9
+ # MIT License
10
+ #
11
+ # Permission is hereby granted, free of charge, to any person obtaining
12
+ # a copy of this software and associated documentation files (the
13
+ # "Software"), to deal in the Software without restriction, including
14
+ # without limitation the rights to use, copy, modify, merge, publish,
15
+ # distribute, sublicense, and/or sell copies of the Software, and to
16
+ # permit persons to whom the Software is furnished to do so, subject to
17
+ # the following conditions:
18
+ #
19
+ # The above copyright notice and this permission notice shall be
20
+ # included in all copies or substantial portions of the Software.
21
+ #
22
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
23
+ # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
24
+ # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
25
+ # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
26
+ # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
27
+ # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
28
+ # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
29
+
30
+ require 'spec_helper'
31
+
32
+ require 'perobs'
33
+ require 'perobs/BigHash'
34
+
35
+ class PEROBS::BigHash
36
+
37
+ # Redefine the hash method to make collisions very likely. This will help to
38
+ # test the collision handling with small amounts of data.
39
+ #def hash_key(key)
40
+ # key.hash & 0xF
41
+ #end
42
+
43
+ end
44
+
45
+ ENTRIES = 200
46
+
47
+ describe PEROBS::Hash do
48
+
49
+ before(:all) do
50
+ @db_name = generate_db_name(__FILE__)
51
+ end
52
+
53
+ before(:each) do
54
+ @store = PEROBS::Store.new(@db_name)
55
+ @h = @store.new(PEROBS::BigHash)
56
+ @store['hash'] = @h
57
+ end
58
+
59
+ after(:each) do
60
+ @store.delete_store
61
+ end
62
+
63
+ it 'should support storing and retrieving an object' do
64
+ expect(@h.check).to be true
65
+ expect(@h.length).to eql(0)
66
+ expect(@h.empty?).to be true
67
+ expect(@h.keys).to eql([])
68
+ @h['foo'] = 'bar'
69
+ expect(@h.check).to be true
70
+ expect(@h['foo']).to eql('bar')
71
+ expect(@h.length).to eql(1)
72
+ expect(@h.keys).to eql([ 'foo' ])
73
+ end
74
+
75
+ it 'should return nil for unknown objects' do
76
+ expect(@h['bar']).to be_nil
77
+ end
78
+
79
+ it 'should be able to store values with hash collisions' do
80
+ ENTRIES.times do |i|
81
+ @h["key#{i}"] = i
82
+ end
83
+ expect(@h.check).to be true
84
+ expect(@h.length).to eql(ENTRIES)
85
+
86
+ ENTRIES.times do |i|
87
+ expect(@h["key#{i}"]).to eql(i)
88
+ end
89
+ end
90
+
91
+ it 'should replace existing entries' do
92
+ ENTRIES.times do |i|
93
+ @h["key#{i}"] = 2 * i
94
+ end
95
+ expect(@h.check).to be true
96
+ expect(@h.length).to eql(ENTRIES)
97
+
98
+ ENTRIES.times do |i|
99
+ expect(@h["key#{i}"]).to eql(2 * i)
100
+ end
101
+ expect(@h.length).to eql(ENTRIES)
102
+ end
103
+
104
+ it 'should fail to delete a non-existing entry' do
105
+ expect(@h.delete('foo')).to be_nil
106
+ expect(@h.check).to be true
107
+ end
108
+
109
+ it 'should delete existing entries' do
110
+ (1..ENTRIES).to_a.shuffle.each do |i|
111
+ @h["key#{i}"] = 2 * i
112
+ end
113
+ expect(@h.check).to be true
114
+ expect(@h.length).to eql(ENTRIES)
115
+ (1..ENTRIES).to_a.shuffle.each do |i|
116
+ expect(@h.delete("key#{i}")).to eql(2 * i)
117
+ end
118
+ end
119
+
120
+ it 'should persist all objects' do
121
+ db_name = generate_db_name(__FILE__ + "_persist")
122
+ store = PEROBS::Store.new(db_name)
123
+ h = store['hash'] = store.new(PEROBS::BigHash)
124
+ n = ENTRIES
125
+ n.times do |i|
126
+ h["key#{i}"] = 2 * i
127
+ end
128
+ expect(h.check).to be true
129
+ expect(h.length).to eql(n)
130
+ store.exit
131
+
132
+ store = PEROBS::Store.new(db_name)
133
+ expect(store.check).to eql(0)
134
+ h = store['hash']
135
+ n.times do |i|
136
+ expect(h["key#{i}"]).to eql(2 * i)
137
+ end
138
+ expect(h.check).to be true
139
+ expect(h.length).to eql(n)
140
+ store.delete_store
141
+ end
142
+
143
+ end
144
+