perobs 4.0.0 → 4.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. checksums.yaml +4 -4
  2. data/lib/perobs.rb +1 -0
  3. data/lib/perobs/Array.rb +66 -19
  4. data/lib/perobs/BTree.rb +83 -12
  5. data/lib/perobs/BTreeBlob.rb +1 -1
  6. data/lib/perobs/BTreeDB.rb +2 -2
  7. data/lib/perobs/BTreeNode.rb +365 -85
  8. data/lib/perobs/BigArray.rb +267 -0
  9. data/lib/perobs/BigArrayNode.rb +998 -0
  10. data/lib/perobs/BigHash.rb +262 -0
  11. data/lib/perobs/BigTree.rb +184 -0
  12. data/lib/perobs/BigTreeNode.rb +873 -0
  13. data/lib/perobs/ConsoleProgressMeter.rb +61 -0
  14. data/lib/perobs/DataBase.rb +4 -3
  15. data/lib/perobs/DynamoDB.rb +57 -15
  16. data/lib/perobs/EquiBlobsFile.rb +143 -51
  17. data/lib/perobs/FNV_Hash_1a_64.rb +54 -0
  18. data/lib/perobs/FlatFile.rb +363 -203
  19. data/lib/perobs/FlatFileBlobHeader.rb +98 -54
  20. data/lib/perobs/FlatFileDB.rb +42 -20
  21. data/lib/perobs/Hash.rb +58 -13
  22. data/lib/perobs/IDList.rb +144 -0
  23. data/lib/perobs/IDListPage.rb +107 -0
  24. data/lib/perobs/IDListPageFile.rb +180 -0
  25. data/lib/perobs/IDListPageRecord.rb +142 -0
  26. data/lib/perobs/Object.rb +18 -15
  27. data/lib/perobs/ObjectBase.rb +38 -4
  28. data/lib/perobs/PersistentObjectCache.rb +53 -67
  29. data/lib/perobs/PersistentObjectCacheLine.rb +24 -12
  30. data/lib/perobs/ProgressMeter.rb +97 -0
  31. data/lib/perobs/SpaceTree.rb +21 -12
  32. data/lib/perobs/SpaceTreeNode.rb +53 -61
  33. data/lib/perobs/Store.rb +71 -32
  34. data/lib/perobs/version.rb +1 -1
  35. data/perobs.gemspec +4 -4
  36. data/test/Array_spec.rb +15 -6
  37. data/test/BTree_spec.rb +5 -2
  38. data/test/BigArray_spec.rb +214 -0
  39. data/test/BigHash_spec.rb +144 -0
  40. data/test/BigTreeNode_spec.rb +153 -0
  41. data/test/BigTree_spec.rb +259 -0
  42. data/test/EquiBlobsFile_spec.rb +105 -1
  43. data/test/FNV_Hash_1a_64_spec.rb +59 -0
  44. data/test/FlatFileDB_spec.rb +63 -14
  45. data/test/Hash_spec.rb +1 -2
  46. data/test/IDList_spec.rb +77 -0
  47. data/test/LegacyDBs/LegacyDB.rb +151 -0
  48. data/test/LegacyDBs/version_3/class_map.json +1 -0
  49. data/test/LegacyDBs/version_3/config.json +1 -0
  50. data/test/LegacyDBs/version_3/database.blobs +0 -0
  51. data/test/LegacyDBs/version_3/database_spaces.blobs +0 -0
  52. data/test/LegacyDBs/version_3/index.blobs +0 -0
  53. data/test/LegacyDBs/version_3/version +1 -0
  54. data/test/LockFile_spec.rb +9 -6
  55. data/test/SpaceTree_spec.rb +4 -1
  56. data/test/Store_spec.rb +290 -199
  57. data/test/spec_helper.rb +9 -4
  58. metadata +47 -10
  59. data/lib/perobs/TreeDB.rb +0 -277
@@ -1,4 +1,4 @@
1
1
  module PEROBS
2
2
  # The version number
3
- VERSION = "4.0.0"
3
+ VERSION = "4.1.0"
4
4
  end
@@ -16,9 +16,9 @@ GEM_SPEC = Gem::Specification.new do |spec|
16
16
  spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
17
17
  spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
18
18
  spec.require_paths = ["lib"]
19
- spec.required_ruby_version = '>=2.0'
19
+ spec.required_ruby_version = '>=2.3'
20
20
 
21
- spec.add_development_dependency 'bundler', '~> 1.6'
22
- spec.add_development_dependency 'yard', '~>0.8.7'
23
- spec.add_development_dependency 'rake', '~> 10.0'
21
+ spec.add_development_dependency 'bundler', '~> 2.3'
22
+ spec.add_development_dependency 'yard', '~>0.9.12'
23
+ spec.add_development_dependency 'rake', '~> 10.1'
24
24
  end
@@ -169,8 +169,13 @@ describe PEROBS::Array do
169
169
 
170
170
  it 'should support collect!()' do
171
171
  a = cpa([ 1, 'cat', 1..1 ])
172
- expect(a.collect! { |e| e.class.to_s }).to eq([ 'Integer', 'String', 'Range' ])
173
- pcheck { expect(@store['a'].to_a).to eq([ 'Integer', 'String', 'Range' ]) }
172
+ if RUBY_VERSION < '2.2'
173
+ expect(a.collect! { |e| e.class.to_s }.to_a).to eq([ 'Fixnum', 'String', 'Range' ])
174
+ pcheck { expect(@store['a'].to_a).to eq([ 'Fixnum', 'String', 'Range' ]) }
175
+ else
176
+ expect(a.collect! { |e| e.class.to_s }.to_a).to eq([ 'Integer', 'String', 'Range' ])
177
+ pcheck { expect(@store['a'].to_a).to eq([ 'Integer', 'String', 'Range' ]) }
178
+ end
174
179
 
175
180
  a = cpa([ 1, 'cat', 1..1 ])
176
181
  expect(a.collect! { 99 }).to eq([ 99, 99, 99])
@@ -179,8 +184,13 @@ describe PEROBS::Array do
179
184
 
180
185
  it 'should support map!()' do
181
186
  a = cpa([ 1, 'cat', 1..1 ])
182
- expect(a.map! { |e| e.class.to_s }).to eq([ 'Integer', 'String', 'Range' ])
183
- pcheck { expect(@store['a']).to eq([ 'Integer', 'String', 'Range' ]) }
187
+ if RUBY_VERSION < '2.2'
188
+ expect(a.map! { |e| e.class.to_s }.to_a).to eq([ 'Fixnum', 'String', 'Range' ])
189
+ pcheck { expect(@store['a'].to_a).to eq([ 'Fixnum', 'String', 'Range' ]) }
190
+ else
191
+ expect(a.map! { |e| e.class.to_s }.to_a).to eq([ 'Integer', 'String', 'Range' ])
192
+ pcheck { expect(@store['a'].to_a).to eq([ 'Integer', 'String', 'Range' ]) }
193
+ end
184
194
 
185
195
  a = cpa([ 1, 'cat', 1..1 ])
186
196
  expect(a.map! { 99 }).to eq([ 99, 99, 99])
@@ -242,9 +252,8 @@ describe PEROBS::Array do
242
252
  it 'should catch a leaked PEROBS::ObjectBase object' do
243
253
  @store['a'] = a = @store.new(PEROBS::Array)
244
254
  o = @store.new(PO)
245
- a[0] = o.get_self
246
255
  PEROBS.log.open(StringIO.new)
247
- expect { @store.exit }.to raise_error(PEROBS::FatalError)
256
+ expect { a[0] = o.get_self }.to raise_error(PEROBS::FatalError)
248
257
  PEROBS.log.open($stderr)
249
258
  end
250
259
 
@@ -27,13 +27,16 @@ require 'fileutils'
27
27
 
28
28
  require 'spec_helper'
29
29
  require 'perobs/BTree'
30
+ require 'perobs/ProgressMeter'
30
31
 
31
32
  describe PEROBS::BTree do
32
33
 
33
34
  before(:all) do
34
35
  @db_dir = generate_db_name('BTree')
35
36
  FileUtils.mkdir_p(@db_dir)
36
- @m = PEROBS::BTree.new(@db_dir, 'btree', 11)
37
+ @m = PEROBS::BTree.new(@db_dir, 'btree', 11, PEROBS::ProgressMeter.new)
38
+ PEROBS.log.level = Logger::ERROR
39
+ PEROBS.log.open($stderr)
37
40
  end
38
41
 
39
42
  after(:all) do
@@ -128,7 +131,7 @@ describe PEROBS::BTree do
128
131
  it 'should survive a real-world usage test' do
129
132
  @m.clear
130
133
  ref = {}
131
- 0.upto(50000) do
134
+ 0.upto(20000) do
132
135
  case rand(5)
133
136
  when 0
134
137
  0.upto(2) do
@@ -0,0 +1,214 @@
1
+ # encoding: UTF-8
2
+ #
3
+ # Copyright (c) 2015, 2016 by Chris Schlaeger <chris@taskjuggler.org>
4
+ #
5
+ # This file contains tests for Array that are similar to the tests for the
6
+ # Array implementation in MRI. The ideas of these tests were replicated in
7
+ # this code.
8
+ #
9
+ # MIT License
10
+ #
11
+ # Permission is hereby granted, free of charge, to any person obtaining
12
+ # a copy of this software and associated documentation files (the
13
+ # "Software"), to deal in the Software without restriction, including
14
+ # without limitation the rights to use, copy, modify, merge, publish,
15
+ # distribute, sublicense, and/or sell copies of the Software, and to
16
+ # permit persons to whom the Software is furnished to do so, subject to
17
+ # the following conditions:
18
+ #
19
+ # The above copyright notice and this permission notice shall be
20
+ # included in all copies or substantial portions of the Software.
21
+ #
22
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
23
+ # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
24
+ # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
25
+ # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
26
+ # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
27
+ # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
28
+ # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
29
+
30
+ require 'spec_helper'
31
+
32
+ require 'perobs'
33
+ require 'perobs/BigArray'
34
+
35
+ NODE_ENTRIES = 4
36
+
37
+ describe PEROBS::BigArray do
38
+
39
+ before(:all) do
40
+ @db_name = generate_db_name(__FILE__)
41
+ @store = PEROBS::Store.new(@db_name)
42
+ end
43
+
44
+ before(:each) do
45
+ @store['array'] = @a = @store.new(PEROBS::BigArray, NODE_ENTRIES)
46
+ end
47
+
48
+ after(:each) do
49
+ @store['array'] = @a = nil
50
+ @store.gc
51
+ end
52
+
53
+ after(:all) do
54
+ @store.delete_store
55
+ end
56
+
57
+ it 'should be empty on create' do
58
+ expect(@a.empty?).to be true
59
+ expect(@a.length).to eq(0)
60
+ expect(@a.check).to be true
61
+ expect(@a[0]).to be nil
62
+ end
63
+
64
+ it 'should append the first element' do
65
+ @a << 0
66
+ expect(@a.empty?).to be false
67
+ expect(@a[0]).to eq(0)
68
+ expect(@a.length).to eq(1)
69
+ expect(@a.check).to be true
70
+ end
71
+
72
+ it 'should fill 10 nodes with appends' do
73
+ (10 * NODE_ENTRIES).times do |i|
74
+ @a << i
75
+ expect(@a.check).to be true
76
+ expect(@a.length).to eq(i + 1)
77
+ end
78
+ end
79
+
80
+ it 'should insert at 0' do
81
+ @a.insert(0, 0)
82
+ expect(@a.empty?).to be false
83
+ end
84
+
85
+ it 'should insert at end' do
86
+ 0.upto(3 * NODE_ENTRIES) do |i|
87
+ @a.insert(i, i)
88
+ expect(@a.check).to be true
89
+ expect(@a.length).to eq(i + 1)
90
+ end
91
+ end
92
+
93
+ it 'should insert in the middle' do
94
+ 0.upto(NODE_ENTRIES - 1) do |i|
95
+ @a << 9999
96
+ end
97
+ 0.upto(3 * NODE_ENTRIES) do |i|
98
+ @a.insert(i, i)
99
+ expect(@a.check).to be true
100
+ expect(@a.length).to eq(i + 1 + NODE_ENTRIES)
101
+ end
102
+ end
103
+
104
+ it 'should convert to a Ruby Array' do
105
+ expect(@a.to_a).to eql([])
106
+ (3 * NODE_ENTRIES).times do |i|
107
+ @a << i
108
+ end
109
+ expect(@a.to_a).to eql((0..3 * NODE_ENTRIES - 1).to_a)
110
+ end
111
+
112
+ it 'should support the [] operator' do
113
+ expect(@a[0]).to be nil
114
+ expect{@a[-1]}.to raise_error(IndexError)
115
+ @a[0] = 0
116
+ expect(@a[0]).to eq(0)
117
+ 1.upto(3 * NODE_ENTRIES) do |i|
118
+ @a.insert(i, i)
119
+ end
120
+ 0.upto(3 * NODE_ENTRIES) do |i|
121
+ expect(@a[i]).to eq(i)
122
+ end
123
+ expect(@a[3 * NODE_ENTRIES + 1]).to be nil
124
+ 0.upto(3 * NODE_ENTRIES) do |i|
125
+ expect(@a[-3 * NODE_ENTRIES + i - 1]).to eq(i)
126
+ end
127
+ expect{@a[-3 * NODE_ENTRIES - 2]}.to raise_error(IndexError)
128
+ (3 * NODE_ENTRIES + 1).upto(4 * NODE_ENTRIES) do |i|
129
+ expect(@a[i]).to be nil
130
+ end
131
+ end
132
+
133
+ it 'should delete elements' do
134
+ expect(@a.delete_at(0)).to be nil
135
+ expect(@a.length).to eq(0)
136
+ expect(@a.check).to be true
137
+ expect(@a.delete_at(-1)).to be nil
138
+ expect(@a.length).to eq(0)
139
+ expect(@a.check).to be true
140
+ @a << 0
141
+ expect(@a.delete_at(0)).to eql(0)
142
+ expect(@a.length).to eq(0)
143
+ expect(@a.check).to be true
144
+
145
+ n = 3 * NODE_ENTRIES
146
+ 0.upto(n) { |i| @a.insert(i, i) }
147
+ 0.upto(n) do |i|
148
+ expect(@a.delete_at(0)).to eql(i)
149
+ expect(@a.check).to be true
150
+ end
151
+
152
+ 0.upto(n) { |i| @a.insert(i, i) }
153
+ n.downto(0) do |i|
154
+ expect(@a.delete_at(-1)).to eql(i)
155
+ expect(@a.check).to be true
156
+ end
157
+
158
+ n = 11 * NODE_ENTRIES
159
+ 0.upto(n - 1) { |i| @a.insert(i, i) }
160
+ expect(@a.delete_at(n + 2)).to be nil
161
+ expect(@a.delete_at(-(n + 2))).to be nil
162
+ expect(@a.size).to eql(n)
163
+
164
+ n.times do |i|
165
+ @a.delete_at(rand(@a.size))
166
+ expect(@a.size).to be (n - 1 - i)
167
+ expect(@a.check).to be true
168
+ end
169
+ expect(@a.size).to eql(0)
170
+ end
171
+
172
+ it 'should iterate over all values' do
173
+ n = 3 * NODE_ENTRIES
174
+ 0.upto(n) { |i| @a.insert(i, i) }
175
+
176
+ i = 0
177
+ @a.each do |v|
178
+ expect(v).to eql(i)
179
+ i += 1
180
+ end
181
+ end
182
+
183
+ it 'should iterate over all values in reverse order' do
184
+ n = 3 * NODE_ENTRIES
185
+ 0.upto(n) { |i| @a.insert(i, i) }
186
+
187
+ i = 0
188
+ @a.reverse_each do |v|
189
+ expect(v).to eql(n - i)
190
+ i += 1
191
+ end
192
+ end
193
+
194
+ it 'should persist the data' do
195
+ db_name = generate_db_name(__FILE__ + "_persist")
196
+ store = PEROBS::Store.new(db_name)
197
+ store['array'] = a = store.new(PEROBS::BigArray, NODE_ENTRIES)
198
+
199
+ (3 * NODE_ENTRIES).times do |i|
200
+ a.insert(i, i)
201
+ end
202
+ expect(a.length).to eq(3 * NODE_ENTRIES)
203
+ store.exit
204
+
205
+ store = PEROBS::Store.new(db_name)
206
+ a = store['array']
207
+ (3 * NODE_ENTRIES).times do |i|
208
+ expect(a[i]).to eql(i)
209
+ end
210
+ expect(a.length).to eq(3 * NODE_ENTRIES)
211
+ store.delete_store
212
+ end
213
+
214
+ end
@@ -0,0 +1,144 @@
1
+ # encoding: UTF-8
2
+ #
3
+ # Copyright (c) 2017, 2019 by Chris Schlaeger <chris@taskjuggler.org>
4
+ #
5
+ # This file contains tests for Hash that are similar to the tests for the
6
+ # Hash implementation in MRI. The ideas of these tests were replicated in
7
+ # this code.
8
+ #
9
+ # MIT License
10
+ #
11
+ # Permission is hereby granted, free of charge, to any person obtaining
12
+ # a copy of this software and associated documentation files (the
13
+ # "Software"), to deal in the Software without restriction, including
14
+ # without limitation the rights to use, copy, modify, merge, publish,
15
+ # distribute, sublicense, and/or sell copies of the Software, and to
16
+ # permit persons to whom the Software is furnished to do so, subject to
17
+ # the following conditions:
18
+ #
19
+ # The above copyright notice and this permission notice shall be
20
+ # included in all copies or substantial portions of the Software.
21
+ #
22
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
23
+ # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
24
+ # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
25
+ # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
26
+ # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
27
+ # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
28
+ # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
29
+
30
+ require 'spec_helper'
31
+
32
+ require 'perobs'
33
+ require 'perobs/BigHash'
34
+
35
+ class PEROBS::BigHash
36
+
37
+ # Redefine the hash method to make collisions very likely. This will help to
38
+ # test the collision handling with small amounts of data.
39
+ #def hash_key(key)
40
+ # key.hash & 0xF
41
+ #end
42
+
43
+ end
44
+
45
+ ENTRIES = 200
46
+
47
+ describe PEROBS::Hash do
48
+
49
+ before(:all) do
50
+ @db_name = generate_db_name(__FILE__)
51
+ end
52
+
53
+ before(:each) do
54
+ @store = PEROBS::Store.new(@db_name)
55
+ @h = @store.new(PEROBS::BigHash)
56
+ @store['hash'] = @h
57
+ end
58
+
59
+ after(:each) do
60
+ @store.delete_store
61
+ end
62
+
63
+ it 'should support storing and retrieving an object' do
64
+ expect(@h.check).to be true
65
+ expect(@h.length).to eql(0)
66
+ expect(@h.empty?).to be true
67
+ expect(@h.keys).to eql([])
68
+ @h['foo'] = 'bar'
69
+ expect(@h.check).to be true
70
+ expect(@h['foo']).to eql('bar')
71
+ expect(@h.length).to eql(1)
72
+ expect(@h.keys).to eql([ 'foo' ])
73
+ end
74
+
75
+ it 'should return nil for unknown objects' do
76
+ expect(@h['bar']).to be_nil
77
+ end
78
+
79
+ it 'should be able to store values with hash collisions' do
80
+ ENTRIES.times do |i|
81
+ @h["key#{i}"] = i
82
+ end
83
+ expect(@h.check).to be true
84
+ expect(@h.length).to eql(ENTRIES)
85
+
86
+ ENTRIES.times do |i|
87
+ expect(@h["key#{i}"]).to eql(i)
88
+ end
89
+ end
90
+
91
+ it 'should replace existing entries' do
92
+ ENTRIES.times do |i|
93
+ @h["key#{i}"] = 2 * i
94
+ end
95
+ expect(@h.check).to be true
96
+ expect(@h.length).to eql(ENTRIES)
97
+
98
+ ENTRIES.times do |i|
99
+ expect(@h["key#{i}"]).to eql(2 * i)
100
+ end
101
+ expect(@h.length).to eql(ENTRIES)
102
+ end
103
+
104
+ it 'should fail to delete a non-existing entry' do
105
+ expect(@h.delete('foo')).to be_nil
106
+ expect(@h.check).to be true
107
+ end
108
+
109
+ it 'should delete existing entries' do
110
+ (1..ENTRIES).to_a.shuffle.each do |i|
111
+ @h["key#{i}"] = 2 * i
112
+ end
113
+ expect(@h.check).to be true
114
+ expect(@h.length).to eql(ENTRIES)
115
+ (1..ENTRIES).to_a.shuffle.each do |i|
116
+ expect(@h.delete("key#{i}")).to eql(2 * i)
117
+ end
118
+ end
119
+
120
+ it 'should persist all objects' do
121
+ db_name = generate_db_name(__FILE__ + "_persist")
122
+ store = PEROBS::Store.new(db_name)
123
+ h = store['hash'] = store.new(PEROBS::BigHash)
124
+ n = ENTRIES
125
+ n.times do |i|
126
+ h["key#{i}"] = 2 * i
127
+ end
128
+ expect(h.check).to be true
129
+ expect(h.length).to eql(n)
130
+ store.exit
131
+
132
+ store = PEROBS::Store.new(db_name)
133
+ expect(store.check).to eql(0)
134
+ h = store['hash']
135
+ n.times do |i|
136
+ expect(h["key#{i}"]).to eql(2 * i)
137
+ end
138
+ expect(h.check).to be true
139
+ expect(h.length).to eql(n)
140
+ store.delete_store
141
+ end
142
+
143
+ end
144
+