perobs 4.0.0 → 4.4.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/README.md +27 -16
- data/lib/perobs/Array.rb +66 -19
- data/lib/perobs/BTree.rb +106 -15
- data/lib/perobs/BTreeBlob.rb +4 -3
- data/lib/perobs/BTreeDB.rb +5 -4
- data/lib/perobs/BTreeNode.rb +482 -156
- data/lib/perobs/BTreeNodeLink.rb +10 -0
- data/lib/perobs/BigArray.rb +285 -0
- data/lib/perobs/BigArrayNode.rb +1002 -0
- data/lib/perobs/BigHash.rb +246 -0
- data/lib/perobs/BigTree.rb +197 -0
- data/lib/perobs/BigTreeNode.rb +873 -0
- data/lib/perobs/Cache.rb +48 -10
- data/lib/perobs/ConsoleProgressMeter.rb +61 -0
- data/lib/perobs/DataBase.rb +4 -3
- data/lib/perobs/DynamoDB.rb +57 -15
- data/lib/perobs/EquiBlobsFile.rb +155 -50
- data/lib/perobs/FNV_Hash_1a_64.rb +54 -0
- data/lib/perobs/FlatFile.rb +519 -227
- data/lib/perobs/FlatFileBlobHeader.rb +113 -54
- data/lib/perobs/FlatFileDB.rb +49 -23
- data/lib/perobs/FuzzyStringMatcher.rb +175 -0
- data/lib/perobs/Hash.rb +127 -33
- data/lib/perobs/IDList.rb +144 -0
- data/lib/perobs/IDListPage.rb +107 -0
- data/lib/perobs/IDListPageFile.rb +180 -0
- data/lib/perobs/IDListPageRecord.rb +142 -0
- data/lib/perobs/Object.rb +18 -15
- data/lib/perobs/ObjectBase.rb +46 -5
- data/lib/perobs/PersistentObjectCache.rb +57 -68
- data/lib/perobs/PersistentObjectCacheLine.rb +24 -12
- data/lib/perobs/ProgressMeter.rb +97 -0
- data/lib/perobs/SpaceManager.rb +273 -0
- data/lib/perobs/SpaceTree.rb +21 -12
- data/lib/perobs/SpaceTreeNode.rb +53 -61
- data/lib/perobs/Store.rb +264 -145
- data/lib/perobs/version.rb +1 -1
- data/lib/perobs.rb +2 -0
- data/perobs.gemspec +4 -4
- data/test/Array_spec.rb +15 -6
- data/test/BTree_spec.rb +6 -2
- data/test/BigArray_spec.rb +261 -0
- data/test/BigHash_spec.rb +152 -0
- data/test/BigTreeNode_spec.rb +153 -0
- data/test/BigTree_spec.rb +259 -0
- data/test/EquiBlobsFile_spec.rb +105 -1
- data/test/FNV_Hash_1a_64_spec.rb +59 -0
- data/test/FlatFileDB_spec.rb +198 -14
- data/test/FuzzyStringMatcher_spec.rb +261 -0
- data/test/Hash_spec.rb +13 -3
- data/test/IDList_spec.rb +77 -0
- data/test/LegacyDBs/LegacyDB.rb +155 -0
- data/test/LegacyDBs/version_3/class_map.json +1 -0
- data/test/LegacyDBs/version_3/config.json +1 -0
- data/test/LegacyDBs/version_3/database.blobs +0 -0
- data/test/LegacyDBs/version_3/database_spaces.blobs +0 -0
- data/test/LegacyDBs/version_3/index.blobs +0 -0
- data/test/LegacyDBs/version_3/version +1 -0
- data/test/LockFile_spec.rb +9 -6
- data/test/SpaceManager_spec.rb +176 -0
- data/test/SpaceTree_spec.rb +4 -1
- data/test/Store_spec.rb +305 -203
- data/test/spec_helper.rb +9 -4
- metadata +57 -16
- data/lib/perobs/BTreeNodeCache.rb +0 -109
- data/lib/perobs/TreeDB.rb +0 -277
@@ -0,0 +1,176 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
#
|
3
|
+
# Copyright (c) 2020 by Chris Schlaeger <chris@taskjuggler.org>
|
4
|
+
#
|
5
|
+
# MIT License
|
6
|
+
#
|
7
|
+
# Permission is hereby granted, free of charge, to any person obtaining
|
8
|
+
# a copy of this software and associated documentation files (the
|
9
|
+
# "Software"), to deal in the Software without restriction, including
|
10
|
+
# without limitation the rights to use, copy, modify, merge, publish,
|
11
|
+
# distribute, sublicense, and/or sell copies of the Software, and to
|
12
|
+
# permit persons to whom the Software is furnished to do so, subject to
|
13
|
+
# the following conditions:
|
14
|
+
#
|
15
|
+
# The above copyright notice and this permission notice shall be
|
16
|
+
# included in all copies or substantial portions of the Software.
|
17
|
+
#
|
18
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
19
|
+
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
20
|
+
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
21
|
+
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
22
|
+
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
23
|
+
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
24
|
+
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
25
|
+
|
26
|
+
require 'fileutils'
|
27
|
+
|
28
|
+
require 'spec_helper'
|
29
|
+
require 'perobs/SpaceManager'
|
30
|
+
require 'perobs/ProgressMeter'
|
31
|
+
|
32
|
+
describe PEROBS::SpaceManager do
|
33
|
+
|
34
|
+
before(:all) do
|
35
|
+
@db_dir = generate_db_name('SpaceManager')
|
36
|
+
FileUtils.mkdir_p(@db_dir)
|
37
|
+
@m = PEROBS::SpaceManager.new(@db_dir, PEROBS::ProgressMeter.new, 5)
|
38
|
+
PEROBS.log.level = Logger::ERROR
|
39
|
+
PEROBS.log.open($stderr)
|
40
|
+
end
|
41
|
+
|
42
|
+
after(:all) do
|
43
|
+
FileUtils.rm_rf(@db_dir)
|
44
|
+
end
|
45
|
+
|
46
|
+
it 'should open the space manager' do
|
47
|
+
@m.open
|
48
|
+
expect(@m.to_a).to eql([])
|
49
|
+
end
|
50
|
+
|
51
|
+
it 'should support adding spaces' do
|
52
|
+
@m.add_space(80, 8)
|
53
|
+
expect(@m.has_space?(80, 8)).to be true
|
54
|
+
expect(@m.to_a).to eql([[80, 8]])
|
55
|
+
expect(@m.check).to be true
|
56
|
+
@m.add_space(40, 4)
|
57
|
+
expect(@m.has_space?(40, 4)).to be true
|
58
|
+
expect(@m.to_a).to eql([[40, 4], [80, 8]])
|
59
|
+
@m.add_space(20, 2)
|
60
|
+
expect(@m.has_space?(20, 2)).to be true
|
61
|
+
expect(@m.to_a).to eql([[20, 2], [40, 4], [80, 8]])
|
62
|
+
@m.add_space(160, 16)
|
63
|
+
expect(@m.has_space?(160, 16)).to be true
|
64
|
+
expect(@m.to_a).to eql([[20, 2], [40, 4], [80, 8], [160, 16]])
|
65
|
+
@m.add_space(320, 32)
|
66
|
+
expect(@m.has_space?(320, 32)).to be true
|
67
|
+
expect(@m.to_a).to eql([[20, 2], [40, 4], [80, 8], [160, 16], [320, 32]])
|
68
|
+
@m.add_space(100, 10)
|
69
|
+
expect(@m.has_space?(100, 10)).to be true
|
70
|
+
expect(@m.to_a).to eql([[20, 2], [40, 4], [80, 8], [100, 10], [160, 16], [320, 32]])
|
71
|
+
@m.add_space(81, 8)
|
72
|
+
expect(@m.has_space?(81, 8)).to be true
|
73
|
+
expect(@m.to_a).to eql([[20, 2], [40, 4], [80, 8], [81, 8], [100, 10], [160, 16], [320, 32]])
|
74
|
+
expect(@m.check).to be true
|
75
|
+
end
|
76
|
+
|
77
|
+
it 'should keep values over an close/open' do
|
78
|
+
@m.add_space(1, 15)
|
79
|
+
expect(@m.check).to be true
|
80
|
+
@m.close
|
81
|
+
@m.open
|
82
|
+
expect(@m.check).to be true
|
83
|
+
expect(@m.to_a).to eql([[1, 15], [20, 2], [40, 4], [80, 8], [81, 8], [100, 10], [160, 16], [320, 32]])
|
84
|
+
end
|
85
|
+
|
86
|
+
it 'should support clearing all spaces' do
|
87
|
+
@m.clear
|
88
|
+
expect(@m.to_a).to eql([])
|
89
|
+
@m.add_space(1, 1)
|
90
|
+
@m.add_space(2, 2)
|
91
|
+
@m.clear
|
92
|
+
expect(@m.to_a).to eql([])
|
93
|
+
end
|
94
|
+
|
95
|
+
it 'should return exactly matching spaces' do
|
96
|
+
@m.clear
|
97
|
+
add_sizes([ 10, 5, 15, 10 ])
|
98
|
+
expect(@m.to_a).to eql([[0, 10], [1, 5], [2, 15], [3, 10]])
|
99
|
+
expect(@m.get_space(10)).to eql([3, 10])
|
100
|
+
expect(@m.get_space(10)).to eql([0, 10])
|
101
|
+
expect(@m.to_a).to eql([[1, 5], [2, 15]])
|
102
|
+
expect(@m.added_spaces).to eql(4)
|
103
|
+
expect(@m.recycled_spaces).to eql(2)
|
104
|
+
expect(@m.failed_requests).to eql(0)
|
105
|
+
|
106
|
+
@m.clear
|
107
|
+
add_sizes([ 10, 5, 15, 10, 10 ])
|
108
|
+
expect(@m.to_a).to eql([[0, 10], [1, 5], [2, 15], [3, 10], [4, 10]])
|
109
|
+
expect(@m.get_space(10)).to eql([4, 10])
|
110
|
+
expect(@m.get_space(10)).to eql([3, 10])
|
111
|
+
expect(@m.get_space(10)).to eql([0, 10])
|
112
|
+
expect(@m.to_a).to eql([[1, 5], [2, 15]])
|
113
|
+
expect(@m.added_spaces).to eql(5)
|
114
|
+
expect(@m.recycled_spaces).to eql(3)
|
115
|
+
expect(@m.failed_requests).to eql(0)
|
116
|
+
end
|
117
|
+
|
118
|
+
it "should return nil if no space can be found" do
|
119
|
+
expect(@m.get_space(42)).to be nil
|
120
|
+
expect(@m.get_space(9)).to be nil
|
121
|
+
expect(@m.get_space(11)).to be nil
|
122
|
+
expect(@m.recycled_spaces).to eql(3)
|
123
|
+
expect(@m.failed_requests).to eql(3)
|
124
|
+
end
|
125
|
+
|
126
|
+
it 'should support a real-world traffic pattern' do
|
127
|
+
address = 0
|
128
|
+
spaces = []
|
129
|
+
@m.clear
|
130
|
+
0.upto(1500) do
|
131
|
+
case rand(4)
|
132
|
+
when 0
|
133
|
+
# Insert new values
|
134
|
+
rand(9).times do
|
135
|
+
size = 20 + rand(80)
|
136
|
+
@m.add_space(address, size)
|
137
|
+
spaces << [ address, size ]
|
138
|
+
address += size
|
139
|
+
end
|
140
|
+
when 1
|
141
|
+
# Remove some values
|
142
|
+
rand(7).times do
|
143
|
+
size = rand(110)
|
144
|
+
if (space = @m.get_space(size))
|
145
|
+
expect(spaces.include?(space)).to be true
|
146
|
+
spaces.delete(space)
|
147
|
+
end
|
148
|
+
end
|
149
|
+
when 2
|
150
|
+
if rand(10) == 0
|
151
|
+
expect(@m.check).to be true
|
152
|
+
spaces.each do |address, size|
|
153
|
+
expect(@m.has_space?(address, size)).to be true
|
154
|
+
end
|
155
|
+
@m.to_a.each do |address, size|
|
156
|
+
expect(spaces.include?([ address, size ])).to be true
|
157
|
+
end
|
158
|
+
end
|
159
|
+
when 3
|
160
|
+
if rand(200) == 0
|
161
|
+
expect(@m.check).to be true
|
162
|
+
@m.close
|
163
|
+
@m.open
|
164
|
+
expect(@m.check).to be true
|
165
|
+
end
|
166
|
+
end
|
167
|
+
end
|
168
|
+
end
|
169
|
+
|
170
|
+
def add_sizes(sizes)
|
171
|
+
sizes.each_with_index do |size, i|
|
172
|
+
@m.add_space(i, size)
|
173
|
+
end
|
174
|
+
end
|
175
|
+
|
176
|
+
end
|
data/test/SpaceTree_spec.rb
CHANGED
@@ -27,13 +27,16 @@ require 'fileutils'
|
|
27
27
|
|
28
28
|
require 'spec_helper'
|
29
29
|
require 'perobs/SpaceTree'
|
30
|
+
require 'perobs/ProgressMeter'
|
30
31
|
|
31
32
|
describe PEROBS::SpaceTree do
|
32
33
|
|
33
34
|
before(:all) do
|
34
35
|
@db_dir = generate_db_name('SpaceTree')
|
35
36
|
FileUtils.mkdir_p(@db_dir)
|
36
|
-
@m = PEROBS::SpaceTree.new(@db_dir)
|
37
|
+
@m = PEROBS::SpaceTree.new(@db_dir, PEROBS::ProgressMeter.new)
|
38
|
+
PEROBS.log.level = Logger::ERROR
|
39
|
+
PEROBS.log.open($stderr)
|
37
40
|
end
|
38
41
|
|
39
42
|
after(:all) do
|