perobs 4.0.0 → 4.4.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (67) hide show
  1. checksums.yaml +5 -5
  2. data/README.md +27 -16
  3. data/lib/perobs/Array.rb +66 -19
  4. data/lib/perobs/BTree.rb +106 -15
  5. data/lib/perobs/BTreeBlob.rb +4 -3
  6. data/lib/perobs/BTreeDB.rb +5 -4
  7. data/lib/perobs/BTreeNode.rb +482 -156
  8. data/lib/perobs/BTreeNodeLink.rb +10 -0
  9. data/lib/perobs/BigArray.rb +285 -0
  10. data/lib/perobs/BigArrayNode.rb +1002 -0
  11. data/lib/perobs/BigHash.rb +246 -0
  12. data/lib/perobs/BigTree.rb +197 -0
  13. data/lib/perobs/BigTreeNode.rb +873 -0
  14. data/lib/perobs/Cache.rb +48 -10
  15. data/lib/perobs/ConsoleProgressMeter.rb +61 -0
  16. data/lib/perobs/DataBase.rb +4 -3
  17. data/lib/perobs/DynamoDB.rb +57 -15
  18. data/lib/perobs/EquiBlobsFile.rb +155 -50
  19. data/lib/perobs/FNV_Hash_1a_64.rb +54 -0
  20. data/lib/perobs/FlatFile.rb +519 -227
  21. data/lib/perobs/FlatFileBlobHeader.rb +113 -54
  22. data/lib/perobs/FlatFileDB.rb +49 -23
  23. data/lib/perobs/FuzzyStringMatcher.rb +175 -0
  24. data/lib/perobs/Hash.rb +127 -33
  25. data/lib/perobs/IDList.rb +144 -0
  26. data/lib/perobs/IDListPage.rb +107 -0
  27. data/lib/perobs/IDListPageFile.rb +180 -0
  28. data/lib/perobs/IDListPageRecord.rb +142 -0
  29. data/lib/perobs/Object.rb +18 -15
  30. data/lib/perobs/ObjectBase.rb +46 -5
  31. data/lib/perobs/PersistentObjectCache.rb +57 -68
  32. data/lib/perobs/PersistentObjectCacheLine.rb +24 -12
  33. data/lib/perobs/ProgressMeter.rb +97 -0
  34. data/lib/perobs/SpaceManager.rb +273 -0
  35. data/lib/perobs/SpaceTree.rb +21 -12
  36. data/lib/perobs/SpaceTreeNode.rb +53 -61
  37. data/lib/perobs/Store.rb +264 -145
  38. data/lib/perobs/version.rb +1 -1
  39. data/lib/perobs.rb +2 -0
  40. data/perobs.gemspec +4 -4
  41. data/test/Array_spec.rb +15 -6
  42. data/test/BTree_spec.rb +6 -2
  43. data/test/BigArray_spec.rb +261 -0
  44. data/test/BigHash_spec.rb +152 -0
  45. data/test/BigTreeNode_spec.rb +153 -0
  46. data/test/BigTree_spec.rb +259 -0
  47. data/test/EquiBlobsFile_spec.rb +105 -1
  48. data/test/FNV_Hash_1a_64_spec.rb +59 -0
  49. data/test/FlatFileDB_spec.rb +198 -14
  50. data/test/FuzzyStringMatcher_spec.rb +261 -0
  51. data/test/Hash_spec.rb +13 -3
  52. data/test/IDList_spec.rb +77 -0
  53. data/test/LegacyDBs/LegacyDB.rb +155 -0
  54. data/test/LegacyDBs/version_3/class_map.json +1 -0
  55. data/test/LegacyDBs/version_3/config.json +1 -0
  56. data/test/LegacyDBs/version_3/database.blobs +0 -0
  57. data/test/LegacyDBs/version_3/database_spaces.blobs +0 -0
  58. data/test/LegacyDBs/version_3/index.blobs +0 -0
  59. data/test/LegacyDBs/version_3/version +1 -0
  60. data/test/LockFile_spec.rb +9 -6
  61. data/test/SpaceManager_spec.rb +176 -0
  62. data/test/SpaceTree_spec.rb +4 -1
  63. data/test/Store_spec.rb +305 -203
  64. data/test/spec_helper.rb +9 -4
  65. metadata +57 -16
  66. data/lib/perobs/BTreeNodeCache.rb +0 -109
  67. data/lib/perobs/TreeDB.rb +0 -277
@@ -0,0 +1,176 @@
1
+ # encoding: UTF-8
2
+ #
3
+ # Copyright (c) 2020 by Chris Schlaeger <chris@taskjuggler.org>
4
+ #
5
+ # MIT License
6
+ #
7
+ # Permission is hereby granted, free of charge, to any person obtaining
8
+ # a copy of this software and associated documentation files (the
9
+ # "Software"), to deal in the Software without restriction, including
10
+ # without limitation the rights to use, copy, modify, merge, publish,
11
+ # distribute, sublicense, and/or sell copies of the Software, and to
12
+ # permit persons to whom the Software is furnished to do so, subject to
13
+ # the following conditions:
14
+ #
15
+ # The above copyright notice and this permission notice shall be
16
+ # included in all copies or substantial portions of the Software.
17
+ #
18
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
19
+ # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
20
+ # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
21
+ # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
22
+ # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
23
+ # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
24
+ # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
25
+
26
+ require 'fileutils'
27
+
28
+ require 'spec_helper'
29
+ require 'perobs/SpaceManager'
30
+ require 'perobs/ProgressMeter'
31
+
32
+ describe PEROBS::SpaceManager do
33
+
34
+ before(:all) do
35
+ @db_dir = generate_db_name('SpaceManager')
36
+ FileUtils.mkdir_p(@db_dir)
37
+ @m = PEROBS::SpaceManager.new(@db_dir, PEROBS::ProgressMeter.new, 5)
38
+ PEROBS.log.level = Logger::ERROR
39
+ PEROBS.log.open($stderr)
40
+ end
41
+
42
+ after(:all) do
43
+ FileUtils.rm_rf(@db_dir)
44
+ end
45
+
46
+ it 'should open the space manager' do
47
+ @m.open
48
+ expect(@m.to_a).to eql([])
49
+ end
50
+
51
+ it 'should support adding spaces' do
52
+ @m.add_space(80, 8)
53
+ expect(@m.has_space?(80, 8)).to be true
54
+ expect(@m.to_a).to eql([[80, 8]])
55
+ expect(@m.check).to be true
56
+ @m.add_space(40, 4)
57
+ expect(@m.has_space?(40, 4)).to be true
58
+ expect(@m.to_a).to eql([[40, 4], [80, 8]])
59
+ @m.add_space(20, 2)
60
+ expect(@m.has_space?(20, 2)).to be true
61
+ expect(@m.to_a).to eql([[20, 2], [40, 4], [80, 8]])
62
+ @m.add_space(160, 16)
63
+ expect(@m.has_space?(160, 16)).to be true
64
+ expect(@m.to_a).to eql([[20, 2], [40, 4], [80, 8], [160, 16]])
65
+ @m.add_space(320, 32)
66
+ expect(@m.has_space?(320, 32)).to be true
67
+ expect(@m.to_a).to eql([[20, 2], [40, 4], [80, 8], [160, 16], [320, 32]])
68
+ @m.add_space(100, 10)
69
+ expect(@m.has_space?(100, 10)).to be true
70
+ expect(@m.to_a).to eql([[20, 2], [40, 4], [80, 8], [100, 10], [160, 16], [320, 32]])
71
+ @m.add_space(81, 8)
72
+ expect(@m.has_space?(81, 8)).to be true
73
+ expect(@m.to_a).to eql([[20, 2], [40, 4], [80, 8], [81, 8], [100, 10], [160, 16], [320, 32]])
74
+ expect(@m.check).to be true
75
+ end
76
+
77
+ it 'should keep values over an close/open' do
78
+ @m.add_space(1, 15)
79
+ expect(@m.check).to be true
80
+ @m.close
81
+ @m.open
82
+ expect(@m.check).to be true
83
+ expect(@m.to_a).to eql([[1, 15], [20, 2], [40, 4], [80, 8], [81, 8], [100, 10], [160, 16], [320, 32]])
84
+ end
85
+
86
+ it 'should support clearing all spaces' do
87
+ @m.clear
88
+ expect(@m.to_a).to eql([])
89
+ @m.add_space(1, 1)
90
+ @m.add_space(2, 2)
91
+ @m.clear
92
+ expect(@m.to_a).to eql([])
93
+ end
94
+
95
+ it 'should return exactly matching spaces' do
96
+ @m.clear
97
+ add_sizes([ 10, 5, 15, 10 ])
98
+ expect(@m.to_a).to eql([[0, 10], [1, 5], [2, 15], [3, 10]])
99
+ expect(@m.get_space(10)).to eql([3, 10])
100
+ expect(@m.get_space(10)).to eql([0, 10])
101
+ expect(@m.to_a).to eql([[1, 5], [2, 15]])
102
+ expect(@m.added_spaces).to eql(4)
103
+ expect(@m.recycled_spaces).to eql(2)
104
+ expect(@m.failed_requests).to eql(0)
105
+
106
+ @m.clear
107
+ add_sizes([ 10, 5, 15, 10, 10 ])
108
+ expect(@m.to_a).to eql([[0, 10], [1, 5], [2, 15], [3, 10], [4, 10]])
109
+ expect(@m.get_space(10)).to eql([4, 10])
110
+ expect(@m.get_space(10)).to eql([3, 10])
111
+ expect(@m.get_space(10)).to eql([0, 10])
112
+ expect(@m.to_a).to eql([[1, 5], [2, 15]])
113
+ expect(@m.added_spaces).to eql(5)
114
+ expect(@m.recycled_spaces).to eql(3)
115
+ expect(@m.failed_requests).to eql(0)
116
+ end
117
+
118
+ it "should return nil if no space can be found" do
119
+ expect(@m.get_space(42)).to be nil
120
+ expect(@m.get_space(9)).to be nil
121
+ expect(@m.get_space(11)).to be nil
122
+ expect(@m.recycled_spaces).to eql(3)
123
+ expect(@m.failed_requests).to eql(3)
124
+ end
125
+
126
+ it 'should support a real-world traffic pattern' do
127
+ address = 0
128
+ spaces = []
129
+ @m.clear
130
+ 0.upto(1500) do
131
+ case rand(4)
132
+ when 0
133
+ # Insert new values
134
+ rand(9).times do
135
+ size = 20 + rand(80)
136
+ @m.add_space(address, size)
137
+ spaces << [ address, size ]
138
+ address += size
139
+ end
140
+ when 1
141
+ # Remove some values
142
+ rand(7).times do
143
+ size = rand(110)
144
+ if (space = @m.get_space(size))
145
+ expect(spaces.include?(space)).to be true
146
+ spaces.delete(space)
147
+ end
148
+ end
149
+ when 2
150
+ if rand(10) == 0
151
+ expect(@m.check).to be true
152
+ spaces.each do |address, size|
153
+ expect(@m.has_space?(address, size)).to be true
154
+ end
155
+ @m.to_a.each do |address, size|
156
+ expect(spaces.include?([ address, size ])).to be true
157
+ end
158
+ end
159
+ when 3
160
+ if rand(200) == 0
161
+ expect(@m.check).to be true
162
+ @m.close
163
+ @m.open
164
+ expect(@m.check).to be true
165
+ end
166
+ end
167
+ end
168
+ end
169
+
170
+ def add_sizes(sizes)
171
+ sizes.each_with_index do |size, i|
172
+ @m.add_space(i, size)
173
+ end
174
+ end
175
+
176
+ end
@@ -27,13 +27,16 @@ require 'fileutils'
27
27
 
28
28
  require 'spec_helper'
29
29
  require 'perobs/SpaceTree'
30
+ require 'perobs/ProgressMeter'
30
31
 
31
32
  describe PEROBS::SpaceTree do
32
33
 
33
34
  before(:all) do
34
35
  @db_dir = generate_db_name('SpaceTree')
35
36
  FileUtils.mkdir_p(@db_dir)
36
- @m = PEROBS::SpaceTree.new(@db_dir)
37
+ @m = PEROBS::SpaceTree.new(@db_dir, PEROBS::ProgressMeter.new)
38
+ PEROBS.log.level = Logger::ERROR
39
+ PEROBS.log.open($stderr)
37
40
  end
38
41
 
39
42
  after(:all) do