perobs 4.0.0 → 4.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. checksums.yaml +5 -5
  2. data/README.md +27 -16
  3. data/lib/perobs/Array.rb +66 -19
  4. data/lib/perobs/BTree.rb +106 -15
  5. data/lib/perobs/BTreeBlob.rb +4 -3
  6. data/lib/perobs/BTreeDB.rb +5 -4
  7. data/lib/perobs/BTreeNode.rb +482 -156
  8. data/lib/perobs/BTreeNodeLink.rb +10 -0
  9. data/lib/perobs/BigArray.rb +285 -0
  10. data/lib/perobs/BigArrayNode.rb +1002 -0
  11. data/lib/perobs/BigHash.rb +246 -0
  12. data/lib/perobs/BigTree.rb +197 -0
  13. data/lib/perobs/BigTreeNode.rb +873 -0
  14. data/lib/perobs/Cache.rb +48 -10
  15. data/lib/perobs/ConsoleProgressMeter.rb +61 -0
  16. data/lib/perobs/DataBase.rb +4 -3
  17. data/lib/perobs/DynamoDB.rb +57 -15
  18. data/lib/perobs/EquiBlobsFile.rb +155 -50
  19. data/lib/perobs/FNV_Hash_1a_64.rb +54 -0
  20. data/lib/perobs/FlatFile.rb +519 -227
  21. data/lib/perobs/FlatFileBlobHeader.rb +113 -54
  22. data/lib/perobs/FlatFileDB.rb +49 -23
  23. data/lib/perobs/FuzzyStringMatcher.rb +175 -0
  24. data/lib/perobs/Hash.rb +127 -33
  25. data/lib/perobs/IDList.rb +144 -0
  26. data/lib/perobs/IDListPage.rb +107 -0
  27. data/lib/perobs/IDListPageFile.rb +180 -0
  28. data/lib/perobs/IDListPageRecord.rb +142 -0
  29. data/lib/perobs/Object.rb +18 -15
  30. data/lib/perobs/ObjectBase.rb +46 -5
  31. data/lib/perobs/PersistentObjectCache.rb +57 -68
  32. data/lib/perobs/PersistentObjectCacheLine.rb +24 -12
  33. data/lib/perobs/ProgressMeter.rb +97 -0
  34. data/lib/perobs/SpaceManager.rb +273 -0
  35. data/lib/perobs/SpaceTree.rb +21 -12
  36. data/lib/perobs/SpaceTreeNode.rb +53 -61
  37. data/lib/perobs/Store.rb +264 -145
  38. data/lib/perobs/version.rb +1 -1
  39. data/lib/perobs.rb +2 -0
  40. data/perobs.gemspec +4 -4
  41. data/test/Array_spec.rb +15 -6
  42. data/test/BTree_spec.rb +6 -2
  43. data/test/BigArray_spec.rb +261 -0
  44. data/test/BigHash_spec.rb +152 -0
  45. data/test/BigTreeNode_spec.rb +153 -0
  46. data/test/BigTree_spec.rb +259 -0
  47. data/test/EquiBlobsFile_spec.rb +105 -1
  48. data/test/FNV_Hash_1a_64_spec.rb +59 -0
  49. data/test/FlatFileDB_spec.rb +198 -14
  50. data/test/FuzzyStringMatcher_spec.rb +261 -0
  51. data/test/Hash_spec.rb +13 -3
  52. data/test/IDList_spec.rb +77 -0
  53. data/test/LegacyDBs/LegacyDB.rb +155 -0
  54. data/test/LegacyDBs/version_3/class_map.json +1 -0
  55. data/test/LegacyDBs/version_3/config.json +1 -0
  56. data/test/LegacyDBs/version_3/database.blobs +0 -0
  57. data/test/LegacyDBs/version_3/database_spaces.blobs +0 -0
  58. data/test/LegacyDBs/version_3/index.blobs +0 -0
  59. data/test/LegacyDBs/version_3/version +1 -0
  60. data/test/LockFile_spec.rb +9 -6
  61. data/test/SpaceManager_spec.rb +176 -0
  62. data/test/SpaceTree_spec.rb +4 -1
  63. data/test/Store_spec.rb +305 -203
  64. data/test/spec_helper.rb +9 -4
  65. metadata +57 -16
  66. data/lib/perobs/BTreeNodeCache.rb +0 -109
  67. data/lib/perobs/TreeDB.rb +0 -277
@@ -0,0 +1,176 @@
1
+ # encoding: UTF-8
2
+ #
3
+ # Copyright (c) 2020 by Chris Schlaeger <chris@taskjuggler.org>
4
+ #
5
+ # MIT License
6
+ #
7
+ # Permission is hereby granted, free of charge, to any person obtaining
8
+ # a copy of this software and associated documentation files (the
9
+ # "Software"), to deal in the Software without restriction, including
10
+ # without limitation the rights to use, copy, modify, merge, publish,
11
+ # distribute, sublicense, and/or sell copies of the Software, and to
12
+ # permit persons to whom the Software is furnished to do so, subject to
13
+ # the following conditions:
14
+ #
15
+ # The above copyright notice and this permission notice shall be
16
+ # included in all copies or substantial portions of the Software.
17
+ #
18
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
19
+ # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
20
+ # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
21
+ # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
22
+ # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
23
+ # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
24
+ # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
25
+
26
+ require 'fileutils'
27
+
28
+ require 'spec_helper'
29
+ require 'perobs/SpaceManager'
30
+ require 'perobs/ProgressMeter'
31
+
32
+ describe PEROBS::SpaceManager do
33
+
34
+ before(:all) do
35
+ @db_dir = generate_db_name('SpaceManager')
36
+ FileUtils.mkdir_p(@db_dir)
37
+ @m = PEROBS::SpaceManager.new(@db_dir, PEROBS::ProgressMeter.new, 5)
38
+ PEROBS.log.level = Logger::ERROR
39
+ PEROBS.log.open($stderr)
40
+ end
41
+
42
+ after(:all) do
43
+ FileUtils.rm_rf(@db_dir)
44
+ end
45
+
46
+ it 'should open the space manager' do
47
+ @m.open
48
+ expect(@m.to_a).to eql([])
49
+ end
50
+
51
+ it 'should support adding spaces' do
52
+ @m.add_space(80, 8)
53
+ expect(@m.has_space?(80, 8)).to be true
54
+ expect(@m.to_a).to eql([[80, 8]])
55
+ expect(@m.check).to be true
56
+ @m.add_space(40, 4)
57
+ expect(@m.has_space?(40, 4)).to be true
58
+ expect(@m.to_a).to eql([[40, 4], [80, 8]])
59
+ @m.add_space(20, 2)
60
+ expect(@m.has_space?(20, 2)).to be true
61
+ expect(@m.to_a).to eql([[20, 2], [40, 4], [80, 8]])
62
+ @m.add_space(160, 16)
63
+ expect(@m.has_space?(160, 16)).to be true
64
+ expect(@m.to_a).to eql([[20, 2], [40, 4], [80, 8], [160, 16]])
65
+ @m.add_space(320, 32)
66
+ expect(@m.has_space?(320, 32)).to be true
67
+ expect(@m.to_a).to eql([[20, 2], [40, 4], [80, 8], [160, 16], [320, 32]])
68
+ @m.add_space(100, 10)
69
+ expect(@m.has_space?(100, 10)).to be true
70
+ expect(@m.to_a).to eql([[20, 2], [40, 4], [80, 8], [100, 10], [160, 16], [320, 32]])
71
+ @m.add_space(81, 8)
72
+ expect(@m.has_space?(81, 8)).to be true
73
+ expect(@m.to_a).to eql([[20, 2], [40, 4], [80, 8], [81, 8], [100, 10], [160, 16], [320, 32]])
74
+ expect(@m.check).to be true
75
+ end
76
+
77
+ it 'should keep values over an close/open' do
78
+ @m.add_space(1, 15)
79
+ expect(@m.check).to be true
80
+ @m.close
81
+ @m.open
82
+ expect(@m.check).to be true
83
+ expect(@m.to_a).to eql([[1, 15], [20, 2], [40, 4], [80, 8], [81, 8], [100, 10], [160, 16], [320, 32]])
84
+ end
85
+
86
+ it 'should support clearing all spaces' do
87
+ @m.clear
88
+ expect(@m.to_a).to eql([])
89
+ @m.add_space(1, 1)
90
+ @m.add_space(2, 2)
91
+ @m.clear
92
+ expect(@m.to_a).to eql([])
93
+ end
94
+
95
+ it 'should return exactly matching spaces' do
96
+ @m.clear
97
+ add_sizes([ 10, 5, 15, 10 ])
98
+ expect(@m.to_a).to eql([[0, 10], [1, 5], [2, 15], [3, 10]])
99
+ expect(@m.get_space(10)).to eql([3, 10])
100
+ expect(@m.get_space(10)).to eql([0, 10])
101
+ expect(@m.to_a).to eql([[1, 5], [2, 15]])
102
+ expect(@m.added_spaces).to eql(4)
103
+ expect(@m.recycled_spaces).to eql(2)
104
+ expect(@m.failed_requests).to eql(0)
105
+
106
+ @m.clear
107
+ add_sizes([ 10, 5, 15, 10, 10 ])
108
+ expect(@m.to_a).to eql([[0, 10], [1, 5], [2, 15], [3, 10], [4, 10]])
109
+ expect(@m.get_space(10)).to eql([4, 10])
110
+ expect(@m.get_space(10)).to eql([3, 10])
111
+ expect(@m.get_space(10)).to eql([0, 10])
112
+ expect(@m.to_a).to eql([[1, 5], [2, 15]])
113
+ expect(@m.added_spaces).to eql(5)
114
+ expect(@m.recycled_spaces).to eql(3)
115
+ expect(@m.failed_requests).to eql(0)
116
+ end
117
+
118
+ it "should return nil if no space can be found" do
119
+ expect(@m.get_space(42)).to be nil
120
+ expect(@m.get_space(9)).to be nil
121
+ expect(@m.get_space(11)).to be nil
122
+ expect(@m.recycled_spaces).to eql(3)
123
+ expect(@m.failed_requests).to eql(3)
124
+ end
125
+
126
+ it 'should support a real-world traffic pattern' do
127
+ address = 0
128
+ spaces = []
129
+ @m.clear
130
+ 0.upto(1500) do
131
+ case rand(4)
132
+ when 0
133
+ # Insert new values
134
+ rand(9).times do
135
+ size = 20 + rand(80)
136
+ @m.add_space(address, size)
137
+ spaces << [ address, size ]
138
+ address += size
139
+ end
140
+ when 1
141
+ # Remove some values
142
+ rand(7).times do
143
+ size = rand(110)
144
+ if (space = @m.get_space(size))
145
+ expect(spaces.include?(space)).to be true
146
+ spaces.delete(space)
147
+ end
148
+ end
149
+ when 2
150
+ if rand(10) == 0
151
+ expect(@m.check).to be true
152
+ spaces.each do |address, size|
153
+ expect(@m.has_space?(address, size)).to be true
154
+ end
155
+ @m.to_a.each do |address, size|
156
+ expect(spaces.include?([ address, size ])).to be true
157
+ end
158
+ end
159
+ when 3
160
+ if rand(200) == 0
161
+ expect(@m.check).to be true
162
+ @m.close
163
+ @m.open
164
+ expect(@m.check).to be true
165
+ end
166
+ end
167
+ end
168
+ end
169
+
170
+ def add_sizes(sizes)
171
+ sizes.each_with_index do |size, i|
172
+ @m.add_space(i, size)
173
+ end
174
+ end
175
+
176
+ end
@@ -27,13 +27,16 @@ require 'fileutils'
27
27
 
28
28
  require 'spec_helper'
29
29
  require 'perobs/SpaceTree'
30
+ require 'perobs/ProgressMeter'
30
31
 
31
32
  describe PEROBS::SpaceTree do
32
33
 
33
34
  before(:all) do
34
35
  @db_dir = generate_db_name('SpaceTree')
35
36
  FileUtils.mkdir_p(@db_dir)
36
- @m = PEROBS::SpaceTree.new(@db_dir)
37
+ @m = PEROBS::SpaceTree.new(@db_dir, PEROBS::ProgressMeter.new)
38
+ PEROBS.log.level = Logger::ERROR
39
+ PEROBS.log.open($stderr)
37
40
  end
38
41
 
39
42
  after(:all) do