perobs 4.0.0 → 4.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. checksums.yaml +4 -4
  2. data/lib/perobs.rb +1 -0
  3. data/lib/perobs/Array.rb +66 -19
  4. data/lib/perobs/BTree.rb +83 -12
  5. data/lib/perobs/BTreeBlob.rb +1 -1
  6. data/lib/perobs/BTreeDB.rb +2 -2
  7. data/lib/perobs/BTreeNode.rb +365 -85
  8. data/lib/perobs/BigArray.rb +267 -0
  9. data/lib/perobs/BigArrayNode.rb +998 -0
  10. data/lib/perobs/BigHash.rb +262 -0
  11. data/lib/perobs/BigTree.rb +184 -0
  12. data/lib/perobs/BigTreeNode.rb +873 -0
  13. data/lib/perobs/ConsoleProgressMeter.rb +61 -0
  14. data/lib/perobs/DataBase.rb +4 -3
  15. data/lib/perobs/DynamoDB.rb +57 -15
  16. data/lib/perobs/EquiBlobsFile.rb +143 -51
  17. data/lib/perobs/FNV_Hash_1a_64.rb +54 -0
  18. data/lib/perobs/FlatFile.rb +363 -203
  19. data/lib/perobs/FlatFileBlobHeader.rb +98 -54
  20. data/lib/perobs/FlatFileDB.rb +42 -20
  21. data/lib/perobs/Hash.rb +58 -13
  22. data/lib/perobs/IDList.rb +144 -0
  23. data/lib/perobs/IDListPage.rb +107 -0
  24. data/lib/perobs/IDListPageFile.rb +180 -0
  25. data/lib/perobs/IDListPageRecord.rb +142 -0
  26. data/lib/perobs/Object.rb +18 -15
  27. data/lib/perobs/ObjectBase.rb +38 -4
  28. data/lib/perobs/PersistentObjectCache.rb +53 -67
  29. data/lib/perobs/PersistentObjectCacheLine.rb +24 -12
  30. data/lib/perobs/ProgressMeter.rb +97 -0
  31. data/lib/perobs/SpaceTree.rb +21 -12
  32. data/lib/perobs/SpaceTreeNode.rb +53 -61
  33. data/lib/perobs/Store.rb +71 -32
  34. data/lib/perobs/version.rb +1 -1
  35. data/perobs.gemspec +4 -4
  36. data/test/Array_spec.rb +15 -6
  37. data/test/BTree_spec.rb +5 -2
  38. data/test/BigArray_spec.rb +214 -0
  39. data/test/BigHash_spec.rb +144 -0
  40. data/test/BigTreeNode_spec.rb +153 -0
  41. data/test/BigTree_spec.rb +259 -0
  42. data/test/EquiBlobsFile_spec.rb +105 -1
  43. data/test/FNV_Hash_1a_64_spec.rb +59 -0
  44. data/test/FlatFileDB_spec.rb +63 -14
  45. data/test/Hash_spec.rb +1 -2
  46. data/test/IDList_spec.rb +77 -0
  47. data/test/LegacyDBs/LegacyDB.rb +151 -0
  48. data/test/LegacyDBs/version_3/class_map.json +1 -0
  49. data/test/LegacyDBs/version_3/config.json +1 -0
  50. data/test/LegacyDBs/version_3/database.blobs +0 -0
  51. data/test/LegacyDBs/version_3/database_spaces.blobs +0 -0
  52. data/test/LegacyDBs/version_3/index.blobs +0 -0
  53. data/test/LegacyDBs/version_3/version +1 -0
  54. data/test/LockFile_spec.rb +9 -6
  55. data/test/SpaceTree_spec.rb +4 -1
  56. data/test/Store_spec.rb +290 -199
  57. data/test/spec_helper.rb +9 -4
  58. metadata +47 -10
  59. data/lib/perobs/TreeDB.rb +0 -277
@@ -41,15 +41,20 @@ def generate_db_name(caller_file)
41
41
  end
42
42
 
43
43
  def capture_io
44
- PEROBS.log.open(io = StringIO.new)
44
+ old_stdout = $stdout
45
+ $stdout = out = StringIO.new
46
+ PEROBS.log.open(log = StringIO.new)
47
+
45
48
  begin
46
49
  yield
47
50
  ensure
51
+ $stdout = old_stdout
48
52
  PEROBS.log.open($stderr)
49
53
  end
50
54
 
51
- io.rewind
52
- io.read
53
- end
55
+ out.rewind
56
+ log.rewind
54
57
 
58
+ Struct.new(:out, :log).new(out.read, log.read)
59
+ end
55
60
 
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: perobs
3
3
  version: !ruby/object:Gem::Version
4
- version: 4.0.0
4
+ version: 4.1.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Chris Schlaeger
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2017-11-05 00:00:00.000000000 Z
11
+ date: 2019-06-08 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -16,42 +16,42 @@ dependencies:
16
16
  requirements:
17
17
  - - "~>"
18
18
  - !ruby/object:Gem::Version
19
- version: '1.6'
19
+ version: '2.3'
20
20
  type: :development
21
21
  prerelease: false
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
24
  - - "~>"
25
25
  - !ruby/object:Gem::Version
26
- version: '1.6'
26
+ version: '2.3'
27
27
  - !ruby/object:Gem::Dependency
28
28
  name: yard
29
29
  requirement: !ruby/object:Gem::Requirement
30
30
  requirements:
31
31
  - - "~>"
32
32
  - !ruby/object:Gem::Version
33
- version: 0.8.7
33
+ version: 0.9.12
34
34
  type: :development
35
35
  prerelease: false
36
36
  version_requirements: !ruby/object:Gem::Requirement
37
37
  requirements:
38
38
  - - "~>"
39
39
  - !ruby/object:Gem::Version
40
- version: 0.8.7
40
+ version: 0.9.12
41
41
  - !ruby/object:Gem::Dependency
42
42
  name: rake
43
43
  requirement: !ruby/object:Gem::Requirement
44
44
  requirements:
45
45
  - - "~>"
46
46
  - !ruby/object:Gem::Version
47
- version: '10.0'
47
+ version: '10.1'
48
48
  type: :development
49
49
  prerelease: false
50
50
  version_requirements: !ruby/object:Gem::Requirement
51
51
  requirements:
52
52
  - - "~>"
53
53
  - !ruby/object:Gem::Version
54
- version: '10.0'
54
+ version: '10.1'
55
55
  description: Library to provide a persistent object store
56
56
  email:
57
57
  - chris@linux.com
@@ -72,29 +72,40 @@ files:
72
72
  - lib/perobs/BTreeNode.rb
73
73
  - lib/perobs/BTreeNodeCache.rb
74
74
  - lib/perobs/BTreeNodeLink.rb
75
+ - lib/perobs/BigArray.rb
76
+ - lib/perobs/BigArrayNode.rb
77
+ - lib/perobs/BigHash.rb
78
+ - lib/perobs/BigTree.rb
79
+ - lib/perobs/BigTreeNode.rb
75
80
  - lib/perobs/Cache.rb
76
81
  - lib/perobs/ClassMap.rb
82
+ - lib/perobs/ConsoleProgressMeter.rb
77
83
  - lib/perobs/DataBase.rb
78
84
  - lib/perobs/DynamoDB.rb
79
85
  - lib/perobs/EquiBlobsFile.rb
86
+ - lib/perobs/FNV_Hash_1a_64.rb
80
87
  - lib/perobs/FlatFile.rb
81
88
  - lib/perobs/FlatFileBlobHeader.rb
82
89
  - lib/perobs/FlatFileDB.rb
83
90
  - lib/perobs/Handle.rb
84
91
  - lib/perobs/Hash.rb
92
+ - lib/perobs/IDList.rb
93
+ - lib/perobs/IDListPage.rb
94
+ - lib/perobs/IDListPageFile.rb
95
+ - lib/perobs/IDListPageRecord.rb
85
96
  - lib/perobs/LockFile.rb
86
97
  - lib/perobs/Log.rb
87
98
  - lib/perobs/Object.rb
88
99
  - lib/perobs/ObjectBase.rb
89
100
  - lib/perobs/PersistentObjectCache.rb
90
101
  - lib/perobs/PersistentObjectCacheLine.rb
102
+ - lib/perobs/ProgressMeter.rb
91
103
  - lib/perobs/RobustFile.rb
92
104
  - lib/perobs/SpaceTree.rb
93
105
  - lib/perobs/SpaceTreeNode.rb
94
106
  - lib/perobs/SpaceTreeNodeLink.rb
95
107
  - lib/perobs/StackFile.rb
96
108
  - lib/perobs/Store.rb
97
- - lib/perobs/TreeDB.rb
98
109
  - lib/perobs/version.rb
99
110
  - perobs.gemspec
100
111
  - tasks/changelog.rake
@@ -104,10 +115,23 @@ files:
104
115
  - test/Array_spec.rb
105
116
  - test/BTreeDB_spec.rb
106
117
  - test/BTree_spec.rb
118
+ - test/BigArray_spec.rb
119
+ - test/BigHash_spec.rb
120
+ - test/BigTreeNode_spec.rb
121
+ - test/BigTree_spec.rb
107
122
  - test/ClassMap_spec.rb
108
123
  - test/EquiBlobsFile_spec.rb
124
+ - test/FNV_Hash_1a_64_spec.rb
109
125
  - test/FlatFileDB_spec.rb
110
126
  - test/Hash_spec.rb
127
+ - test/IDList_spec.rb
128
+ - test/LegacyDBs/LegacyDB.rb
129
+ - test/LegacyDBs/version_3/class_map.json
130
+ - test/LegacyDBs/version_3/config.json
131
+ - test/LegacyDBs/version_3/database.blobs
132
+ - test/LegacyDBs/version_3/database_spaces.blobs
133
+ - test/LegacyDBs/version_3/index.blobs
134
+ - test/LegacyDBs/version_3/version
111
135
  - test/LockFile_spec.rb
112
136
  - test/Object_spec.rb
113
137
  - test/SpaceTree_spec.rb
@@ -127,7 +151,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
127
151
  requirements:
128
152
  - - ">="
129
153
  - !ruby/object:Gem::Version
130
- version: '2.0'
154
+ version: '2.3'
131
155
  required_rubygems_version: !ruby/object:Gem::Requirement
132
156
  requirements:
133
157
  - - ">="
@@ -143,10 +167,23 @@ test_files:
143
167
  - test/Array_spec.rb
144
168
  - test/BTreeDB_spec.rb
145
169
  - test/BTree_spec.rb
170
+ - test/BigArray_spec.rb
171
+ - test/BigHash_spec.rb
172
+ - test/BigTreeNode_spec.rb
173
+ - test/BigTree_spec.rb
146
174
  - test/ClassMap_spec.rb
147
175
  - test/EquiBlobsFile_spec.rb
176
+ - test/FNV_Hash_1a_64_spec.rb
148
177
  - test/FlatFileDB_spec.rb
149
178
  - test/Hash_spec.rb
179
+ - test/IDList_spec.rb
180
+ - test/LegacyDBs/LegacyDB.rb
181
+ - test/LegacyDBs/version_3/class_map.json
182
+ - test/LegacyDBs/version_3/config.json
183
+ - test/LegacyDBs/version_3/database.blobs
184
+ - test/LegacyDBs/version_3/database_spaces.blobs
185
+ - test/LegacyDBs/version_3/index.blobs
186
+ - test/LegacyDBs/version_3/version
150
187
  - test/LockFile_spec.rb
151
188
  - test/Object_spec.rb
152
189
  - test/SpaceTree_spec.rb
@@ -1,277 +0,0 @@
1
- # encoding: UTF-8
2
- #
3
- # = BTreeDB.rb -- Persistent Ruby Object Store
4
- #
5
- # Copyright (c) 2015, 2016 by Chris Schlaeger <chris@taskjuggler.org>
6
- #
7
- # MIT License
8
- #
9
- # Permission is hereby granted, free of charge, to any person obtaining
10
- # a copy of this software and associated documentation files (the
11
- # "Software"), to deal in the Software without restriction, including
12
- # without limitation the rights to use, copy, modify, merge, publish,
13
- # distribute, sublicense, and/or sell copies of the Software, and to
14
- # permit persons to whom the Software is furnished to do so, subject to
15
- # the following conditions:
16
- #
17
- # The above copyright notice and this permission notice shall be
18
- # included in all copies or substantial portions of the Software.
19
- #
20
- # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
21
- # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
22
- # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
23
- # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
24
- # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
25
- # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
26
- # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
27
-
28
- require 'fileutils'
29
-
30
- require 'perobs/Log'
31
- require 'perobs/RobustFile'
32
- require 'perobs/DataBase'
33
- require 'perobs/BTreeBlob'
34
-
35
- module PEROBS
36
-
37
- # This class implements a BTree database using filesystem directories as
38
- # nodes and blob files as leafs. The BTree grows with the number of stored
39
- # entries. Each leaf node blob can hold a fixed number of entries. If more
40
- # entries need to be stored, the blob is replaced by a node with multiple
41
- # new leafs that store the entries of the previous node. The leafs are
42
- # implemented by the BTreeBlob class.
43
- class BTreeDB < DataBase
44
-
45
- attr_reader :max_blob_size
46
-
47
- # Create a new BTreeDB object.
48
- # @param db_name [String] name of the DB directory
49
- # @param options [Hash] options to customize the behavior. Currently only
50
- # the following options are supported:
51
- # :serializer : Can be :marshal, :json, :yaml
52
- # :dir_bits : The number of bits to use for the BTree nodes.
53
- # The value must be between 4 and 14. The larger
54
- # the number the more back-end directories are
55
- # being used. The default is 12 which results in
56
- # 4096 directories per node.
57
- # :max_blob_size : The maximum number of entries in the BTree leaf
58
- # nodes. The insert/find/delete time grows
59
- # linearly with the size.
60
- def initialize(db_name, options = {})
61
- super(options[:serializer] || :json)
62
-
63
- @db_dir = db_name
64
- # Create the database directory if it doesn't exist yet.
65
- ensure_dir_exists(@db_dir)
66
-
67
- # Read the existing DB config.
68
- @config = get_hash('config')
69
- check_option('serializer')
70
-
71
- # Check and set @dir_bits, the number of bits used for each tree level.
72
- @dir_bits = options[:dir_bits] || 12
73
- if @dir_bits < 4 || @dir_bits > 14
74
- PEROBS.log.fatal "dir_bits option (#{@dir_bits}) must be between 4 " +
75
- "and 12"
76
- end
77
- check_option('dir_bits')
78
-
79
- @max_blob_size = options[:max_blob_size] || 32
80
- if @max_blob_size < 4 || @max_blob_size > 128
81
- PEROBS.log.fatal "max_blob_size option (#{@max_blob_size}) must be " +
82
- "between 4 and 128"
83
- end
84
- check_option('max_blob_size')
85
-
86
- put_hash('config', @config)
87
-
88
- # This format string is used to create the directory name.
89
- @dir_format_string = "%0#{(@dir_bits / 4) +
90
- (@dir_bits % 4 == 0 ? 0 : 1)}X"
91
- # Bit mask to extract the dir_bits LSBs.
92
- @dir_mask = 2 ** @dir_bits - 1
93
- end
94
-
95
- # Delete the entire database. The database is no longer usable after this
96
- # method was called.
97
- def delete_database
98
- FileUtils.rm_rf(@db_dir)
99
- end
100
-
101
- def BTreeDB::delete_db(db_name)
102
- FileUtils.rm_rf(db_name)
103
- end
104
-
105
- # Return true if the object with given ID exists
106
- # @param id [Integer]
107
- def include?(id)
108
- !(blob = find_blob(id)).nil? && !blob.find(id).nil?
109
- end
110
-
111
- # Store a simple Hash as a JSON encoded file into the DB directory.
112
- # @param name [String] Name of the hash. Will be used as file name.
113
- # @param hash [Hash] A Hash that maps String objects to strings or
114
- # numbers.
115
- def put_hash(name, hash)
116
- file_name = File.join(@db_dir, name + '.json')
117
- begin
118
- RobustFile.write(file_name, hash.to_json)
119
- rescue IOError => e
120
- PEROBS.log.fatal "Cannot write hash file '#{file_name}': #{e.message}"
121
- end
122
- end
123
-
124
- # Load the Hash with the given name.
125
- # @param name [String] Name of the hash.
126
- # @return [Hash] A Hash that maps String objects to strings or numbers.
127
- def get_hash(name)
128
- file_name = File.join(@db_dir, name + '.json')
129
- return ::Hash.new unless File.exist?(file_name)
130
-
131
- begin
132
- json = File.read(file_name)
133
- rescue => e
134
- PEROBS.log.fatal "Cannot read hash file '#{file_name}': #{e.message}"
135
- end
136
- JSON.parse(json, :create_additions => true)
137
- end
138
-
139
- # Store the given object into the cluster files.
140
- # @param obj [Hash] Object as defined by PEROBS::ObjectBase
141
- def put_object(obj, id)
142
- find_blob(id, true).write_object(id, serialize(obj))
143
- end
144
-
145
- # Load the given object from the filesystem.
146
- # @param id [Integer] object ID
147
- # @return [Hash] Object as defined by PEROBS::ObjectBase or nil if ID does
148
- # not exist
149
- def get_object(id)
150
- return nil unless (blob = find_blob(id)) && (obj = blob.read_object(id))
151
- deserialize(obj)
152
- end
153
-
154
- # This method must be called to initiate the marking process.
155
- def clear_marks
156
- each_blob { |blob| blob.clear_marks }
157
- end
158
-
159
- # Permanently delete all objects that have not been marked. Those are
160
- # orphaned and are no longer referenced by any actively used object.
161
- # @return [Array] List of IDs that have been removed from the DB.
162
- def delete_unmarked_objects
163
- deleted_ids = []
164
- each_blob { |blob| deleted_ids += blob.delete_unmarked_entries }
165
- deleted_ids
166
- end
167
-
168
- # Mark an object.
169
- # @param id [Integer] ID of the object to mark
170
- def mark(id)
171
- (blob = find_blob(id)) && blob.mark(id)
172
- end
173
-
174
- # Check if the object is marked.
175
- # @param id [Integer] ID of the object to check
176
- # @param ignore_errors [Boolean] If set to true no errors will be raised
177
- # for non-existing objects.
178
- def is_marked?(id, ignore_errors = false)
179
- (blob = find_blob(id)) && blob.is_marked?(id, ignore_errors)
180
- end
181
-
182
- # Basic consistency check.
183
- # @param repair [TrueClass/FalseClass] True if found errors should be
184
- # repaired.
185
- def check_db(repair = false)
186
- each_blob { |blob| blob.check(repair) }
187
- end
188
-
189
- # Check if the stored object is syntactically correct.
190
- # @param id [Integer] Object ID
191
- # @param repair [TrueClass/FalseClass] True if an repair attempt should be
192
- # made.
193
- # @return [TrueClass/FalseClass] True if the object is OK, otherwise
194
- # false.
195
- def check(id, repair)
196
- begin
197
- get_object(id)
198
- rescue => e
199
- PEROBS.log.error "Cannot read object with ID #{id}: #{e.message}"
200
- return false
201
- end
202
-
203
- true
204
- end
205
-
206
- # Store the given serialized object into the cluster files. This method is
207
- # for internal use only!
208
- # @param raw [String] Serialized Object as defined by PEROBS::ObjectBase
209
- # @param id [Integer] Object ID
210
- def put_raw_object(raw, id)
211
- find_blob(id, true).write_object(id, raw)
212
- end
213
-
214
- private
215
-
216
- def find_blob(id, create_missing_blob = false, dir_name = @db_dir)
217
- dir_bits = id & @dir_mask
218
- sub_dir_name = File.join(dir_name, @dir_format_string % dir_bits)
219
-
220
- if Dir.exist?(sub_dir_name)
221
- if File.exist?(File.join(sub_dir_name, 'index'))
222
- # The directory is a blob directory and not a BTree node dir.
223
- return BTreeBlob.new(sub_dir_name, self)
224
- end
225
- else
226
- Dir.glob(File.join(dir_name, '*.index')).each do |fqfn|
227
- # Extract the 01-part of the filename
228
- lsb_string = File.basename(fqfn)[0..-6]
229
- # Convert the lsb_string into a Integer
230
- lsb = Integer('0b' + lsb_string)
231
- # Bit mask to match the LSBs
232
- mask = (2 ** lsb_string.length) - 1
233
- if (id & mask) == lsb
234
- return TreeBlob.new(sub_dir_name, lsb_string, self)
235
- end
236
- end
237
- if create_missing_blob
238
- # Create the new blob directory.
239
- Dir.mkdir(dir_name)
240
- # And initialize the blob DB.
241
- return BTreeBlob.new(dir_name, self)
242
- else
243
- return nil
244
- end
245
- end
246
-
247
- # Discard the least significant @dir_bits bits and start over again
248
- # with the directory that matches the @dir_bits LSBs of the new ID.
249
- id = id >> @dir_bits
250
- end
251
-
252
- def each_blob(&block)
253
- each_blob_r(@db_dir, &block)
254
- end
255
-
256
- def each_blob_r(dir, &block)
257
- Dir.glob(File.join(dir, '*')) do |dir_name|
258
- if is_blob_dir?(dir_name)
259
- block.call(BTreeBlob.new(dir_name, self))
260
- else
261
- each_blob_r(dir_name, &block)
262
- end
263
- end
264
- end
265
-
266
- def is_blob_dir?(dir_name)
267
- # A blob directory contains an 'index' and 'data' file. This is in
268
- # contrast to BTree node directories that only contain other
269
- # directories.
270
- index_file = File.join(dir_name, 'index')
271
- File.exist?(index_file)
272
- end
273
-
274
- end
275
-
276
- end
277
-