perobs 3.0.1 → 4.3.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (75) hide show
  1. checksums.yaml +5 -5
  2. data/README.md +19 -18
  3. data/lib/perobs.rb +2 -0
  4. data/lib/perobs/Array.rb +68 -21
  5. data/lib/perobs/BTree.rb +110 -54
  6. data/lib/perobs/BTreeBlob.rb +14 -13
  7. data/lib/perobs/BTreeDB.rb +11 -10
  8. data/lib/perobs/BTreeNode.rb +551 -197
  9. data/lib/perobs/BTreeNodeCache.rb +10 -8
  10. data/lib/perobs/BTreeNodeLink.rb +11 -1
  11. data/lib/perobs/BigArray.rb +285 -0
  12. data/lib/perobs/BigArrayNode.rb +1002 -0
  13. data/lib/perobs/BigHash.rb +246 -0
  14. data/lib/perobs/BigTree.rb +197 -0
  15. data/lib/perobs/BigTreeNode.rb +873 -0
  16. data/lib/perobs/Cache.rb +47 -22
  17. data/lib/perobs/ClassMap.rb +2 -2
  18. data/lib/perobs/ConsoleProgressMeter.rb +61 -0
  19. data/lib/perobs/DataBase.rb +4 -3
  20. data/lib/perobs/DynamoDB.rb +62 -20
  21. data/lib/perobs/EquiBlobsFile.rb +174 -59
  22. data/lib/perobs/FNV_Hash_1a_64.rb +54 -0
  23. data/lib/perobs/FlatFile.rb +536 -242
  24. data/lib/perobs/FlatFileBlobHeader.rb +120 -84
  25. data/lib/perobs/FlatFileDB.rb +58 -27
  26. data/lib/perobs/FuzzyStringMatcher.rb +175 -0
  27. data/lib/perobs/Hash.rb +129 -35
  28. data/lib/perobs/IDList.rb +144 -0
  29. data/lib/perobs/IDListPage.rb +107 -0
  30. data/lib/perobs/IDListPageFile.rb +180 -0
  31. data/lib/perobs/IDListPageRecord.rb +142 -0
  32. data/lib/perobs/LockFile.rb +3 -0
  33. data/lib/perobs/Object.rb +28 -20
  34. data/lib/perobs/ObjectBase.rb +53 -10
  35. data/lib/perobs/PersistentObjectCache.rb +142 -0
  36. data/lib/perobs/PersistentObjectCacheLine.rb +99 -0
  37. data/lib/perobs/ProgressMeter.rb +97 -0
  38. data/lib/perobs/SpaceManager.rb +273 -0
  39. data/lib/perobs/SpaceTree.rb +63 -47
  40. data/lib/perobs/SpaceTreeNode.rb +134 -115
  41. data/lib/perobs/SpaceTreeNodeLink.rb +1 -1
  42. data/lib/perobs/StackFile.rb +1 -1
  43. data/lib/perobs/Store.rb +180 -70
  44. data/lib/perobs/version.rb +1 -1
  45. data/perobs.gemspec +4 -4
  46. data/test/Array_spec.rb +48 -39
  47. data/test/BTreeDB_spec.rb +2 -2
  48. data/test/BTree_spec.rb +50 -1
  49. data/test/BigArray_spec.rb +261 -0
  50. data/test/BigHash_spec.rb +152 -0
  51. data/test/BigTreeNode_spec.rb +153 -0
  52. data/test/BigTree_spec.rb +259 -0
  53. data/test/EquiBlobsFile_spec.rb +105 -5
  54. data/test/FNV_Hash_1a_64_spec.rb +59 -0
  55. data/test/FlatFileDB_spec.rb +199 -15
  56. data/test/FuzzyStringMatcher_spec.rb +261 -0
  57. data/test/Hash_spec.rb +27 -16
  58. data/test/IDList_spec.rb +77 -0
  59. data/test/LegacyDBs/LegacyDB.rb +155 -0
  60. data/test/LegacyDBs/version_3/class_map.json +1 -0
  61. data/test/LegacyDBs/version_3/config.json +1 -0
  62. data/test/LegacyDBs/version_3/database.blobs +0 -0
  63. data/test/LegacyDBs/version_3/database_spaces.blobs +0 -0
  64. data/test/LegacyDBs/version_3/index.blobs +0 -0
  65. data/test/LegacyDBs/version_3/version +1 -0
  66. data/test/LockFile_spec.rb +9 -6
  67. data/test/Object_spec.rb +5 -5
  68. data/test/SpaceManager_spec.rb +176 -0
  69. data/test/SpaceTree_spec.rb +27 -9
  70. data/test/Store_spec.rb +353 -206
  71. data/test/perobs_spec.rb +7 -3
  72. data/test/spec_helper.rb +9 -4
  73. metadata +59 -16
  74. data/lib/perobs/SpaceTreeNodeCache.rb +0 -76
  75. data/lib/perobs/TreeDB.rb +0 -277
data/test/perobs_spec.rb CHANGED
@@ -28,7 +28,7 @@ require 'perobs'
28
28
 
29
29
  class Person < PEROBS::Object
30
30
 
31
- po_attr :name, :zip, :bmi, :married, :related, :relatives
31
+ attr_persist :name, :zip, :bmi, :married, :related, :relatives
32
32
 
33
33
  def initialize(store)
34
34
  super
@@ -84,7 +84,7 @@ describe PEROBS::Store do
84
84
  jane.married = true
85
85
  jane.relatives = 'test'
86
86
 
87
- store.sync
87
+ store.exit
88
88
 
89
89
  store = PEROBS::Store.new(@db_name)
90
90
  john = store['john']
@@ -114,6 +114,7 @@ describe PEROBS::Store do
114
114
  0.upto(20) do |i|
115
115
  expect(store["person#{i}"].name).to eq("Person #{i}")
116
116
  end
117
+ store.exit
117
118
  end
118
119
 
119
120
  it 'should detect modification to non-working objects' do
@@ -125,11 +126,12 @@ describe PEROBS::Store do
125
126
  0.upto(20) do |i|
126
127
  store["person#{i}"].name = "New Person #{i}"
127
128
  end
128
- store.sync
129
+ store.exit
129
130
  store = PEROBS::Store.new(@db_name)
130
131
  0.upto(20) do |i|
131
132
  expect(store["person#{i}"].name).to eq("New Person #{i}")
132
133
  end
134
+ store.exit
133
135
  end
134
136
 
135
137
  it 'should garbage collect unlinked objects' do
@@ -143,10 +145,12 @@ describe PEROBS::Store do
143
145
  store.sync
144
146
  store['person1'] = nil
145
147
  store.gc
148
+ store.exit
146
149
  store = PEROBS::Store.new(@db_name)
147
150
  expect(store.object_by_id(id1)).to be_nil
148
151
  expect(store['person2']._id).to eq(id2)
149
152
  expect(store['person2'].related._id).to eq(id3)
153
+ store.exit
150
154
  end
151
155
 
152
156
  end
data/test/spec_helper.rb CHANGED
@@ -41,15 +41,20 @@ def generate_db_name(caller_file)
41
41
  end
42
42
 
43
43
  def capture_io
44
- PEROBS.log.open(io = StringIO.new)
44
+ old_stdout = $stdout
45
+ $stdout = out = StringIO.new
46
+ PEROBS.log.open(log = StringIO.new)
47
+
45
48
  begin
46
49
  yield
47
50
  ensure
51
+ $stdout = old_stdout
48
52
  PEROBS.log.open($stderr)
49
53
  end
50
54
 
51
- io.rewind
52
- io.read
53
- end
55
+ out.rewind
56
+ log.rewind
54
57
 
58
+ Struct.new(:out, :log).new(out.read, log.read)
59
+ end
55
60
 
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: perobs
3
3
  version: !ruby/object:Gem::Version
4
- version: 3.0.1
4
+ version: 4.3.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Chris Schlaeger
8
- autorequire:
8
+ autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2017-08-13 00:00:00.000000000 Z
11
+ date: 2021-02-14 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -16,42 +16,42 @@ dependencies:
16
16
  requirements:
17
17
  - - "~>"
18
18
  - !ruby/object:Gem::Version
19
- version: '1.6'
19
+ version: '2.3'
20
20
  type: :development
21
21
  prerelease: false
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
24
  - - "~>"
25
25
  - !ruby/object:Gem::Version
26
- version: '1.6'
26
+ version: '2.3'
27
27
  - !ruby/object:Gem::Dependency
28
28
  name: yard
29
29
  requirement: !ruby/object:Gem::Requirement
30
30
  requirements:
31
31
  - - "~>"
32
32
  - !ruby/object:Gem::Version
33
- version: 0.8.7
33
+ version: 0.9.12
34
34
  type: :development
35
35
  prerelease: false
36
36
  version_requirements: !ruby/object:Gem::Requirement
37
37
  requirements:
38
38
  - - "~>"
39
39
  - !ruby/object:Gem::Version
40
- version: 0.8.7
40
+ version: 0.9.12
41
41
  - !ruby/object:Gem::Dependency
42
42
  name: rake
43
43
  requirement: !ruby/object:Gem::Requirement
44
44
  requirements:
45
45
  - - "~>"
46
46
  - !ruby/object:Gem::Version
47
- version: '10.0'
47
+ version: 12.3.3
48
48
  type: :development
49
49
  prerelease: false
50
50
  version_requirements: !ruby/object:Gem::Requirement
51
51
  requirements:
52
52
  - - "~>"
53
53
  - !ruby/object:Gem::Version
54
- version: '10.0'
54
+ version: 12.3.3
55
55
  description: Library to provide a persistent object store
56
56
  email:
57
57
  - chris@linux.com
@@ -72,28 +72,42 @@ files:
72
72
  - lib/perobs/BTreeNode.rb
73
73
  - lib/perobs/BTreeNodeCache.rb
74
74
  - lib/perobs/BTreeNodeLink.rb
75
+ - lib/perobs/BigArray.rb
76
+ - lib/perobs/BigArrayNode.rb
77
+ - lib/perobs/BigHash.rb
78
+ - lib/perobs/BigTree.rb
79
+ - lib/perobs/BigTreeNode.rb
75
80
  - lib/perobs/Cache.rb
76
81
  - lib/perobs/ClassMap.rb
82
+ - lib/perobs/ConsoleProgressMeter.rb
77
83
  - lib/perobs/DataBase.rb
78
84
  - lib/perobs/DynamoDB.rb
79
85
  - lib/perobs/EquiBlobsFile.rb
86
+ - lib/perobs/FNV_Hash_1a_64.rb
80
87
  - lib/perobs/FlatFile.rb
81
88
  - lib/perobs/FlatFileBlobHeader.rb
82
89
  - lib/perobs/FlatFileDB.rb
90
+ - lib/perobs/FuzzyStringMatcher.rb
83
91
  - lib/perobs/Handle.rb
84
92
  - lib/perobs/Hash.rb
93
+ - lib/perobs/IDList.rb
94
+ - lib/perobs/IDListPage.rb
95
+ - lib/perobs/IDListPageFile.rb
96
+ - lib/perobs/IDListPageRecord.rb
85
97
  - lib/perobs/LockFile.rb
86
98
  - lib/perobs/Log.rb
87
99
  - lib/perobs/Object.rb
88
100
  - lib/perobs/ObjectBase.rb
101
+ - lib/perobs/PersistentObjectCache.rb
102
+ - lib/perobs/PersistentObjectCacheLine.rb
103
+ - lib/perobs/ProgressMeter.rb
89
104
  - lib/perobs/RobustFile.rb
105
+ - lib/perobs/SpaceManager.rb
90
106
  - lib/perobs/SpaceTree.rb
91
107
  - lib/perobs/SpaceTreeNode.rb
92
- - lib/perobs/SpaceTreeNodeCache.rb
93
108
  - lib/perobs/SpaceTreeNodeLink.rb
94
109
  - lib/perobs/StackFile.rb
95
110
  - lib/perobs/Store.rb
96
- - lib/perobs/TreeDB.rb
97
111
  - lib/perobs/version.rb
98
112
  - perobs.gemspec
99
113
  - tasks/changelog.rake
@@ -103,12 +117,27 @@ files:
103
117
  - test/Array_spec.rb
104
118
  - test/BTreeDB_spec.rb
105
119
  - test/BTree_spec.rb
120
+ - test/BigArray_spec.rb
121
+ - test/BigHash_spec.rb
122
+ - test/BigTreeNode_spec.rb
123
+ - test/BigTree_spec.rb
106
124
  - test/ClassMap_spec.rb
107
125
  - test/EquiBlobsFile_spec.rb
126
+ - test/FNV_Hash_1a_64_spec.rb
108
127
  - test/FlatFileDB_spec.rb
128
+ - test/FuzzyStringMatcher_spec.rb
109
129
  - test/Hash_spec.rb
130
+ - test/IDList_spec.rb
131
+ - test/LegacyDBs/LegacyDB.rb
132
+ - test/LegacyDBs/version_3/class_map.json
133
+ - test/LegacyDBs/version_3/config.json
134
+ - test/LegacyDBs/version_3/database.blobs
135
+ - test/LegacyDBs/version_3/database_spaces.blobs
136
+ - test/LegacyDBs/version_3/index.blobs
137
+ - test/LegacyDBs/version_3/version
110
138
  - test/LockFile_spec.rb
111
139
  - test/Object_spec.rb
140
+ - test/SpaceManager_spec.rb
112
141
  - test/SpaceTree_spec.rb
113
142
  - test/StackFile_spec.rb
114
143
  - test/Store_spec.rb
@@ -118,7 +147,7 @@ homepage: https://github.com/scrapper/perobs
118
147
  licenses:
119
148
  - MIT
120
149
  metadata: {}
121
- post_install_message:
150
+ post_install_message:
122
151
  rdoc_options: []
123
152
  require_paths:
124
153
  - lib
@@ -126,28 +155,42 @@ required_ruby_version: !ruby/object:Gem::Requirement
126
155
  requirements:
127
156
  - - ">="
128
157
  - !ruby/object:Gem::Version
129
- version: '2.0'
158
+ version: '2.4'
130
159
  required_rubygems_version: !ruby/object:Gem::Requirement
131
160
  requirements:
132
161
  - - ">="
133
162
  - !ruby/object:Gem::Version
134
163
  version: '0'
135
164
  requirements: []
136
- rubyforge_project:
137
- rubygems_version: 2.2.5
138
- signing_key:
165
+ rubygems_version: 3.2.3
166
+ signing_key:
139
167
  specification_version: 4
140
168
  summary: Persistent Ruby Object Store
141
169
  test_files:
142
170
  - test/Array_spec.rb
143
171
  - test/BTreeDB_spec.rb
144
172
  - test/BTree_spec.rb
173
+ - test/BigArray_spec.rb
174
+ - test/BigHash_spec.rb
175
+ - test/BigTreeNode_spec.rb
176
+ - test/BigTree_spec.rb
145
177
  - test/ClassMap_spec.rb
146
178
  - test/EquiBlobsFile_spec.rb
179
+ - test/FNV_Hash_1a_64_spec.rb
147
180
  - test/FlatFileDB_spec.rb
181
+ - test/FuzzyStringMatcher_spec.rb
148
182
  - test/Hash_spec.rb
183
+ - test/IDList_spec.rb
184
+ - test/LegacyDBs/LegacyDB.rb
185
+ - test/LegacyDBs/version_3/class_map.json
186
+ - test/LegacyDBs/version_3/config.json
187
+ - test/LegacyDBs/version_3/database.blobs
188
+ - test/LegacyDBs/version_3/database_spaces.blobs
189
+ - test/LegacyDBs/version_3/index.blobs
190
+ - test/LegacyDBs/version_3/version
149
191
  - test/LockFile_spec.rb
150
192
  - test/Object_spec.rb
193
+ - test/SpaceManager_spec.rb
151
194
  - test/SpaceTree_spec.rb
152
195
  - test/StackFile_spec.rb
153
196
  - test/Store_spec.rb
@@ -1,76 +0,0 @@
1
- # encoding: UTF-8
2
- #
3
- # = SpaceTree.rb -- Persistent Ruby Object Store
4
- #
5
- # Copyright (c) 2016, 2017 by Chris Schlaeger <chris@taskjuggler.org>
6
- #
7
- # MIT License
8
- #
9
- # Permission is hereby granted, free of charge, to any person obtaining
10
- # a copy of this software and associated documentation files (the
11
- # "Software"), to deal in the Software without restriction, including
12
- # without limitation the rights to use, copy, modify, merge, publish,
13
- # distribute, sublicense, and/or sell copies of the Software, and to
14
- # permit persons to whom the Software is furnished to do so, subject to
15
- # the following conditions:
16
- #
17
- # The above copyright notice and this permission notice shall be
18
- # included in all copies or substantial portions of the Software.
19
- #
20
- # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
21
- # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
22
- # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
23
- # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
24
- # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
25
- # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
26
- # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
27
-
28
- require 'perobs/SpaceTreeNode'
29
-
30
- module PEROBS
31
-
32
- class SpaceTreeNodeCache
33
-
34
- # Simple cache that can hold up to size SpaceTreeNode entries. Entries are
35
- # hashed with a simple node_address % size function. This keeps the
36
- # overhead for managing the cache extremely low yet giving an OK
37
- # probability to have cache hits.
38
- # @param size [Integer] maximum number of cache entries
39
- def initialize(size)
40
- @size = size
41
- clear
42
- end
43
-
44
- # Insert a node into the cache.
45
- # @param node [SpaceTreeNode]
46
- def insert(node)
47
- @entries[node.node_address % @size] = node
48
- end
49
-
50
- # Retrieve a node reference from the cache.
51
- # @param address [Integer] address of the node to retrieve.
52
- def get(address)
53
- node = @entries[address % @size]
54
- # We can have collisions. Check if the cached node really matches the
55
- # requested address.
56
- (node && node.node_address == address) ? node : nil
57
- end
58
-
59
- # Remove a node from the cache.
60
- # @param address [Integer] address of node to remove.
61
- def delete(address)
62
- index = address % @size
63
- if (node = @entries[index]) && node.node_address == address
64
- @entries[index] = nil
65
- end
66
- end
67
-
68
- # Remove all entries from the cache.
69
- def clear
70
- @entries = ::Array.new(@size)
71
- end
72
-
73
- end
74
-
75
- end
76
-
data/lib/perobs/TreeDB.rb DELETED
@@ -1,277 +0,0 @@
1
- # encoding: UTF-8
2
- #
3
- # = BTreeDB.rb -- Persistent Ruby Object Store
4
- #
5
- # Copyright (c) 2015, 2016 by Chris Schlaeger <chris@taskjuggler.org>
6
- #
7
- # MIT License
8
- #
9
- # Permission is hereby granted, free of charge, to any person obtaining
10
- # a copy of this software and associated documentation files (the
11
- # "Software"), to deal in the Software without restriction, including
12
- # without limitation the rights to use, copy, modify, merge, publish,
13
- # distribute, sublicense, and/or sell copies of the Software, and to
14
- # permit persons to whom the Software is furnished to do so, subject to
15
- # the following conditions:
16
- #
17
- # The above copyright notice and this permission notice shall be
18
- # included in all copies or substantial portions of the Software.
19
- #
20
- # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
21
- # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
22
- # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
23
- # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
24
- # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
25
- # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
26
- # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
27
-
28
- require 'fileutils'
29
-
30
- require 'perobs/Log'
31
- require 'perobs/RobustFile'
32
- require 'perobs/DataBase'
33
- require 'perobs/BTreeBlob'
34
-
35
- module PEROBS
36
-
37
- # This class implements a BTree database using filesystem directories as
38
- # nodes and blob files as leafs. The BTree grows with the number of stored
39
- # entries. Each leaf node blob can hold a fixed number of entries. If more
40
- # entries need to be stored, the blob is replaced by a node with multiple
41
- # new leafs that store the entries of the previous node. The leafs are
42
- # implemented by the BTreeBlob class.
43
- class BTreeDB < DataBase
44
-
45
- attr_reader :max_blob_size
46
-
47
- # Create a new BTreeDB object.
48
- # @param db_name [String] name of the DB directory
49
- # @param options [Hash] options to customize the behavior. Currently only
50
- # the following options are supported:
51
- # :serializer : Can be :marshal, :json, :yaml
52
- # :dir_bits : The number of bits to use for the BTree nodes.
53
- # The value must be between 4 and 14. The larger
54
- # the number the more back-end directories are
55
- # being used. The default is 12 which results in
56
- # 4096 directories per node.
57
- # :max_blob_size : The maximum number of entries in the BTree leaf
58
- # nodes. The insert/find/delete time grows
59
- # linearly with the size.
60
- def initialize(db_name, options = {})
61
- super(options[:serializer] || :json)
62
-
63
- @db_dir = db_name
64
- # Create the database directory if it doesn't exist yet.
65
- ensure_dir_exists(@db_dir)
66
-
67
- # Read the existing DB config.
68
- @config = get_hash('config')
69
- check_option('serializer')
70
-
71
- # Check and set @dir_bits, the number of bits used for each tree level.
72
- @dir_bits = options[:dir_bits] || 12
73
- if @dir_bits < 4 || @dir_bits > 14
74
- PEROBS.log.fatal "dir_bits option (#{@dir_bits}) must be between 4 " +
75
- "and 12"
76
- end
77
- check_option('dir_bits')
78
-
79
- @max_blob_size = options[:max_blob_size] || 32
80
- if @max_blob_size < 4 || @max_blob_size > 128
81
- PEROBS.log.fatal "max_blob_size option (#{@max_blob_size}) must be " +
82
- "between 4 and 128"
83
- end
84
- check_option('max_blob_size')
85
-
86
- put_hash('config', @config)
87
-
88
- # This format string is used to create the directory name.
89
- @dir_format_string = "%0#{(@dir_bits / 4) +
90
- (@dir_bits % 4 == 0 ? 0 : 1)}X"
91
- # Bit mask to extract the dir_bits LSBs.
92
- @dir_mask = 2 ** @dir_bits - 1
93
- end
94
-
95
- # Delete the entire database. The database is no longer usable after this
96
- # method was called.
97
- def delete_database
98
- FileUtils.rm_rf(@db_dir)
99
- end
100
-
101
- def BTreeDB::delete_db(db_name)
102
- FileUtils.rm_rf(db_name)
103
- end
104
-
105
- # Return true if the object with given ID exists
106
- # @param id [Fixnum or Bignum]
107
- def include?(id)
108
- !(blob = find_blob(id)).nil? && !blob.find(id).nil?
109
- end
110
-
111
- # Store a simple Hash as a JSON encoded file into the DB directory.
112
- # @param name [String] Name of the hash. Will be used as file name.
113
- # @param hash [Hash] A Hash that maps String objects to strings or
114
- # numbers.
115
- def put_hash(name, hash)
116
- file_name = File.join(@db_dir, name + '.json')
117
- begin
118
- RobustFile.write(file_name, hash.to_json)
119
- rescue IOError => e
120
- PEROBS.log.fatal "Cannot write hash file '#{file_name}': #{e.message}"
121
- end
122
- end
123
-
124
- # Load the Hash with the given name.
125
- # @param name [String] Name of the hash.
126
- # @return [Hash] A Hash that maps String objects to strings or numbers.
127
- def get_hash(name)
128
- file_name = File.join(@db_dir, name + '.json')
129
- return ::Hash.new unless File.exist?(file_name)
130
-
131
- begin
132
- json = File.read(file_name)
133
- rescue => e
134
- PEROBS.log.fatal "Cannot read hash file '#{file_name}': #{e.message}"
135
- end
136
- JSON.parse(json, :create_additions => true)
137
- end
138
-
139
- # Store the given object into the cluster files.
140
- # @param obj [Hash] Object as defined by PEROBS::ObjectBase
141
- def put_object(obj, id)
142
- find_blob(id, true).write_object(id, serialize(obj))
143
- end
144
-
145
- # Load the given object from the filesystem.
146
- # @param id [Fixnum or Bignum] object ID
147
- # @return [Hash] Object as defined by PEROBS::ObjectBase or nil if ID does
148
- # not exist
149
- def get_object(id)
150
- return nil unless (blob = find_blob(id)) && (obj = blob.read_object(id))
151
- deserialize(obj)
152
- end
153
-
154
- # This method must be called to initiate the marking process.
155
- def clear_marks
156
- each_blob { |blob| blob.clear_marks }
157
- end
158
-
159
- # Permanently delete all objects that have not been marked. Those are
160
- # orphaned and are no longer referenced by any actively used object.
161
- # @return [Array] List of IDs that have been removed from the DB.
162
- def delete_unmarked_objects
163
- deleted_ids = []
164
- each_blob { |blob| deleted_ids += blob.delete_unmarked_entries }
165
- deleted_ids
166
- end
167
-
168
- # Mark an object.
169
- # @param id [Fixnum or Bignum] ID of the object to mark
170
- def mark(id)
171
- (blob = find_blob(id)) && blob.mark(id)
172
- end
173
-
174
- # Check if the object is marked.
175
- # @param id [Fixnum or Bignum] ID of the object to check
176
- # @param ignore_errors [Boolean] If set to true no errors will be raised
177
- # for non-existing objects.
178
- def is_marked?(id, ignore_errors = false)
179
- (blob = find_blob(id)) && blob.is_marked?(id, ignore_errors)
180
- end
181
-
182
- # Basic consistency check.
183
- # @param repair [TrueClass/FalseClass] True if found errors should be
184
- # repaired.
185
- def check_db(repair = false)
186
- each_blob { |blob| blob.check(repair) }
187
- end
188
-
189
- # Check if the stored object is syntactically correct.
190
- # @param id [Fixnum/Bignum] Object ID
191
- # @param repair [TrueClass/FalseClass] True if an repair attempt should be
192
- # made.
193
- # @return [TrueClass/FalseClass] True if the object is OK, otherwise
194
- # false.
195
- def check(id, repair)
196
- begin
197
- get_object(id)
198
- rescue => e
199
- PEROBS.log.error "Cannot read object with ID #{id}: #{e.message}"
200
- return false
201
- end
202
-
203
- true
204
- end
205
-
206
- # Store the given serialized object into the cluster files. This method is
207
- # for internal use only!
208
- # @param raw [String] Serialized Object as defined by PEROBS::ObjectBase
209
- # @param id [Fixnum or Bignum] Object ID
210
- def put_raw_object(raw, id)
211
- find_blob(id, true).write_object(id, raw)
212
- end
213
-
214
- private
215
-
216
- def find_blob(id, create_missing_blob = false, dir_name = @db_dir)
217
- dir_bits = id & @dir_mask
218
- sub_dir_name = File.join(dir_name, @dir_format_string % dir_bits)
219
-
220
- if Dir.exist?(sub_dir_name)
221
- if File.exist?(File.join(sub_dir_name, 'index'))
222
- # The directory is a blob directory and not a BTree node dir.
223
- return BTreeBlob.new(sub_dir_name, self)
224
- end
225
- else
226
- Dir.glob(File.join(dir_name, '*.index')).each do |fqfn|
227
- # Extract the 01-part of the filename
228
- lsb_string = File.basename(fqfn)[0..-6]
229
- # Convert the lsb_string into a Fixnum
230
- lsb = Integer('0b' + lsb_string)
231
- # Bit mask to match the LSBs
232
- mask = (2 ** lsb_string.length) - 1
233
- if (id & mask) == lsb
234
- return TreeBlob.new(sub_dir_name, lsb_string, self)
235
- end
236
- end
237
- if create_missing_blob
238
- # Create the new blob directory.
239
- Dir.mkdir(dir_name)
240
- # And initialize the blob DB.
241
- return BTreeBlob.new(dir_name, self)
242
- else
243
- return nil
244
- end
245
- end
246
-
247
- # Discard the least significant @dir_bits bits and start over again
248
- # with the directory that matches the @dir_bits LSBs of the new ID.
249
- id = id >> @dir_bits
250
- end
251
-
252
- def each_blob(&block)
253
- each_blob_r(@db_dir, &block)
254
- end
255
-
256
- def each_blob_r(dir, &block)
257
- Dir.glob(File.join(dir, '*')) do |dir_name|
258
- if is_blob_dir?(dir_name)
259
- block.call(BTreeBlob.new(dir_name, self))
260
- else
261
- each_blob_r(dir_name, &block)
262
- end
263
- end
264
- end
265
-
266
- def is_blob_dir?(dir_name)
267
- # A blob directory contains an 'index' and 'data' file. This is in
268
- # contrast to BTree node directories that only contain other
269
- # directories.
270
- index_file = File.join(dir_name, 'index')
271
- File.exist?(index_file)
272
- end
273
-
274
- end
275
-
276
- end
277
-