perobs 3.0.1 → 4.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/README.md +19 -18
- data/lib/perobs.rb +2 -0
- data/lib/perobs/Array.rb +68 -21
- data/lib/perobs/BTree.rb +110 -54
- data/lib/perobs/BTreeBlob.rb +14 -13
- data/lib/perobs/BTreeDB.rb +11 -10
- data/lib/perobs/BTreeNode.rb +551 -197
- data/lib/perobs/BTreeNodeCache.rb +10 -8
- data/lib/perobs/BTreeNodeLink.rb +11 -1
- data/lib/perobs/BigArray.rb +285 -0
- data/lib/perobs/BigArrayNode.rb +1002 -0
- data/lib/perobs/BigHash.rb +246 -0
- data/lib/perobs/BigTree.rb +197 -0
- data/lib/perobs/BigTreeNode.rb +873 -0
- data/lib/perobs/Cache.rb +47 -22
- data/lib/perobs/ClassMap.rb +2 -2
- data/lib/perobs/ConsoleProgressMeter.rb +61 -0
- data/lib/perobs/DataBase.rb +4 -3
- data/lib/perobs/DynamoDB.rb +62 -20
- data/lib/perobs/EquiBlobsFile.rb +174 -59
- data/lib/perobs/FNV_Hash_1a_64.rb +54 -0
- data/lib/perobs/FlatFile.rb +536 -242
- data/lib/perobs/FlatFileBlobHeader.rb +120 -84
- data/lib/perobs/FlatFileDB.rb +58 -27
- data/lib/perobs/FuzzyStringMatcher.rb +175 -0
- data/lib/perobs/Hash.rb +129 -35
- data/lib/perobs/IDList.rb +144 -0
- data/lib/perobs/IDListPage.rb +107 -0
- data/lib/perobs/IDListPageFile.rb +180 -0
- data/lib/perobs/IDListPageRecord.rb +142 -0
- data/lib/perobs/LockFile.rb +3 -0
- data/lib/perobs/Object.rb +28 -20
- data/lib/perobs/ObjectBase.rb +53 -10
- data/lib/perobs/PersistentObjectCache.rb +142 -0
- data/lib/perobs/PersistentObjectCacheLine.rb +99 -0
- data/lib/perobs/ProgressMeter.rb +97 -0
- data/lib/perobs/SpaceManager.rb +273 -0
- data/lib/perobs/SpaceTree.rb +63 -47
- data/lib/perobs/SpaceTreeNode.rb +134 -115
- data/lib/perobs/SpaceTreeNodeLink.rb +1 -1
- data/lib/perobs/StackFile.rb +1 -1
- data/lib/perobs/Store.rb +180 -70
- data/lib/perobs/version.rb +1 -1
- data/perobs.gemspec +4 -4
- data/test/Array_spec.rb +48 -39
- data/test/BTreeDB_spec.rb +2 -2
- data/test/BTree_spec.rb +50 -1
- data/test/BigArray_spec.rb +261 -0
- data/test/BigHash_spec.rb +152 -0
- data/test/BigTreeNode_spec.rb +153 -0
- data/test/BigTree_spec.rb +259 -0
- data/test/EquiBlobsFile_spec.rb +105 -5
- data/test/FNV_Hash_1a_64_spec.rb +59 -0
- data/test/FlatFileDB_spec.rb +199 -15
- data/test/FuzzyStringMatcher_spec.rb +261 -0
- data/test/Hash_spec.rb +27 -16
- data/test/IDList_spec.rb +77 -0
- data/test/LegacyDBs/LegacyDB.rb +155 -0
- data/test/LegacyDBs/version_3/class_map.json +1 -0
- data/test/LegacyDBs/version_3/config.json +1 -0
- data/test/LegacyDBs/version_3/database.blobs +0 -0
- data/test/LegacyDBs/version_3/database_spaces.blobs +0 -0
- data/test/LegacyDBs/version_3/index.blobs +0 -0
- data/test/LegacyDBs/version_3/version +1 -0
- data/test/LockFile_spec.rb +9 -6
- data/test/Object_spec.rb +5 -5
- data/test/SpaceManager_spec.rb +176 -0
- data/test/SpaceTree_spec.rb +27 -9
- data/test/Store_spec.rb +353 -206
- data/test/perobs_spec.rb +7 -3
- data/test/spec_helper.rb +9 -4
- metadata +59 -16
- data/lib/perobs/SpaceTreeNodeCache.rb +0 -76
- data/lib/perobs/TreeDB.rb +0 -277
data/test/perobs_spec.rb
CHANGED
@@ -28,7 +28,7 @@ require 'perobs'
|
|
28
28
|
|
29
29
|
class Person < PEROBS::Object
|
30
30
|
|
31
|
-
|
31
|
+
attr_persist :name, :zip, :bmi, :married, :related, :relatives
|
32
32
|
|
33
33
|
def initialize(store)
|
34
34
|
super
|
@@ -84,7 +84,7 @@ describe PEROBS::Store do
|
|
84
84
|
jane.married = true
|
85
85
|
jane.relatives = 'test'
|
86
86
|
|
87
|
-
store.
|
87
|
+
store.exit
|
88
88
|
|
89
89
|
store = PEROBS::Store.new(@db_name)
|
90
90
|
john = store['john']
|
@@ -114,6 +114,7 @@ describe PEROBS::Store do
|
|
114
114
|
0.upto(20) do |i|
|
115
115
|
expect(store["person#{i}"].name).to eq("Person #{i}")
|
116
116
|
end
|
117
|
+
store.exit
|
117
118
|
end
|
118
119
|
|
119
120
|
it 'should detect modification to non-working objects' do
|
@@ -125,11 +126,12 @@ describe PEROBS::Store do
|
|
125
126
|
0.upto(20) do |i|
|
126
127
|
store["person#{i}"].name = "New Person #{i}"
|
127
128
|
end
|
128
|
-
store.
|
129
|
+
store.exit
|
129
130
|
store = PEROBS::Store.new(@db_name)
|
130
131
|
0.upto(20) do |i|
|
131
132
|
expect(store["person#{i}"].name).to eq("New Person #{i}")
|
132
133
|
end
|
134
|
+
store.exit
|
133
135
|
end
|
134
136
|
|
135
137
|
it 'should garbage collect unlinked objects' do
|
@@ -143,10 +145,12 @@ describe PEROBS::Store do
|
|
143
145
|
store.sync
|
144
146
|
store['person1'] = nil
|
145
147
|
store.gc
|
148
|
+
store.exit
|
146
149
|
store = PEROBS::Store.new(@db_name)
|
147
150
|
expect(store.object_by_id(id1)).to be_nil
|
148
151
|
expect(store['person2']._id).to eq(id2)
|
149
152
|
expect(store['person2'].related._id).to eq(id3)
|
153
|
+
store.exit
|
150
154
|
end
|
151
155
|
|
152
156
|
end
|
data/test/spec_helper.rb
CHANGED
@@ -41,15 +41,20 @@ def generate_db_name(caller_file)
|
|
41
41
|
end
|
42
42
|
|
43
43
|
def capture_io
|
44
|
-
|
44
|
+
old_stdout = $stdout
|
45
|
+
$stdout = out = StringIO.new
|
46
|
+
PEROBS.log.open(log = StringIO.new)
|
47
|
+
|
45
48
|
begin
|
46
49
|
yield
|
47
50
|
ensure
|
51
|
+
$stdout = old_stdout
|
48
52
|
PEROBS.log.open($stderr)
|
49
53
|
end
|
50
54
|
|
51
|
-
|
52
|
-
|
53
|
-
end
|
55
|
+
out.rewind
|
56
|
+
log.rewind
|
54
57
|
|
58
|
+
Struct.new(:out, :log).new(out.read, log.read)
|
59
|
+
end
|
55
60
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: perobs
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 3.0
|
4
|
+
version: 4.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Chris Schlaeger
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2021-02-14 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -16,42 +16,42 @@ dependencies:
|
|
16
16
|
requirements:
|
17
17
|
- - "~>"
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version: '
|
19
|
+
version: '2.3'
|
20
20
|
type: :development
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
24
|
- - "~>"
|
25
25
|
- !ruby/object:Gem::Version
|
26
|
-
version: '
|
26
|
+
version: '2.3'
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
28
|
name: yard
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
30
30
|
requirements:
|
31
31
|
- - "~>"
|
32
32
|
- !ruby/object:Gem::Version
|
33
|
-
version: 0.
|
33
|
+
version: 0.9.12
|
34
34
|
type: :development
|
35
35
|
prerelease: false
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
38
|
- - "~>"
|
39
39
|
- !ruby/object:Gem::Version
|
40
|
-
version: 0.
|
40
|
+
version: 0.9.12
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
42
|
name: rake
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
44
44
|
requirements:
|
45
45
|
- - "~>"
|
46
46
|
- !ruby/object:Gem::Version
|
47
|
-
version:
|
47
|
+
version: 12.3.3
|
48
48
|
type: :development
|
49
49
|
prerelease: false
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
51
51
|
requirements:
|
52
52
|
- - "~>"
|
53
53
|
- !ruby/object:Gem::Version
|
54
|
-
version:
|
54
|
+
version: 12.3.3
|
55
55
|
description: Library to provide a persistent object store
|
56
56
|
email:
|
57
57
|
- chris@linux.com
|
@@ -72,28 +72,42 @@ files:
|
|
72
72
|
- lib/perobs/BTreeNode.rb
|
73
73
|
- lib/perobs/BTreeNodeCache.rb
|
74
74
|
- lib/perobs/BTreeNodeLink.rb
|
75
|
+
- lib/perobs/BigArray.rb
|
76
|
+
- lib/perobs/BigArrayNode.rb
|
77
|
+
- lib/perobs/BigHash.rb
|
78
|
+
- lib/perobs/BigTree.rb
|
79
|
+
- lib/perobs/BigTreeNode.rb
|
75
80
|
- lib/perobs/Cache.rb
|
76
81
|
- lib/perobs/ClassMap.rb
|
82
|
+
- lib/perobs/ConsoleProgressMeter.rb
|
77
83
|
- lib/perobs/DataBase.rb
|
78
84
|
- lib/perobs/DynamoDB.rb
|
79
85
|
- lib/perobs/EquiBlobsFile.rb
|
86
|
+
- lib/perobs/FNV_Hash_1a_64.rb
|
80
87
|
- lib/perobs/FlatFile.rb
|
81
88
|
- lib/perobs/FlatFileBlobHeader.rb
|
82
89
|
- lib/perobs/FlatFileDB.rb
|
90
|
+
- lib/perobs/FuzzyStringMatcher.rb
|
83
91
|
- lib/perobs/Handle.rb
|
84
92
|
- lib/perobs/Hash.rb
|
93
|
+
- lib/perobs/IDList.rb
|
94
|
+
- lib/perobs/IDListPage.rb
|
95
|
+
- lib/perobs/IDListPageFile.rb
|
96
|
+
- lib/perobs/IDListPageRecord.rb
|
85
97
|
- lib/perobs/LockFile.rb
|
86
98
|
- lib/perobs/Log.rb
|
87
99
|
- lib/perobs/Object.rb
|
88
100
|
- lib/perobs/ObjectBase.rb
|
101
|
+
- lib/perobs/PersistentObjectCache.rb
|
102
|
+
- lib/perobs/PersistentObjectCacheLine.rb
|
103
|
+
- lib/perobs/ProgressMeter.rb
|
89
104
|
- lib/perobs/RobustFile.rb
|
105
|
+
- lib/perobs/SpaceManager.rb
|
90
106
|
- lib/perobs/SpaceTree.rb
|
91
107
|
- lib/perobs/SpaceTreeNode.rb
|
92
|
-
- lib/perobs/SpaceTreeNodeCache.rb
|
93
108
|
- lib/perobs/SpaceTreeNodeLink.rb
|
94
109
|
- lib/perobs/StackFile.rb
|
95
110
|
- lib/perobs/Store.rb
|
96
|
-
- lib/perobs/TreeDB.rb
|
97
111
|
- lib/perobs/version.rb
|
98
112
|
- perobs.gemspec
|
99
113
|
- tasks/changelog.rake
|
@@ -103,12 +117,27 @@ files:
|
|
103
117
|
- test/Array_spec.rb
|
104
118
|
- test/BTreeDB_spec.rb
|
105
119
|
- test/BTree_spec.rb
|
120
|
+
- test/BigArray_spec.rb
|
121
|
+
- test/BigHash_spec.rb
|
122
|
+
- test/BigTreeNode_spec.rb
|
123
|
+
- test/BigTree_spec.rb
|
106
124
|
- test/ClassMap_spec.rb
|
107
125
|
- test/EquiBlobsFile_spec.rb
|
126
|
+
- test/FNV_Hash_1a_64_spec.rb
|
108
127
|
- test/FlatFileDB_spec.rb
|
128
|
+
- test/FuzzyStringMatcher_spec.rb
|
109
129
|
- test/Hash_spec.rb
|
130
|
+
- test/IDList_spec.rb
|
131
|
+
- test/LegacyDBs/LegacyDB.rb
|
132
|
+
- test/LegacyDBs/version_3/class_map.json
|
133
|
+
- test/LegacyDBs/version_3/config.json
|
134
|
+
- test/LegacyDBs/version_3/database.blobs
|
135
|
+
- test/LegacyDBs/version_3/database_spaces.blobs
|
136
|
+
- test/LegacyDBs/version_3/index.blobs
|
137
|
+
- test/LegacyDBs/version_3/version
|
110
138
|
- test/LockFile_spec.rb
|
111
139
|
- test/Object_spec.rb
|
140
|
+
- test/SpaceManager_spec.rb
|
112
141
|
- test/SpaceTree_spec.rb
|
113
142
|
- test/StackFile_spec.rb
|
114
143
|
- test/Store_spec.rb
|
@@ -118,7 +147,7 @@ homepage: https://github.com/scrapper/perobs
|
|
118
147
|
licenses:
|
119
148
|
- MIT
|
120
149
|
metadata: {}
|
121
|
-
post_install_message:
|
150
|
+
post_install_message:
|
122
151
|
rdoc_options: []
|
123
152
|
require_paths:
|
124
153
|
- lib
|
@@ -126,28 +155,42 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
126
155
|
requirements:
|
127
156
|
- - ">="
|
128
157
|
- !ruby/object:Gem::Version
|
129
|
-
version: '2.
|
158
|
+
version: '2.4'
|
130
159
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
131
160
|
requirements:
|
132
161
|
- - ">="
|
133
162
|
- !ruby/object:Gem::Version
|
134
163
|
version: '0'
|
135
164
|
requirements: []
|
136
|
-
|
137
|
-
|
138
|
-
signing_key:
|
165
|
+
rubygems_version: 3.2.3
|
166
|
+
signing_key:
|
139
167
|
specification_version: 4
|
140
168
|
summary: Persistent Ruby Object Store
|
141
169
|
test_files:
|
142
170
|
- test/Array_spec.rb
|
143
171
|
- test/BTreeDB_spec.rb
|
144
172
|
- test/BTree_spec.rb
|
173
|
+
- test/BigArray_spec.rb
|
174
|
+
- test/BigHash_spec.rb
|
175
|
+
- test/BigTreeNode_spec.rb
|
176
|
+
- test/BigTree_spec.rb
|
145
177
|
- test/ClassMap_spec.rb
|
146
178
|
- test/EquiBlobsFile_spec.rb
|
179
|
+
- test/FNV_Hash_1a_64_spec.rb
|
147
180
|
- test/FlatFileDB_spec.rb
|
181
|
+
- test/FuzzyStringMatcher_spec.rb
|
148
182
|
- test/Hash_spec.rb
|
183
|
+
- test/IDList_spec.rb
|
184
|
+
- test/LegacyDBs/LegacyDB.rb
|
185
|
+
- test/LegacyDBs/version_3/class_map.json
|
186
|
+
- test/LegacyDBs/version_3/config.json
|
187
|
+
- test/LegacyDBs/version_3/database.blobs
|
188
|
+
- test/LegacyDBs/version_3/database_spaces.blobs
|
189
|
+
- test/LegacyDBs/version_3/index.blobs
|
190
|
+
- test/LegacyDBs/version_3/version
|
149
191
|
- test/LockFile_spec.rb
|
150
192
|
- test/Object_spec.rb
|
193
|
+
- test/SpaceManager_spec.rb
|
151
194
|
- test/SpaceTree_spec.rb
|
152
195
|
- test/StackFile_spec.rb
|
153
196
|
- test/Store_spec.rb
|
@@ -1,76 +0,0 @@
|
|
1
|
-
# encoding: UTF-8
|
2
|
-
#
|
3
|
-
# = SpaceTree.rb -- Persistent Ruby Object Store
|
4
|
-
#
|
5
|
-
# Copyright (c) 2016, 2017 by Chris Schlaeger <chris@taskjuggler.org>
|
6
|
-
#
|
7
|
-
# MIT License
|
8
|
-
#
|
9
|
-
# Permission is hereby granted, free of charge, to any person obtaining
|
10
|
-
# a copy of this software and associated documentation files (the
|
11
|
-
# "Software"), to deal in the Software without restriction, including
|
12
|
-
# without limitation the rights to use, copy, modify, merge, publish,
|
13
|
-
# distribute, sublicense, and/or sell copies of the Software, and to
|
14
|
-
# permit persons to whom the Software is furnished to do so, subject to
|
15
|
-
# the following conditions:
|
16
|
-
#
|
17
|
-
# The above copyright notice and this permission notice shall be
|
18
|
-
# included in all copies or substantial portions of the Software.
|
19
|
-
#
|
20
|
-
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
21
|
-
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
22
|
-
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
23
|
-
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
24
|
-
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
25
|
-
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
26
|
-
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
27
|
-
|
28
|
-
require 'perobs/SpaceTreeNode'
|
29
|
-
|
30
|
-
module PEROBS
|
31
|
-
|
32
|
-
class SpaceTreeNodeCache
|
33
|
-
|
34
|
-
# Simple cache that can hold up to size SpaceTreeNode entries. Entries are
|
35
|
-
# hashed with a simple node_address % size function. This keeps the
|
36
|
-
# overhead for managing the cache extremely low yet giving an OK
|
37
|
-
# probability to have cache hits.
|
38
|
-
# @param size [Integer] maximum number of cache entries
|
39
|
-
def initialize(size)
|
40
|
-
@size = size
|
41
|
-
clear
|
42
|
-
end
|
43
|
-
|
44
|
-
# Insert a node into the cache.
|
45
|
-
# @param node [SpaceTreeNode]
|
46
|
-
def insert(node)
|
47
|
-
@entries[node.node_address % @size] = node
|
48
|
-
end
|
49
|
-
|
50
|
-
# Retrieve a node reference from the cache.
|
51
|
-
# @param address [Integer] address of the node to retrieve.
|
52
|
-
def get(address)
|
53
|
-
node = @entries[address % @size]
|
54
|
-
# We can have collisions. Check if the cached node really matches the
|
55
|
-
# requested address.
|
56
|
-
(node && node.node_address == address) ? node : nil
|
57
|
-
end
|
58
|
-
|
59
|
-
# Remove a node from the cache.
|
60
|
-
# @param address [Integer] address of node to remove.
|
61
|
-
def delete(address)
|
62
|
-
index = address % @size
|
63
|
-
if (node = @entries[index]) && node.node_address == address
|
64
|
-
@entries[index] = nil
|
65
|
-
end
|
66
|
-
end
|
67
|
-
|
68
|
-
# Remove all entries from the cache.
|
69
|
-
def clear
|
70
|
-
@entries = ::Array.new(@size)
|
71
|
-
end
|
72
|
-
|
73
|
-
end
|
74
|
-
|
75
|
-
end
|
76
|
-
|
data/lib/perobs/TreeDB.rb
DELETED
@@ -1,277 +0,0 @@
|
|
1
|
-
# encoding: UTF-8
|
2
|
-
#
|
3
|
-
# = BTreeDB.rb -- Persistent Ruby Object Store
|
4
|
-
#
|
5
|
-
# Copyright (c) 2015, 2016 by Chris Schlaeger <chris@taskjuggler.org>
|
6
|
-
#
|
7
|
-
# MIT License
|
8
|
-
#
|
9
|
-
# Permission is hereby granted, free of charge, to any person obtaining
|
10
|
-
# a copy of this software and associated documentation files (the
|
11
|
-
# "Software"), to deal in the Software without restriction, including
|
12
|
-
# without limitation the rights to use, copy, modify, merge, publish,
|
13
|
-
# distribute, sublicense, and/or sell copies of the Software, and to
|
14
|
-
# permit persons to whom the Software is furnished to do so, subject to
|
15
|
-
# the following conditions:
|
16
|
-
#
|
17
|
-
# The above copyright notice and this permission notice shall be
|
18
|
-
# included in all copies or substantial portions of the Software.
|
19
|
-
#
|
20
|
-
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
21
|
-
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
22
|
-
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
23
|
-
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
24
|
-
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
25
|
-
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
26
|
-
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
27
|
-
|
28
|
-
require 'fileutils'
|
29
|
-
|
30
|
-
require 'perobs/Log'
|
31
|
-
require 'perobs/RobustFile'
|
32
|
-
require 'perobs/DataBase'
|
33
|
-
require 'perobs/BTreeBlob'
|
34
|
-
|
35
|
-
module PEROBS
|
36
|
-
|
37
|
-
# This class implements a BTree database using filesystem directories as
|
38
|
-
# nodes and blob files as leafs. The BTree grows with the number of stored
|
39
|
-
# entries. Each leaf node blob can hold a fixed number of entries. If more
|
40
|
-
# entries need to be stored, the blob is replaced by a node with multiple
|
41
|
-
# new leafs that store the entries of the previous node. The leafs are
|
42
|
-
# implemented by the BTreeBlob class.
|
43
|
-
class BTreeDB < DataBase
|
44
|
-
|
45
|
-
attr_reader :max_blob_size
|
46
|
-
|
47
|
-
# Create a new BTreeDB object.
|
48
|
-
# @param db_name [String] name of the DB directory
|
49
|
-
# @param options [Hash] options to customize the behavior. Currently only
|
50
|
-
# the following options are supported:
|
51
|
-
# :serializer : Can be :marshal, :json, :yaml
|
52
|
-
# :dir_bits : The number of bits to use for the BTree nodes.
|
53
|
-
# The value must be between 4 and 14. The larger
|
54
|
-
# the number the more back-end directories are
|
55
|
-
# being used. The default is 12 which results in
|
56
|
-
# 4096 directories per node.
|
57
|
-
# :max_blob_size : The maximum number of entries in the BTree leaf
|
58
|
-
# nodes. The insert/find/delete time grows
|
59
|
-
# linearly with the size.
|
60
|
-
def initialize(db_name, options = {})
|
61
|
-
super(options[:serializer] || :json)
|
62
|
-
|
63
|
-
@db_dir = db_name
|
64
|
-
# Create the database directory if it doesn't exist yet.
|
65
|
-
ensure_dir_exists(@db_dir)
|
66
|
-
|
67
|
-
# Read the existing DB config.
|
68
|
-
@config = get_hash('config')
|
69
|
-
check_option('serializer')
|
70
|
-
|
71
|
-
# Check and set @dir_bits, the number of bits used for each tree level.
|
72
|
-
@dir_bits = options[:dir_bits] || 12
|
73
|
-
if @dir_bits < 4 || @dir_bits > 14
|
74
|
-
PEROBS.log.fatal "dir_bits option (#{@dir_bits}) must be between 4 " +
|
75
|
-
"and 12"
|
76
|
-
end
|
77
|
-
check_option('dir_bits')
|
78
|
-
|
79
|
-
@max_blob_size = options[:max_blob_size] || 32
|
80
|
-
if @max_blob_size < 4 || @max_blob_size > 128
|
81
|
-
PEROBS.log.fatal "max_blob_size option (#{@max_blob_size}) must be " +
|
82
|
-
"between 4 and 128"
|
83
|
-
end
|
84
|
-
check_option('max_blob_size')
|
85
|
-
|
86
|
-
put_hash('config', @config)
|
87
|
-
|
88
|
-
# This format string is used to create the directory name.
|
89
|
-
@dir_format_string = "%0#{(@dir_bits / 4) +
|
90
|
-
(@dir_bits % 4 == 0 ? 0 : 1)}X"
|
91
|
-
# Bit mask to extract the dir_bits LSBs.
|
92
|
-
@dir_mask = 2 ** @dir_bits - 1
|
93
|
-
end
|
94
|
-
|
95
|
-
# Delete the entire database. The database is no longer usable after this
|
96
|
-
# method was called.
|
97
|
-
def delete_database
|
98
|
-
FileUtils.rm_rf(@db_dir)
|
99
|
-
end
|
100
|
-
|
101
|
-
def BTreeDB::delete_db(db_name)
|
102
|
-
FileUtils.rm_rf(db_name)
|
103
|
-
end
|
104
|
-
|
105
|
-
# Return true if the object with given ID exists
|
106
|
-
# @param id [Fixnum or Bignum]
|
107
|
-
def include?(id)
|
108
|
-
!(blob = find_blob(id)).nil? && !blob.find(id).nil?
|
109
|
-
end
|
110
|
-
|
111
|
-
# Store a simple Hash as a JSON encoded file into the DB directory.
|
112
|
-
# @param name [String] Name of the hash. Will be used as file name.
|
113
|
-
# @param hash [Hash] A Hash that maps String objects to strings or
|
114
|
-
# numbers.
|
115
|
-
def put_hash(name, hash)
|
116
|
-
file_name = File.join(@db_dir, name + '.json')
|
117
|
-
begin
|
118
|
-
RobustFile.write(file_name, hash.to_json)
|
119
|
-
rescue IOError => e
|
120
|
-
PEROBS.log.fatal "Cannot write hash file '#{file_name}': #{e.message}"
|
121
|
-
end
|
122
|
-
end
|
123
|
-
|
124
|
-
# Load the Hash with the given name.
|
125
|
-
# @param name [String] Name of the hash.
|
126
|
-
# @return [Hash] A Hash that maps String objects to strings or numbers.
|
127
|
-
def get_hash(name)
|
128
|
-
file_name = File.join(@db_dir, name + '.json')
|
129
|
-
return ::Hash.new unless File.exist?(file_name)
|
130
|
-
|
131
|
-
begin
|
132
|
-
json = File.read(file_name)
|
133
|
-
rescue => e
|
134
|
-
PEROBS.log.fatal "Cannot read hash file '#{file_name}': #{e.message}"
|
135
|
-
end
|
136
|
-
JSON.parse(json, :create_additions => true)
|
137
|
-
end
|
138
|
-
|
139
|
-
# Store the given object into the cluster files.
|
140
|
-
# @param obj [Hash] Object as defined by PEROBS::ObjectBase
|
141
|
-
def put_object(obj, id)
|
142
|
-
find_blob(id, true).write_object(id, serialize(obj))
|
143
|
-
end
|
144
|
-
|
145
|
-
# Load the given object from the filesystem.
|
146
|
-
# @param id [Fixnum or Bignum] object ID
|
147
|
-
# @return [Hash] Object as defined by PEROBS::ObjectBase or nil if ID does
|
148
|
-
# not exist
|
149
|
-
def get_object(id)
|
150
|
-
return nil unless (blob = find_blob(id)) && (obj = blob.read_object(id))
|
151
|
-
deserialize(obj)
|
152
|
-
end
|
153
|
-
|
154
|
-
# This method must be called to initiate the marking process.
|
155
|
-
def clear_marks
|
156
|
-
each_blob { |blob| blob.clear_marks }
|
157
|
-
end
|
158
|
-
|
159
|
-
# Permanently delete all objects that have not been marked. Those are
|
160
|
-
# orphaned and are no longer referenced by any actively used object.
|
161
|
-
# @return [Array] List of IDs that have been removed from the DB.
|
162
|
-
def delete_unmarked_objects
|
163
|
-
deleted_ids = []
|
164
|
-
each_blob { |blob| deleted_ids += blob.delete_unmarked_entries }
|
165
|
-
deleted_ids
|
166
|
-
end
|
167
|
-
|
168
|
-
# Mark an object.
|
169
|
-
# @param id [Fixnum or Bignum] ID of the object to mark
|
170
|
-
def mark(id)
|
171
|
-
(blob = find_blob(id)) && blob.mark(id)
|
172
|
-
end
|
173
|
-
|
174
|
-
# Check if the object is marked.
|
175
|
-
# @param id [Fixnum or Bignum] ID of the object to check
|
176
|
-
# @param ignore_errors [Boolean] If set to true no errors will be raised
|
177
|
-
# for non-existing objects.
|
178
|
-
def is_marked?(id, ignore_errors = false)
|
179
|
-
(blob = find_blob(id)) && blob.is_marked?(id, ignore_errors)
|
180
|
-
end
|
181
|
-
|
182
|
-
# Basic consistency check.
|
183
|
-
# @param repair [TrueClass/FalseClass] True if found errors should be
|
184
|
-
# repaired.
|
185
|
-
def check_db(repair = false)
|
186
|
-
each_blob { |blob| blob.check(repair) }
|
187
|
-
end
|
188
|
-
|
189
|
-
# Check if the stored object is syntactically correct.
|
190
|
-
# @param id [Fixnum/Bignum] Object ID
|
191
|
-
# @param repair [TrueClass/FalseClass] True if an repair attempt should be
|
192
|
-
# made.
|
193
|
-
# @return [TrueClass/FalseClass] True if the object is OK, otherwise
|
194
|
-
# false.
|
195
|
-
def check(id, repair)
|
196
|
-
begin
|
197
|
-
get_object(id)
|
198
|
-
rescue => e
|
199
|
-
PEROBS.log.error "Cannot read object with ID #{id}: #{e.message}"
|
200
|
-
return false
|
201
|
-
end
|
202
|
-
|
203
|
-
true
|
204
|
-
end
|
205
|
-
|
206
|
-
# Store the given serialized object into the cluster files. This method is
|
207
|
-
# for internal use only!
|
208
|
-
# @param raw [String] Serialized Object as defined by PEROBS::ObjectBase
|
209
|
-
# @param id [Fixnum or Bignum] Object ID
|
210
|
-
def put_raw_object(raw, id)
|
211
|
-
find_blob(id, true).write_object(id, raw)
|
212
|
-
end
|
213
|
-
|
214
|
-
private
|
215
|
-
|
216
|
-
def find_blob(id, create_missing_blob = false, dir_name = @db_dir)
|
217
|
-
dir_bits = id & @dir_mask
|
218
|
-
sub_dir_name = File.join(dir_name, @dir_format_string % dir_bits)
|
219
|
-
|
220
|
-
if Dir.exist?(sub_dir_name)
|
221
|
-
if File.exist?(File.join(sub_dir_name, 'index'))
|
222
|
-
# The directory is a blob directory and not a BTree node dir.
|
223
|
-
return BTreeBlob.new(sub_dir_name, self)
|
224
|
-
end
|
225
|
-
else
|
226
|
-
Dir.glob(File.join(dir_name, '*.index')).each do |fqfn|
|
227
|
-
# Extract the 01-part of the filename
|
228
|
-
lsb_string = File.basename(fqfn)[0..-6]
|
229
|
-
# Convert the lsb_string into a Fixnum
|
230
|
-
lsb = Integer('0b' + lsb_string)
|
231
|
-
# Bit mask to match the LSBs
|
232
|
-
mask = (2 ** lsb_string.length) - 1
|
233
|
-
if (id & mask) == lsb
|
234
|
-
return TreeBlob.new(sub_dir_name, lsb_string, self)
|
235
|
-
end
|
236
|
-
end
|
237
|
-
if create_missing_blob
|
238
|
-
# Create the new blob directory.
|
239
|
-
Dir.mkdir(dir_name)
|
240
|
-
# And initialize the blob DB.
|
241
|
-
return BTreeBlob.new(dir_name, self)
|
242
|
-
else
|
243
|
-
return nil
|
244
|
-
end
|
245
|
-
end
|
246
|
-
|
247
|
-
# Discard the least significant @dir_bits bits and start over again
|
248
|
-
# with the directory that matches the @dir_bits LSBs of the new ID.
|
249
|
-
id = id >> @dir_bits
|
250
|
-
end
|
251
|
-
|
252
|
-
def each_blob(&block)
|
253
|
-
each_blob_r(@db_dir, &block)
|
254
|
-
end
|
255
|
-
|
256
|
-
def each_blob_r(dir, &block)
|
257
|
-
Dir.glob(File.join(dir, '*')) do |dir_name|
|
258
|
-
if is_blob_dir?(dir_name)
|
259
|
-
block.call(BTreeBlob.new(dir_name, self))
|
260
|
-
else
|
261
|
-
each_blob_r(dir_name, &block)
|
262
|
-
end
|
263
|
-
end
|
264
|
-
end
|
265
|
-
|
266
|
-
def is_blob_dir?(dir_name)
|
267
|
-
# A blob directory contains an 'index' and 'data' file. This is in
|
268
|
-
# contrast to BTree node directories that only contain other
|
269
|
-
# directories.
|
270
|
-
index_file = File.join(dir_name, 'index')
|
271
|
-
File.exist?(index_file)
|
272
|
-
end
|
273
|
-
|
274
|
-
end
|
275
|
-
|
276
|
-
end
|
277
|
-
|