perobs 2.4.1 → 2.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/perobs/BTreeBlob.rb +8 -0
- data/lib/perobs/Cache.rb +1 -1
- data/lib/perobs/FixedSizeBlobFile.rb +4 -0
- data/lib/perobs/FlatFile.rb +14 -4
- data/lib/perobs/IndexTree.rb +7 -36
- data/lib/perobs/IndexTreeNode.rb +23 -9
- data/lib/perobs/StackFile.rb +4 -0
- data/lib/perobs/version.rb +1 -1
- data/test/FlatFileDB_spec.rb +5 -0
- data/test/Store_spec.rb +1 -1
- metadata +2 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 7913287f5998dfad64a99f239723ecb905798fe9
|
4
|
+
data.tar.gz: d667fe5eab1c96b4003640c4d924d97393fdf4b3
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 0140b0b11d4f6f7c5f01743d5e361cf045c4533ab339ee92e32e72b4ddca7046684a6ac00445667c0a08d953946cc0fd93d6563a9b3482759e436151d0a548df
|
7
|
+
data.tar.gz: 5b6b9101321994f98f42c153c788bfece79183a074baf01d1e2ed377a0ff929b3308947b272bca1892807e425abfd6d2decd208f07179151fbca703d76eaa8ac
|
data/lib/perobs/BTreeBlob.rb
CHANGED
@@ -300,6 +300,9 @@ module PEROBS
|
|
300
300
|
if File.exist?(@index_file_name)
|
301
301
|
begin
|
302
302
|
File.open(@index_file_name, 'rb') do |f|
|
303
|
+
unless f.flock(File::LOCK_NB | File::LOCK_EX)
|
304
|
+
PEROBS.log.fatal 'BTreeDB Database is locked by another process'
|
305
|
+
end
|
303
306
|
# Since version 2.3.0, all index files start with a header.
|
304
307
|
# Earlier versions did not yet have this header. The header is 24
|
305
308
|
# bytes long. The 2nd set of 8 bytes must be 0 to distinguish the
|
@@ -347,6 +350,7 @@ module PEROBS
|
|
347
350
|
@entries << e
|
348
351
|
@entries_by_id[e[ID]] = e
|
349
352
|
end
|
353
|
+
f.flock(File::LOCK_UN)
|
350
354
|
end
|
351
355
|
rescue => e
|
352
356
|
PEROBS.log.fatal "BTreeBlob file #{@index_file_name} corrupted: " +
|
@@ -358,11 +362,15 @@ module PEROBS
|
|
358
362
|
def write_index
|
359
363
|
begin
|
360
364
|
File.open(@index_file_name, 'wb') do |f|
|
365
|
+
unless f.flock(File::LOCK_NB | File::LOCK_EX)
|
366
|
+
PEROBS.log.fatal 'BTreeDB Database is locked by another process'
|
367
|
+
end
|
361
368
|
# See read_index for data format documentation.
|
362
369
|
f.write([ PEROBS_MAGIC, 0, 1].pack('QQQ'))
|
363
370
|
@entries.each do |entry|
|
364
371
|
f.write(entry.pack('QQQCL'))
|
365
372
|
end
|
373
|
+
f.flock(File::LOCK_UN)
|
366
374
|
end
|
367
375
|
rescue => e
|
368
376
|
PEROBS.log.fatal "Cannot write BTreeBlob index file " +
|
data/lib/perobs/Cache.rb
CHANGED
@@ -124,7 +124,7 @@ module PEROBS
|
|
124
124
|
end
|
125
125
|
|
126
126
|
# Tell the cache to start a new transaction. If no other transaction is
|
127
|
-
# active, the write
|
127
|
+
# active, the write cache is flushed before the transaction is started.
|
128
128
|
def begin_transaction
|
129
129
|
if @transaction_stack.empty?
|
130
130
|
# The new transaction is the top-level transaction. Flush the write
|
@@ -59,6 +59,9 @@ module PEROBS
|
|
59
59
|
rescue IOError => e
|
60
60
|
PEROBS.log.fatal "Cannot open blob file #{@file_name}: #{e.message}"
|
61
61
|
end
|
62
|
+
unless @f.flock(File::LOCK_NB | File::LOCK_EX)
|
63
|
+
PEROBS.log.fatal 'Database blob file is locked by another process'
|
64
|
+
end
|
62
65
|
@free_list.open
|
63
66
|
end
|
64
67
|
|
@@ -68,6 +71,7 @@ module PEROBS
|
|
68
71
|
@free_list.close
|
69
72
|
begin
|
70
73
|
@f.flush
|
74
|
+
@f.flock(File::LOCK_UN)
|
71
75
|
@f.close
|
72
76
|
rescue IOError => e
|
73
77
|
PEROBS.log.fatal "Cannot close blob file #{@file_name}: #{e.message}"
|
data/lib/perobs/FlatFile.rb
CHANGED
@@ -81,6 +81,9 @@ module PEROBS
|
|
81
81
|
PEROBS.log.fatal "Cannot open flat file database #{file_name}: " +
|
82
82
|
e.message
|
83
83
|
end
|
84
|
+
unless @f.flock(File::LOCK_NB | File::LOCK_EX)
|
85
|
+
PEROBS.log.fatal 'Database is locked by another process'
|
86
|
+
end
|
84
87
|
@index.open
|
85
88
|
@space_list.open
|
86
89
|
end
|
@@ -91,6 +94,7 @@ module PEROBS
|
|
91
94
|
@space_list.close
|
92
95
|
@index.close
|
93
96
|
@f.flush
|
97
|
+
@f.flock(File::LOCK_UN)
|
94
98
|
@f.close
|
95
99
|
@f = nil
|
96
100
|
end
|
@@ -287,7 +291,8 @@ module PEROBS
|
|
287
291
|
|
288
292
|
each_blob_header do |pos, mark, length, blob_id, crc|
|
289
293
|
total_blob_count += 1
|
290
|
-
if (mark &
|
294
|
+
if (mark & 3 == 3)
|
295
|
+
# Clear all valid and marked blocks.
|
291
296
|
marked_blob_count += 1
|
292
297
|
begin
|
293
298
|
@f.seek(pos)
|
@@ -307,14 +312,17 @@ module PEROBS
|
|
307
312
|
# implementation. No additional space will be needed on the file system.
|
308
313
|
def defragmentize
|
309
314
|
distance = 0
|
315
|
+
deleted_blobs = 0
|
316
|
+
valid_blobs = 0
|
310
317
|
t = Time.now
|
311
|
-
PEROBS.log.
|
318
|
+
PEROBS.log.info "Defragmenting FlatFile"
|
312
319
|
# Iterate over all entries.
|
313
320
|
each_blob_header do |pos, mark, length, blob_id, crc|
|
314
321
|
# Total size of the current entry
|
315
322
|
entry_bytes = BLOB_HEADER_LENGTH + length
|
316
323
|
if (mark & 1 == 1)
|
317
324
|
# We have found a valid entry.
|
325
|
+
valid_blobs += 1
|
318
326
|
if distance > 0
|
319
327
|
begin
|
320
328
|
# Read current entry into a buffer
|
@@ -336,11 +344,13 @@ module PEROBS
|
|
336
344
|
end
|
337
345
|
end
|
338
346
|
else
|
347
|
+
deleted_blobs += 1
|
339
348
|
distance += entry_bytes
|
340
349
|
end
|
341
350
|
end
|
342
|
-
PEROBS.log.
|
343
|
-
PEROBS.log.
|
351
|
+
PEROBS.log.info "FlatFile defragmented in #{Time.now - t} seconds"
|
352
|
+
PEROBS.log.info "#{distance / 1000} KiB/#{deleted_blobs} blobs of " +
|
353
|
+
"#{@f.size / 1000} KiB/#{valid_blobs} blobs or " +
|
344
354
|
"#{'%.1f' % (distance.to_f / @f.size * 100.0)}% reclaimed"
|
345
355
|
|
346
356
|
@f.flush
|
data/lib/perobs/IndexTree.rb
CHANGED
@@ -35,11 +35,6 @@ module PEROBS
|
|
35
35
|
# search in the tree is much faster than the linear search in the FlatFile.
|
36
36
|
class IndexTree
|
37
37
|
|
38
|
-
# Determines how many levels of the IndexTree will be kept in memory to
|
39
|
-
# accerlerate the access. A number of 7 will keep up to 21845 entries in
|
40
|
-
# the cache but will accelerate the access to the FlatFile address.
|
41
|
-
MAX_CACHED_LEVEL = 7
|
42
|
-
|
43
38
|
attr_reader :nodes, :ids
|
44
39
|
|
45
40
|
def initialize(db_dir)
|
@@ -55,10 +50,6 @@ module PEROBS
|
|
55
50
|
# file which contains the full object ID and the address of the
|
56
51
|
# corresponding object in the FlatFile.
|
57
52
|
@ids = FixedSizeBlobFile.new(db_dir, 'object_id_index', 2 * 8)
|
58
|
-
|
59
|
-
# The first MAX_CACHED_LEVEL levels of nodes will be cached in memory to
|
60
|
-
# improve access times.
|
61
|
-
@node_cache = {}
|
62
53
|
end
|
63
54
|
|
64
55
|
# Open the tree files.
|
@@ -72,6 +63,7 @@ module PEROBS
|
|
72
63
|
def close
|
73
64
|
@ids.close
|
74
65
|
@nodes.close
|
66
|
+
@root = nil
|
75
67
|
end
|
76
68
|
|
77
69
|
# Flush out all unwritten data
|
@@ -84,7 +76,6 @@ module PEROBS
|
|
84
76
|
def clear
|
85
77
|
@nodes.clear
|
86
78
|
@ids.clear
|
87
|
-
@node_cache = {}
|
88
79
|
@root = IndexTreeNode.new(self, 0, 0)
|
89
80
|
end
|
90
81
|
|
@@ -96,22 +87,10 @@ module PEROBS
|
|
96
87
|
# We only support 64 bit keys, so nibble cannot be larger than 15.
|
97
88
|
PEROBS.log.fatal "Nibble must be within 0 - 15 but is #{nibble}"
|
98
89
|
end
|
99
|
-
#
|
100
|
-
#
|
101
|
-
|
102
|
-
|
103
|
-
# # We have an address and have found the node in the node cache.
|
104
|
-
# return node
|
105
|
-
#else
|
106
|
-
begin
|
107
|
-
# We don't have a IndexTreeNode object yet for this node. Create it
|
108
|
-
# with the data from the 'database_index' file.
|
109
|
-
node = IndexTreeNode.new(self, nibble, address)
|
110
|
-
# Add the node to the node cache if it's up to MAX_CACHED_LEVEL levels
|
111
|
-
# down from the root.
|
112
|
-
#@node_cache[address & mask] = node if nibble <= MAX_CACHED_LEVEL
|
113
|
-
return node
|
114
|
-
end
|
90
|
+
# We don't have a IndexTreeNode object yet for this node. Create it
|
91
|
+
# with the data from the 'database_index' file.
|
92
|
+
node = IndexTreeNode.new(self, nibble, address)
|
93
|
+
return node
|
115
94
|
end
|
116
95
|
|
117
96
|
# Delete a node from the tree that corresponds to the address.
|
@@ -122,10 +101,8 @@ module PEROBS
|
|
122
101
|
# We only support 64 bit keys, so nibble cannot be larger than 15.
|
123
102
|
PEROBS.log.fatal "Nibble must be within 0 - 15 but is #{nibble}"
|
124
103
|
end
|
125
|
-
|
126
|
-
|
127
|
-
#@node_cache.delete(address & mask)
|
128
|
-
# Then delete it from the 'database_index' file.
|
104
|
+
|
105
|
+
# Delete it from the 'database_index' file.
|
129
106
|
@nodes.delete_blob(address)
|
130
107
|
end
|
131
108
|
|
@@ -135,12 +112,6 @@ module PEROBS
|
|
135
112
|
# @param id [Integer] ID or key
|
136
113
|
# @param value [Integer] value to store
|
137
114
|
def put_value(id, value)
|
138
|
-
#MAX_CACHED_LEVEL.downto(0) do |i|
|
139
|
-
# mask = (2 ** ((1 + i) * 4)) - 1
|
140
|
-
# if (node = @node_cache[value & mask])
|
141
|
-
# return node.put_value(id, value)
|
142
|
-
# end
|
143
|
-
#end
|
144
115
|
@root.put_value(id, value)
|
145
116
|
end
|
146
117
|
|
data/lib/perobs/IndexTreeNode.rb
CHANGED
@@ -44,6 +44,8 @@ module PEROBS
|
|
44
44
|
ENTRY_BYTES = 8
|
45
45
|
TYPE_BYTES = 4
|
46
46
|
NODE_BYTES = TYPE_BYTES + ENTRIES * ENTRY_BYTES
|
47
|
+
# How many levels of the tree should be kept in memory.
|
48
|
+
CACHED_LEVELS = 4
|
47
49
|
|
48
50
|
# Create a new IndexTreeNode.
|
49
51
|
# @param tree [IndexTree] The tree this node belongs to
|
@@ -64,6 +66,9 @@ module PEROBS
|
|
64
66
|
@address = @tree.nodes.free_address
|
65
67
|
write_node
|
66
68
|
end
|
69
|
+
# These are the pointers that point to the next level of IndexTreeNode
|
70
|
+
# elements.
|
71
|
+
@node_ptrs = ::Array.new(ENTRIES, nil)
|
67
72
|
end
|
68
73
|
|
69
74
|
# Store a value for the given ID. Existing values will be overwritten.
|
@@ -96,6 +101,7 @@ module PEROBS
|
|
96
101
|
# The entry of the current node is now a reference to the new node.
|
97
102
|
set_entry_type(index, 2)
|
98
103
|
@entries[index] = node.address
|
104
|
+
@node_ptrs[index] = node if @nibble_idx < CACHED_LEVELS
|
99
105
|
# Store the existing value and the new value with their IDs.
|
100
106
|
node.set_entry(existing_id, existing_value)
|
101
107
|
node.put_value(id, value)
|
@@ -103,8 +109,7 @@ module PEROBS
|
|
103
109
|
write_node
|
104
110
|
when 2
|
105
111
|
# The entry is a reference to another node.
|
106
|
-
|
107
|
-
node.put_value(id, value)
|
112
|
+
get_node(index).put_value(id, value)
|
108
113
|
else
|
109
114
|
PEROBS.log.fatal "Illegal node type #{get_entry_type(index)}"
|
110
115
|
end
|
@@ -134,8 +139,7 @@ module PEROBS
|
|
134
139
|
when 2
|
135
140
|
# The entry is a reference to another node. Just follow it and look at
|
136
141
|
# the next nibble.
|
137
|
-
return
|
138
|
-
get_value(id)
|
142
|
+
return get_node(index).get_value(id)
|
139
143
|
else
|
140
144
|
PEROBS.log.fatal "Illegal node type #{get_entry_type(index)}"
|
141
145
|
end
|
@@ -156,6 +160,7 @@ module PEROBS
|
|
156
160
|
if id == stored_id
|
157
161
|
@tree.ids.delete_blob(@entries[index])
|
158
162
|
@entries[index] = 0
|
163
|
+
@node_ptrs[index] = nil
|
159
164
|
set_entry_type(index, 0)
|
160
165
|
write_node
|
161
166
|
return true
|
@@ -165,7 +170,7 @@ module PEROBS
|
|
165
170
|
end
|
166
171
|
when 2
|
167
172
|
# The entry is a reference to another node.
|
168
|
-
node =
|
173
|
+
node = get_node(index)
|
169
174
|
result = node.delete_value(id)
|
170
175
|
if node.empty?
|
171
176
|
# If the sub-node is empty after the delete we delete the whole
|
@@ -210,8 +215,7 @@ module PEROBS
|
|
210
215
|
when 2
|
211
216
|
# The entry is a reference to another node. Just follow it and look
|
212
217
|
# at the next nibble.
|
213
|
-
unless
|
214
|
-
check(flat_file, tree_level + 1)
|
218
|
+
unless get_node(index).check(flat_file, tree_level + 1)
|
215
219
|
return false
|
216
220
|
end
|
217
221
|
else
|
@@ -233,8 +237,7 @@ module PEROBS
|
|
233
237
|
id, address = get_id_and_address(@entries[i])
|
234
238
|
str += " #{id} => #{address},\n"
|
235
239
|
when 2
|
236
|
-
str += " " +
|
237
|
-
inspect.gsub(/\n/, "\n ")
|
240
|
+
str += " " + get_node(i).inspect.gsub(/\n/, "\n ")
|
238
241
|
end
|
239
242
|
end
|
240
243
|
str + "}\n"
|
@@ -262,6 +265,17 @@ module PEROBS
|
|
262
265
|
(id >> (4 * @nibble_idx)) & 0xF
|
263
266
|
end
|
264
267
|
|
268
|
+
def get_node(index)
|
269
|
+
unless (node = @node_ptrs[index])
|
270
|
+
node = @tree.get_node(@nibble_idx + 1, @entries[index])
|
271
|
+
# We only cache the first levels of the tree to limit the memory
|
272
|
+
# consumption.
|
273
|
+
@node_ptrs[index] = node if @nibble_idx < CACHED_LEVELS
|
274
|
+
end
|
275
|
+
|
276
|
+
node
|
277
|
+
end
|
278
|
+
|
265
279
|
def read_node
|
266
280
|
return false unless (bytes = @tree.nodes.retrieve_blob(@address))
|
267
281
|
@entry_types = bytes[0, TYPE_BYTES].unpack('L')[0]
|
data/lib/perobs/StackFile.rb
CHANGED
@@ -54,6 +54,9 @@ module PEROBS
|
|
54
54
|
rescue => e
|
55
55
|
PEROBS.log.fatal "Cannot open stack file #{@file_name}: #{e.message}"
|
56
56
|
end
|
57
|
+
unless @f.flock(File::LOCK_NB | File::LOCK_EX)
|
58
|
+
PEROBS.log.fatal 'Database stack file is locked by another process'
|
59
|
+
end
|
57
60
|
end
|
58
61
|
|
59
62
|
# Close the stack file. This method must be called before the program is
|
@@ -61,6 +64,7 @@ module PEROBS
|
|
61
64
|
def close
|
62
65
|
begin
|
63
66
|
@f.flush
|
67
|
+
@f.flock(File::LOCK_UN)
|
64
68
|
@f.close
|
65
69
|
rescue IOError => e
|
66
70
|
PEROBS.log.fatal "Cannot close stack file #{@file_name}: #{e.message}"
|
data/lib/perobs/version.rb
CHANGED
data/test/FlatFileDB_spec.rb
CHANGED
@@ -52,5 +52,10 @@ describe PEROBS::FlatFileDB do
|
|
52
52
|
expect(File.read(version_file).to_i).to eq(PEROBS::FlatFileDB::VERSION)
|
53
53
|
end
|
54
54
|
|
55
|
+
it 'should fail to open the same DB twice' do
|
56
|
+
db2 = PEROBS::FlatFileDB.new(@db_dir)
|
57
|
+
expect { db2.open }.to raise_error(PEROBS::FatalError)
|
58
|
+
end
|
59
|
+
|
55
60
|
end
|
56
61
|
|
data/test/Store_spec.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: perobs
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.4.
|
4
|
+
version: 2.4.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Chris Schlaeger
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-
|
11
|
+
date: 2017-03-14 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -144,4 +144,3 @@ test_files:
|
|
144
144
|
- test/Store_spec.rb
|
145
145
|
- test/perobs_spec.rb
|
146
146
|
- test/spec_helper.rb
|
147
|
-
has_rdoc:
|