perobs 2.4.1 → 2.4.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 3580a4c1778350e195656c42ca100c651ec18386
4
- data.tar.gz: e3c964bb3f135372dd07ffc7b36c108f27990569
3
+ metadata.gz: 7913287f5998dfad64a99f239723ecb905798fe9
4
+ data.tar.gz: d667fe5eab1c96b4003640c4d924d97393fdf4b3
5
5
  SHA512:
6
- metadata.gz: 6c12c89af79a909c65346fe8ea35e2de608dbba0ce4eb85be35b56bd07a5c50858cd5773ab94ee8e6dd3c8b75c1a872ac7400a0bbc78927c715954c337a7288e
7
- data.tar.gz: f93ee4dfb7552724b78f14e682a0750e53c7984910c588050e9dc92b1d2571c35ea8ff7a5fb095ef6b537a238d15da4bb7d31a6126d1ec7aeea1d89d19138dda
6
+ metadata.gz: 0140b0b11d4f6f7c5f01743d5e361cf045c4533ab339ee92e32e72b4ddca7046684a6ac00445667c0a08d953946cc0fd93d6563a9b3482759e436151d0a548df
7
+ data.tar.gz: 5b6b9101321994f98f42c153c788bfece79183a074baf01d1e2ed377a0ff929b3308947b272bca1892807e425abfd6d2decd208f07179151fbca703d76eaa8ac
@@ -300,6 +300,9 @@ module PEROBS
300
300
  if File.exist?(@index_file_name)
301
301
  begin
302
302
  File.open(@index_file_name, 'rb') do |f|
303
+ unless f.flock(File::LOCK_NB | File::LOCK_EX)
304
+ PEROBS.log.fatal 'BTreeDB Database is locked by another process'
305
+ end
303
306
  # Since version 2.3.0, all index files start with a header.
304
307
  # Earlier versions did not yet have this header. The header is 24
305
308
  # bytes long. The 2nd set of 8 bytes must be 0 to distinguish the
@@ -347,6 +350,7 @@ module PEROBS
347
350
  @entries << e
348
351
  @entries_by_id[e[ID]] = e
349
352
  end
353
+ f.flock(File::LOCK_UN)
350
354
  end
351
355
  rescue => e
352
356
  PEROBS.log.fatal "BTreeBlob file #{@index_file_name} corrupted: " +
@@ -358,11 +362,15 @@ module PEROBS
358
362
  def write_index
359
363
  begin
360
364
  File.open(@index_file_name, 'wb') do |f|
365
+ unless f.flock(File::LOCK_NB | File::LOCK_EX)
366
+ PEROBS.log.fatal 'BTreeDB Database is locked by another process'
367
+ end
361
368
  # See read_index for data format documentation.
362
369
  f.write([ PEROBS_MAGIC, 0, 1].pack('QQQ'))
363
370
  @entries.each do |entry|
364
371
  f.write(entry.pack('QQQCL'))
365
372
  end
373
+ f.flock(File::LOCK_UN)
366
374
  end
367
375
  rescue => e
368
376
  PEROBS.log.fatal "Cannot write BTreeBlob index file " +
data/lib/perobs/Cache.rb CHANGED
@@ -124,7 +124,7 @@ module PEROBS
124
124
  end
125
125
 
126
126
  # Tell the cache to start a new transaction. If no other transaction is
127
- # active, the write cached is flushed before the transaction is started.
127
+ # active, the write cache is flushed before the transaction is started.
128
128
  def begin_transaction
129
129
  if @transaction_stack.empty?
130
130
  # The new transaction is the top-level transaction. Flush the write
@@ -59,6 +59,9 @@ module PEROBS
59
59
  rescue IOError => e
60
60
  PEROBS.log.fatal "Cannot open blob file #{@file_name}: #{e.message}"
61
61
  end
62
+ unless @f.flock(File::LOCK_NB | File::LOCK_EX)
63
+ PEROBS.log.fatal 'Database blob file is locked by another process'
64
+ end
62
65
  @free_list.open
63
66
  end
64
67
 
@@ -68,6 +71,7 @@ module PEROBS
68
71
  @free_list.close
69
72
  begin
70
73
  @f.flush
74
+ @f.flock(File::LOCK_UN)
71
75
  @f.close
72
76
  rescue IOError => e
73
77
  PEROBS.log.fatal "Cannot close blob file #{@file_name}: #{e.message}"
@@ -81,6 +81,9 @@ module PEROBS
81
81
  PEROBS.log.fatal "Cannot open flat file database #{file_name}: " +
82
82
  e.message
83
83
  end
84
+ unless @f.flock(File::LOCK_NB | File::LOCK_EX)
85
+ PEROBS.log.fatal 'Database is locked by another process'
86
+ end
84
87
  @index.open
85
88
  @space_list.open
86
89
  end
@@ -91,6 +94,7 @@ module PEROBS
91
94
  @space_list.close
92
95
  @index.close
93
96
  @f.flush
97
+ @f.flock(File::LOCK_UN)
94
98
  @f.close
95
99
  @f = nil
96
100
  end
@@ -287,7 +291,8 @@ module PEROBS
287
291
 
288
292
  each_blob_header do |pos, mark, length, blob_id, crc|
289
293
  total_blob_count += 1
290
- if (mark & 1 == 1)
294
+ if (mark & 3 == 3)
295
+ # Clear all valid and marked blocks.
291
296
  marked_blob_count += 1
292
297
  begin
293
298
  @f.seek(pos)
@@ -307,14 +312,17 @@ module PEROBS
307
312
  # implementation. No additional space will be needed on the file system.
308
313
  def defragmentize
309
314
  distance = 0
315
+ deleted_blobs = 0
316
+ valid_blobs = 0
310
317
  t = Time.now
311
- PEROBS.log.debug "Defragmenting FlatFile"
318
+ PEROBS.log.info "Defragmenting FlatFile"
312
319
  # Iterate over all entries.
313
320
  each_blob_header do |pos, mark, length, blob_id, crc|
314
321
  # Total size of the current entry
315
322
  entry_bytes = BLOB_HEADER_LENGTH + length
316
323
  if (mark & 1 == 1)
317
324
  # We have found a valid entry.
325
+ valid_blobs += 1
318
326
  if distance > 0
319
327
  begin
320
328
  # Read current entry into a buffer
@@ -336,11 +344,13 @@ module PEROBS
336
344
  end
337
345
  end
338
346
  else
347
+ deleted_blobs += 1
339
348
  distance += entry_bytes
340
349
  end
341
350
  end
342
- PEROBS.log.debug "FlatFile defragmented in #{Time.now - t} seconds"
343
- PEROBS.log.debug "#{distance} bytes or " +
351
+ PEROBS.log.info "FlatFile defragmented in #{Time.now - t} seconds"
352
+ PEROBS.log.info "#{distance / 1000} KiB/#{deleted_blobs} blobs of " +
353
+ "#{@f.size / 1000} KiB/#{valid_blobs} blobs or " +
344
354
  "#{'%.1f' % (distance.to_f / @f.size * 100.0)}% reclaimed"
345
355
 
346
356
  @f.flush
@@ -35,11 +35,6 @@ module PEROBS
35
35
  # search in the tree is much faster than the linear search in the FlatFile.
36
36
  class IndexTree
37
37
 
38
- # Determines how many levels of the IndexTree will be kept in memory to
39
- # accerlerate the access. A number of 7 will keep up to 21845 entries in
40
- # the cache but will accelerate the access to the FlatFile address.
41
- MAX_CACHED_LEVEL = 7
42
-
43
38
  attr_reader :nodes, :ids
44
39
 
45
40
  def initialize(db_dir)
@@ -55,10 +50,6 @@ module PEROBS
55
50
  # file which contains the full object ID and the address of the
56
51
  # corresponding object in the FlatFile.
57
52
  @ids = FixedSizeBlobFile.new(db_dir, 'object_id_index', 2 * 8)
58
-
59
- # The first MAX_CACHED_LEVEL levels of nodes will be cached in memory to
60
- # improve access times.
61
- @node_cache = {}
62
53
  end
63
54
 
64
55
  # Open the tree files.
@@ -72,6 +63,7 @@ module PEROBS
72
63
  def close
73
64
  @ids.close
74
65
  @nodes.close
66
+ @root = nil
75
67
  end
76
68
 
77
69
  # Flush out all unwritten data
@@ -84,7 +76,6 @@ module PEROBS
84
76
  def clear
85
77
  @nodes.clear
86
78
  @ids.clear
87
- @node_cache = {}
88
79
  @root = IndexTreeNode.new(self, 0, 0)
89
80
  end
90
81
 
@@ -96,22 +87,10 @@ module PEROBS
96
87
  # We only support 64 bit keys, so nibble cannot be larger than 15.
97
88
  PEROBS.log.fatal "Nibble must be within 0 - 15 but is #{nibble}"
98
89
  end
99
- # Generate a mask for the least significant bits up to and including the
100
- # nibble.
101
- mask = (2 ** ((1 + nibble) * 4)) - 1
102
- #if address && (node = @node_cache[address & mask])
103
- # # We have an address and have found the node in the node cache.
104
- # return node
105
- #else
106
- begin
107
- # We don't have a IndexTreeNode object yet for this node. Create it
108
- # with the data from the 'database_index' file.
109
- node = IndexTreeNode.new(self, nibble, address)
110
- # Add the node to the node cache if it's up to MAX_CACHED_LEVEL levels
111
- # down from the root.
112
- #@node_cache[address & mask] = node if nibble <= MAX_CACHED_LEVEL
113
- return node
114
- end
90
+ # We don't have a IndexTreeNode object yet for this node. Create it
91
+ # with the data from the 'database_index' file.
92
+ node = IndexTreeNode.new(self, nibble, address)
93
+ return node
115
94
  end
116
95
 
117
96
  # Delete a node from the tree that corresponds to the address.
@@ -122,10 +101,8 @@ module PEROBS
122
101
  # We only support 64 bit keys, so nibble cannot be larger than 15.
123
102
  PEROBS.log.fatal "Nibble must be within 0 - 15 but is #{nibble}"
124
103
  end
125
- # First delete the node from the node cache.
126
- mask = (2 ** ((1 + nibble) * 4)) - 1
127
- #@node_cache.delete(address & mask)
128
- # Then delete it from the 'database_index' file.
104
+
105
+ # Delete it from the 'database_index' file.
129
106
  @nodes.delete_blob(address)
130
107
  end
131
108
 
@@ -135,12 +112,6 @@ module PEROBS
135
112
  # @param id [Integer] ID or key
136
113
  # @param value [Integer] value to store
137
114
  def put_value(id, value)
138
- #MAX_CACHED_LEVEL.downto(0) do |i|
139
- # mask = (2 ** ((1 + i) * 4)) - 1
140
- # if (node = @node_cache[value & mask])
141
- # return node.put_value(id, value)
142
- # end
143
- #end
144
115
  @root.put_value(id, value)
145
116
  end
146
117
 
@@ -44,6 +44,8 @@ module PEROBS
44
44
  ENTRY_BYTES = 8
45
45
  TYPE_BYTES = 4
46
46
  NODE_BYTES = TYPE_BYTES + ENTRIES * ENTRY_BYTES
47
+ # How many levels of the tree should be kept in memory.
48
+ CACHED_LEVELS = 4
47
49
 
48
50
  # Create a new IndexTreeNode.
49
51
  # @param tree [IndexTree] The tree this node belongs to
@@ -64,6 +66,9 @@ module PEROBS
64
66
  @address = @tree.nodes.free_address
65
67
  write_node
66
68
  end
69
+ # These are the pointers that point to the next level of IndexTreeNode
70
+ # elements.
71
+ @node_ptrs = ::Array.new(ENTRIES, nil)
67
72
  end
68
73
 
69
74
  # Store a value for the given ID. Existing values will be overwritten.
@@ -96,6 +101,7 @@ module PEROBS
96
101
  # The entry of the current node is now a reference to the new node.
97
102
  set_entry_type(index, 2)
98
103
  @entries[index] = node.address
104
+ @node_ptrs[index] = node if @nibble_idx < CACHED_LEVELS
99
105
  # Store the existing value and the new value with their IDs.
100
106
  node.set_entry(existing_id, existing_value)
101
107
  node.put_value(id, value)
@@ -103,8 +109,7 @@ module PEROBS
103
109
  write_node
104
110
  when 2
105
111
  # The entry is a reference to another node.
106
- node = @tree.get_node(@nibble_idx + 1, @entries[index])
107
- node.put_value(id, value)
112
+ get_node(index).put_value(id, value)
108
113
  else
109
114
  PEROBS.log.fatal "Illegal node type #{get_entry_type(index)}"
110
115
  end
@@ -134,8 +139,7 @@ module PEROBS
134
139
  when 2
135
140
  # The entry is a reference to another node. Just follow it and look at
136
141
  # the next nibble.
137
- return @tree.get_node(@nibble_idx + 1, @entries[index]).
138
- get_value(id)
142
+ return get_node(index).get_value(id)
139
143
  else
140
144
  PEROBS.log.fatal "Illegal node type #{get_entry_type(index)}"
141
145
  end
@@ -156,6 +160,7 @@ module PEROBS
156
160
  if id == stored_id
157
161
  @tree.ids.delete_blob(@entries[index])
158
162
  @entries[index] = 0
163
+ @node_ptrs[index] = nil
159
164
  set_entry_type(index, 0)
160
165
  write_node
161
166
  return true
@@ -165,7 +170,7 @@ module PEROBS
165
170
  end
166
171
  when 2
167
172
  # The entry is a reference to another node.
168
- node = @tree.get_node(@nibble_idx + 1, @entries[index])
173
+ node = get_node(index)
169
174
  result = node.delete_value(id)
170
175
  if node.empty?
171
176
  # If the sub-node is empty after the delete we delete the whole
@@ -210,8 +215,7 @@ module PEROBS
210
215
  when 2
211
216
  # The entry is a reference to another node. Just follow it and look
212
217
  # at the next nibble.
213
- unless @tree.get_node(@nibble_idx + 1, @entries[index]).
214
- check(flat_file, tree_level + 1)
218
+ unless get_node(index).check(flat_file, tree_level + 1)
215
219
  return false
216
220
  end
217
221
  else
@@ -233,8 +237,7 @@ module PEROBS
233
237
  id, address = get_id_and_address(@entries[i])
234
238
  str += " #{id} => #{address},\n"
235
239
  when 2
236
- str += " " + @tree.get_node(@nibble_idx + 1, @entries[i]).
237
- inspect.gsub(/\n/, "\n ")
240
+ str += " " + get_node(i).inspect.gsub(/\n/, "\n ")
238
241
  end
239
242
  end
240
243
  str + "}\n"
@@ -262,6 +265,17 @@ module PEROBS
262
265
  (id >> (4 * @nibble_idx)) & 0xF
263
266
  end
264
267
 
268
+ def get_node(index)
269
+ unless (node = @node_ptrs[index])
270
+ node = @tree.get_node(@nibble_idx + 1, @entries[index])
271
+ # We only cache the first levels of the tree to limit the memory
272
+ # consumption.
273
+ @node_ptrs[index] = node if @nibble_idx < CACHED_LEVELS
274
+ end
275
+
276
+ node
277
+ end
278
+
265
279
  def read_node
266
280
  return false unless (bytes = @tree.nodes.retrieve_blob(@address))
267
281
  @entry_types = bytes[0, TYPE_BYTES].unpack('L')[0]
@@ -54,6 +54,9 @@ module PEROBS
54
54
  rescue => e
55
55
  PEROBS.log.fatal "Cannot open stack file #{@file_name}: #{e.message}"
56
56
  end
57
+ unless @f.flock(File::LOCK_NB | File::LOCK_EX)
58
+ PEROBS.log.fatal 'Database stack file is locked by another process'
59
+ end
57
60
  end
58
61
 
59
62
  # Close the stack file. This method must be called before the program is
@@ -61,6 +64,7 @@ module PEROBS
61
64
  def close
62
65
  begin
63
66
  @f.flush
67
+ @f.flock(File::LOCK_UN)
64
68
  @f.close
65
69
  rescue IOError => e
66
70
  PEROBS.log.fatal "Cannot close stack file #{@file_name}: #{e.message}"
@@ -1,4 +1,4 @@
1
1
  module PEROBS
2
2
  # The version number
3
- VERSION = "2.4.1"
3
+ VERSION = "2.4.2"
4
4
  end
@@ -52,5 +52,10 @@ describe PEROBS::FlatFileDB do
52
52
  expect(File.read(version_file).to_i).to eq(PEROBS::FlatFileDB::VERSION)
53
53
  end
54
54
 
55
+ it 'should fail to open the same DB twice' do
56
+ db2 = PEROBS::FlatFileDB.new(@db_dir)
57
+ expect { db2.open }.to raise_error(PEROBS::FatalError)
58
+ end
59
+
55
60
  end
56
61
 
data/test/Store_spec.rb CHANGED
@@ -479,7 +479,7 @@ describe PEROBS::Store do
479
479
  when 4
480
480
  # Sync store and reload
481
481
  if rand(15) == 0
482
- @store.sync
482
+ @store.exit
483
483
  @store = PEROBS::Store.new(@db_file, options)
484
484
  end
485
485
  when 5
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: perobs
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.4.1
4
+ version: 2.4.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - Chris Schlaeger
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2017-02-26 00:00:00.000000000 Z
11
+ date: 2017-03-14 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -144,4 +144,3 @@ test_files:
144
144
  - test/Store_spec.rb
145
145
  - test/perobs_spec.rb
146
146
  - test/spec_helper.rb
147
- has_rdoc: