perobs 3.0.1 → 3.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -34,40 +34,113 @@ module PEROBS
34
34
  # Simple cache that can hold up to size SpaceTreeNode entries. Entries are
35
35
  # hashed with a simple node_address % size function. This keeps the
36
36
  # overhead for managing the cache extremely low yet giving an OK
37
- # probability to have cache hits.
37
+ # probability to have cache hits. The cache also keeps track if a node is
38
+ # still in memory or needs to be reloaded from the file. All node accesses
39
+ # must always go through this cache to avoid having duplicate in-memory
40
+ # nodes for the same on-disk node.
38
41
  # @param size [Integer] maximum number of cache entries
39
- def initialize(size)
42
+ def initialize(tree, size)
43
+ @tree = tree
40
44
  @size = size
41
45
  clear
42
46
  end
43
47
 
44
- # Insert a node into the cache.
45
- # @param node [SpaceTreeNode]
46
- def insert(node)
47
- @entries[node.node_address % @size] = node
48
+ # Insert an unmodified node into the cache.
49
+ # @param node [SpaceTreeNode] Unmodified SpaceTreeNode
50
+ def insert_unmodified(node)
51
+ @in_memory_nodes[node.node_address] = node.object_id
52
+ @unmodified_nodess[node.node_address % @size] = node
53
+ end
54
+
55
+ # Insert a modified node into the cache.
56
+ # @param node [SpaceTreeNode] Modified SpaceTreeNode
57
+ # @param address [Integer] Address of the node in the file
58
+ def insert_modified(node)
59
+ @in_memory_nodes[node.node_address] = node.object_id
60
+ index = node.node_address % @size
61
+ if (old_node = @modified_nodes[index])
62
+ # If the object is already in the modified object list, we don't have
63
+ # to do anything.
64
+ return if old_node.node_address == node.node_address
65
+ # If the new object will replace an existing entry in the cash we have
66
+ # to save the object first.
67
+ old_node.save
68
+ end
69
+
70
+ @modified_nodes[index] = node
48
71
  end
49
72
 
50
73
  # Retrieve a node reference from the cache.
51
74
  # @param address [Integer] address of the node to retrieve.
52
75
  def get(address)
53
- node = @entries[address % @size]
76
+ node = @unmodified_nodess[address % @size]
54
77
  # We can have collisions. Check if the cached node really matches the
55
78
  # requested address.
56
- (node && node.node_address == address) ? node : nil
79
+ return node if node && node.node_address == address
80
+
81
+ node = @modified_nodes[address % @size]
82
+ # We can have collisions. Check if the cached node really matches the
83
+ # requested address.
84
+ return node if node && node.node_address == address
85
+
86
+ if (obj_id = @in_memory_nodes[address])
87
+ # We have the node in memory so we can just return it.
88
+ begin
89
+ node = ObjectSpace._id2ref(obj_id)
90
+ unless node.node_address == address
91
+ raise RuntimeError, "In memory list is corrupted"
92
+ end
93
+ insert_unmodified(node)
94
+ return node
95
+ rescue RangeError
96
+ # Due to a race condition the object can still be in the
97
+ # @in_memory_nodes list but has been collected already by the Ruby
98
+ # GC. In that case we need to load it again.
99
+ @in_memory_nodes.delete(address)
100
+ end
101
+ end
102
+
103
+ SpaceTreeNode::load(@tree, address)
57
104
  end
58
105
 
59
106
  # Remove a node from the cache.
60
107
  # @param address [Integer] address of node to remove.
61
108
  def delete(address)
109
+ # The object is likely still in memory, but we really don't want to
110
+ # access it anymore.
111
+ @in_memory_nodes.delete(address)
112
+
62
113
  index = address % @size
63
- if (node = @entries[index]) && node.node_address == address
64
- @entries[index] = nil
114
+ if (node = @unmodified_nodess[index]) && node.node_address == address
115
+ @unmodified_nodess[index] = nil
116
+ end
117
+
118
+ if (node = @modified_nodes[index]) && node.node_address == address
119
+ @modified_nodes[index] = nil
65
120
  end
66
121
  end
67
122
 
123
+ # Remove a node from the in-memory list. This is an internal method
124
+ # and should never be called from user code. It will be called from a
125
+ # finalizer, so many restrictions apply!
126
+ # @param node_address [Integer] Node address of the node to remove from
127
+ # the list
128
+ def _collect(address)
129
+ @in_memory_nodes.delete(address)
130
+ end
131
+
132
+ # Write all modified objects into the backing store.
133
+ def flush
134
+ @modified_nodes.each { |node| node.save if node }
135
+ @modified_nodes = ::Array.new(@size)
136
+ end
137
+
68
138
  # Remove all entries from the cache.
69
139
  def clear
70
- @entries = ::Array.new(@size)
140
+ # A hash that stores all objects by ID that are currently in memory.
141
+ @in_memory_nodes = {}
142
+ @unmodified_nodess = ::Array.new(@size)
143
+ @modified_nodes = ::Array.new(@size)
71
144
  end
72
145
 
73
146
  end
@@ -94,7 +94,7 @@ module PEROBS
94
94
  private
95
95
 
96
96
  def get_node
97
- @tree.get_node(@node_address)
97
+ @tree.cache.get(@node_address)
98
98
  end
99
99
 
100
100
  end
@@ -36,7 +36,7 @@ module PEROBS
36
36
  # Create a new stack file in the given directory with the given file name.
37
37
  # @param dir [String] Directory
38
38
  # @param name [String] File name
39
- # @param entry_bytes [Fixnum] Number of bytes each entry must have
39
+ # @param entry_bytes [Integer] Number of bytes each entry must have
40
40
  def initialize(dir, name, entry_bytes)
41
41
  @file_name = File.join(dir, name + '.stack')
42
42
  @entry_bytes = entry_bytes
data/lib/perobs/Store.rb CHANGED
@@ -26,7 +26,6 @@
26
26
  # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
27
27
 
28
28
  require 'set'
29
- require 'weakref'
30
29
 
31
30
  require 'perobs/Log'
32
31
  require 'perobs/Handle'
@@ -120,7 +119,7 @@ module PEROBS
120
119
  # :json : About half as fast as marshal, but the
121
120
  # format is rock solid and portable between
122
121
  # languages. It only supports basic Ruby data types
123
- # like String, Fixnum, Float, Array, Hash. This is
122
+ # like String, Integer, Float, Array, Hash. This is
124
123
  # the default option.
125
124
  # :yaml : Can also handle most Ruby data types and is
126
125
  # portable between Ruby versions (1.9 and later).
@@ -221,7 +220,7 @@ module PEROBS
221
220
  # For library internal use only!
222
221
  # This method will create a new PEROBS object.
223
222
  # @param klass [BasicObject] Class of the object to create
224
- # @param id [Fixnum, Bignum] Requested object ID
223
+ # @param id [Integer] Requested object ID
225
224
  # @param args [Array] Arguments to pass to the object constructor.
226
225
  # @return [BasicObject] Newly constructed PEROBS object
227
226
  def _construct_po(klass, id, *args)
@@ -297,7 +296,7 @@ module PEROBS
297
296
  # from the back-end storage. The garbage collector is not invoked
298
297
  # automatically. Depending on your usage pattern, you need to call this
299
298
  # method periodically.
300
- # @return [Fixnum] The number of collected objects
299
+ # @return [Integer] The number of collected objects
301
300
  def gc
302
301
  if @cache.in_transaction?
303
302
  PEROBS.log.fatal 'You cannot call gc() during a transaction'
@@ -314,11 +313,12 @@ module PEROBS
314
313
  if (obj = @in_memory_objects[id])
315
314
  # We have the object in memory so we can just return it.
316
315
  begin
317
- return obj.__getobj__
318
- rescue WeakRef::RefError
316
+ return ObjectSpace._id2ref(obj)
317
+ rescue RangeError
319
318
  # Due to a race condition the object can still be in the
320
319
  # @in_memory_objects list but has been collected already by the Ruby
321
320
  # GC. In that case we need to load it again.
321
+ @in_memory_objects.delete(id)
322
322
  end
323
323
  end
324
324
 
@@ -342,7 +342,7 @@ module PEROBS
342
342
  # unreadable object is found, the reference will simply be deleted.
343
343
  # @param repair [TrueClass/FalseClass] true if a repair attempt should be
344
344
  # made.
345
- # @return [Fixnum] The number of references to bad objects found.
345
+ # @return [Integer] The number of references to bad objects found.
346
346
  def check(repair = false)
347
347
  # All objects must have in-db version.
348
348
  sync
@@ -436,7 +436,7 @@ module PEROBS
436
436
  # Internal method. Don't use this outside of this library!
437
437
  # Generate a new unique ID that is not used by any other object. It uses
438
438
  # random numbers between 0 and 2**64 - 1.
439
- # @return [Fixnum or Bignum]
439
+ # @return [Integer]
440
440
  def _new_id
441
441
  begin
442
442
  # Generate a random number. It's recommended to not store more than
@@ -454,15 +454,15 @@ module PEROBS
454
454
  # happens the object finalizer is triggered and calls _forget() to
455
455
  # remove the object from this hash again.
456
456
  # @param obj [BasicObject] Object to register
457
- # @param id [Fixnum or Bignum] object ID
457
+ # @param id [Integer] object ID
458
458
  def _register_in_memory(obj, id)
459
- @in_memory_objects[id] = WeakRef.new(obj)
459
+ @in_memory_objects[id] = obj.object_id
460
460
  end
461
461
 
462
462
  # Remove the object from the in-memory list. This is an internal method
463
463
  # and should never be called from user code. It will be called from a
464
464
  # finalizer, so many restrictions apply!
465
- # @param id [Fixnum or Bignum] Object ID of object to remove from the list
465
+ # @param id [Integer] Object ID of object to remove from the list
466
466
  def _collect(id, ignore_errors = false)
467
467
  @in_memory_objects.delete(id)
468
468
  end
@@ -502,10 +502,10 @@ module PEROBS
502
502
 
503
503
  # Check the object with the given start_id and all other objects that are
504
504
  # somehow reachable from the start object.
505
- # @param start_id [Fixnum or Bignum] ID of the top-level object to start
505
+ # @param start_id [Integer] ID of the top-level object to start
506
506
  # with
507
507
  # @param repair [Boolean] Delete refernces to broken objects if true
508
- # @return [Fixnum] The number of references to bad objects.
508
+ # @return [Integer] The number of references to bad objects.
509
509
  def check_object(start_id, repair)
510
510
  errors = 0
511
511
  @db.mark(start_id)
@@ -518,7 +518,7 @@ module PEROBS
518
518
  # Get the next PEROBS object to check
519
519
  ref_obj, id = todo_list.pop
520
520
 
521
- if (obj = object_by_id(id)) && (obj_ok = @db.check(id, repair))
521
+ if (obj = object_by_id(id))
522
522
  # The object exists and is OK. Mark is as checked.
523
523
  @db.mark(id)
524
524
  # Now look at all other objects referenced by this object.
data/lib/perobs/TreeDB.rb CHANGED
@@ -103,7 +103,7 @@ module PEROBS
103
103
  end
104
104
 
105
105
  # Return true if the object with given ID exists
106
- # @param id [Fixnum or Bignum]
106
+ # @param id [Integer]
107
107
  def include?(id)
108
108
  !(blob = find_blob(id)).nil? && !blob.find(id).nil?
109
109
  end
@@ -143,7 +143,7 @@ module PEROBS
143
143
  end
144
144
 
145
145
  # Load the given object from the filesystem.
146
- # @param id [Fixnum or Bignum] object ID
146
+ # @param id [Integer] object ID
147
147
  # @return [Hash] Object as defined by PEROBS::ObjectBase or nil if ID does
148
148
  # not exist
149
149
  def get_object(id)
@@ -166,13 +166,13 @@ module PEROBS
166
166
  end
167
167
 
168
168
  # Mark an object.
169
- # @param id [Fixnum or Bignum] ID of the object to mark
169
+ # @param id [Integer] ID of the object to mark
170
170
  def mark(id)
171
171
  (blob = find_blob(id)) && blob.mark(id)
172
172
  end
173
173
 
174
174
  # Check if the object is marked.
175
- # @param id [Fixnum or Bignum] ID of the object to check
175
+ # @param id [Integer] ID of the object to check
176
176
  # @param ignore_errors [Boolean] If set to true no errors will be raised
177
177
  # for non-existing objects.
178
178
  def is_marked?(id, ignore_errors = false)
@@ -187,7 +187,7 @@ module PEROBS
187
187
  end
188
188
 
189
189
  # Check if the stored object is syntactically correct.
190
- # @param id [Fixnum/Bignum] Object ID
190
+ # @param id [Integer] Object ID
191
191
  # @param repair [TrueClass/FalseClass] True if an repair attempt should be
192
192
  # made.
193
193
  # @return [TrueClass/FalseClass] True if the object is OK, otherwise
@@ -206,7 +206,7 @@ module PEROBS
206
206
  # Store the given serialized object into the cluster files. This method is
207
207
  # for internal use only!
208
208
  # @param raw [String] Serialized Object as defined by PEROBS::ObjectBase
209
- # @param id [Fixnum or Bignum] Object ID
209
+ # @param id [Integer] Object ID
210
210
  def put_raw_object(raw, id)
211
211
  find_blob(id, true).write_object(id, raw)
212
212
  end
@@ -226,7 +226,7 @@ module PEROBS
226
226
  Dir.glob(File.join(dir_name, '*.index')).each do |fqfn|
227
227
  # Extract the 01-part of the filename
228
228
  lsb_string = File.basename(fqfn)[0..-6]
229
- # Convert the lsb_string into a Fixnum
229
+ # Convert the lsb_string into a Integer
230
230
  lsb = Integer('0b' + lsb_string)
231
231
  # Bit mask to match the LSBs
232
232
  mask = (2 ** lsb_string.length) - 1
@@ -1,4 +1,4 @@
1
1
  module PEROBS
2
2
  # The version number
3
- VERSION = "3.0.1"
3
+ VERSION = "3.0.2"
4
4
  end
data/test/Array_spec.rb CHANGED
@@ -169,8 +169,8 @@ describe PEROBS::Array do
169
169
 
170
170
  it 'should support collect!()' do
171
171
  a = cpa([ 1, 'cat', 1..1 ])
172
- expect(a.collect! { |e| e.class }).to eq([ Fixnum, String, Range ])
173
- pcheck { expect(a).to eq([ Fixnum, String, Range ]) }
172
+ expect(a.collect! { |e| e.class }).to eq([ Integer, String, Range ])
173
+ pcheck { expect(a).to eq([ Integer, String, Range ]) }
174
174
 
175
175
  a = cpa([ 1, 'cat', 1..1 ])
176
176
  expect(a.collect! { 99 }).to eq([ 99, 99, 99])
@@ -179,8 +179,8 @@ describe PEROBS::Array do
179
179
 
180
180
  it 'should support map!()' do
181
181
  a = cpa([ 1, 'cat', 1..1 ])
182
- expect(a.map! { |e| e.class }).to eq([ Fixnum, String, Range ])
183
- pcheck { expect(a).to eq([ Fixnum, String, Range ]) }
182
+ expect(a.map! { |e| e.class }).to eq([ Integer, String, Range ])
183
+ pcheck { expect(a).to eq([ Integer, String, Range ]) }
184
184
 
185
185
  a = cpa([ 1, 'cat', 1..1 ])
186
186
  expect(a.map! { 99 }).to eq([ 99, 99, 99])
data/test/BTreeDB_spec.rb CHANGED
@@ -56,7 +56,7 @@ describe PEROBS::BTreeDB do
56
56
  expect(@db.include?(0)).to be false
57
57
  h = {
58
58
  'String' => 'What god has wrought',
59
- 'Fixnum' => 42,
59
+ 'Integer' => 42,
60
60
  'Float' => 3.14,
61
61
  'True' => true,
62
62
  'False' => false,
@@ -75,7 +75,7 @@ describe PEROBS::BTreeDB do
75
75
  expect(@db.include?(0)).to be false
76
76
  h = {
77
77
  'String' => 'What god has wrought',
78
- 'Fixnum' => 42,
78
+ 'Integer' => 42,
79
79
  'Float' => 3.14,
80
80
  'True' => true,
81
81
  'False' => false,
@@ -94,12 +94,12 @@ EOT
94
94
  end
95
95
 
96
96
  it 'should find the smallest node' do
97
- node = @m.instance_variable_get('@root').find_smallest_node
97
+ node = @m.root.find_smallest_node
98
98
  expect(node.size).to eql(2)
99
99
  end
100
100
 
101
101
  it 'should find the largest node' do
102
- node = @m.instance_variable_get('@root').find_largest_node
102
+ node = @m.root.find_largest_node
103
103
  expect(node.size).to eql(32)
104
104
  expect(node.node_address).to eql(5)
105
105
  expect(node.parent.size).to eql(16)
@@ -119,12 +119,14 @@ EOT
119
119
  add_sizes([ 10, 5, 15, 10 ])
120
120
  expect(@m.to_a).to eql([[0, 10], [1, 5], [3, 10], [2, 15]])
121
121
  expect(@m.get_space(10)).to eql([0, 10])
122
+ expect(@m.to_a).to eql([[3, 10], [1, 5], [2, 15]])
122
123
 
123
124
  @m.clear
124
125
  add_sizes([ 10, 5, 15, 10, 10 ])
125
126
  expect(@m.to_a).to eql([[0, 10], [1, 5], [4, 10], [3, 10], [2, 15]])
126
127
  expect(@m.get_space(10)).to eql([0, 10])
127
128
  expect(@m.get_space(10)).to eql([4, 10])
129
+ expect(@m.to_a).to eql([[3, 10], [1, 5], [2, 15]])
128
130
  end
129
131
 
130
132
  it 'should delete a smaller node' do
@@ -208,14 +210,15 @@ EOT
208
210
  it 'should support a real-world traffic pattern' do
209
211
  address = 0
210
212
  spaces = []
211
- 0.upto(500) do
212
- case rand(3)
213
+ @m.clear
214
+ 0.upto(1000) do
215
+ case rand(4)
213
216
  when 0
214
217
  # Insert new values
215
218
  rand(9).times do
216
219
  size = 20 + rand(5000)
217
220
  @m.add_space(address, size)
218
- spaces << [address, size]
221
+ spaces << [ address, size ]
219
222
  address += size
220
223
  end
221
224
  when 1
@@ -228,9 +231,21 @@ EOT
228
231
  end
229
232
  end
230
233
  when 2
231
- expect(@m.check).to be true
232
- spaces.each do |address, size|
233
- expect(@m.has_space?(address, size)).to be true
234
+ if rand(10) == 0
235
+ expect(@m.check).to be true
236
+ spaces.each do |address, size|
237
+ expect(@m.has_space?(address, size)).to be true
238
+ end
239
+ @m.each do |address, size|
240
+ expect(spaces.include?([ address, size ])).to be true
241
+ end
242
+ end
243
+ when 3
244
+ if rand(100) == 0
245
+ expect(@m.check).to be true
246
+ @m.close
247
+ @m.open
248
+ expect(@m.check).to be true
234
249
  end
235
250
  end
236
251
  end
data/test/Store_spec.rb CHANGED
@@ -422,7 +422,9 @@ describe PEROBS::Store do
422
422
  @store.sync
423
423
  GC.start
424
424
  # Now the Person should be gone from memory.
425
- expect(@store.statistics[:in_memory_objects]).to eq(1)
425
+ # Ruby 2.3 and later has changed the GC so that this does not work
426
+ # reliably anymore. The GC seems to operate lazyly.
427
+ #expect(@store.statistics[:in_memory_objects]).to eq(1)
426
428
  end
427
429
 
428
430
  it 'should handle nested constructors' do
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: perobs
3
3
  version: !ruby/object:Gem::Version
4
- version: 3.0.1
4
+ version: 3.0.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - Chris Schlaeger
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2017-08-13 00:00:00.000000000 Z
11
+ date: 2017-10-07 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler