perobs 3.0.2 → 4.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -31,11 +31,12 @@ module PEROBS
31
31
 
32
32
  class BTreeNodeCache
33
33
 
34
- def initialize
34
+ def initialize(tree)
35
+ @tree = tree
35
36
  clear
36
37
  end
37
38
 
38
- def [](address)
39
+ def get(address)
39
40
  if (node = @modified_nodes[address])
40
41
  return node
41
42
  end
@@ -48,7 +49,7 @@ module PEROBS
48
49
  return node
49
50
  end
50
51
 
51
- nil
52
+ BTreeNode::load(@tree, address)
52
53
  end
53
54
 
54
55
  def set_root(node)
@@ -58,12 +59,15 @@ module PEROBS
58
59
  @top_nodes[node.node_address] = node
59
60
  end
60
61
 
61
- def insert(node)
62
+ def insert(node, modified = true)
62
63
  unless node
63
64
  PEROBS.log.fatal "nil cannot be cached"
64
65
  end
65
66
  node = node.get_node if node.is_a?(BTreeNodeLink)
66
67
 
68
+ if modified
69
+ @modified_nodes[node.node_address] = node
70
+ end
67
71
  @ephemeral_nodes[node.node_address] = node
68
72
 
69
73
  if !@top_nodes.include?(node) && node.is_top?
@@ -71,10 +75,8 @@ module PEROBS
71
75
  end
72
76
  end
73
77
 
74
- def mark_as_modified(node)
75
- node = node.get_node if node.is_a?(BTreeNodeLink)
76
- @modified_nodes[node.node_address] = node
77
- insert(node)
78
+ def _collect(address, ruby_object_id)
79
+ # Just a dummy for now
78
80
  end
79
81
 
80
82
  # Remove a node from the cache.
@@ -132,7 +132,7 @@ module PEROBS
132
132
  end
133
133
 
134
134
  def get_node
135
- @tree.get_node(@node_address)
135
+ @tree.node_cache.get(@node_address)
136
136
  end
137
137
 
138
138
  end
data/lib/perobs/Cache.rb CHANGED
@@ -95,20 +95,20 @@ module PEROBS
95
95
 
96
96
  # Return the PEROBS::Object with the specified ID or nil if not found.
97
97
  # @param id [Integer] ID of the cached PEROBS::ObjectBase
98
- #def object_by_id(id)
99
- # idx = id & @mask
100
- # # The index is just a hash. We still need to check if the object IDs are
101
- # # actually the same before we can return the object.
102
- # if (obj = @writes[idx]) && obj._id == id
103
- # # The object was in the write cache.
104
- # return obj
105
- # elsif (obj = @reads[idx]) && obj._id == id
106
- # # The object was in the read cache.
107
- # return obj
108
- # end
109
-
110
- # nil
111
- #end
98
+ def object_by_id(id)
99
+ idx = id & @mask
100
+ # The index is just a hash. We still need to check if the object IDs are
101
+ # actually the same before we can return the object.
102
+ if (obj = @writes[idx]) && obj._id == id
103
+ # The object was in the write cache.
104
+ return obj
105
+ elsif (obj = @reads[idx]) && obj._id == id
106
+ # The object was in the read cache.
107
+ return obj
108
+ end
109
+
110
+ nil
111
+ end
112
112
 
113
113
  # Flush all pending writes to the persistant storage back-end.
114
114
  def flush
@@ -25,7 +25,7 @@
25
25
  # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
26
26
  # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
27
27
 
28
- require 'aws-sdk-core'
28
+ require 'aws-sdk'
29
29
 
30
30
  require 'perobs/DataBase'
31
31
  require 'perobs/BTreeBlob'
@@ -83,6 +83,7 @@ module PEROBS
83
83
  unless @f.flock(File::LOCK_NB | File::LOCK_EX)
84
84
  PEROBS.log.fatal 'Database blob file is locked by another process'
85
85
  end
86
+ @f.sync = true
86
87
  end
87
88
 
88
89
  # Close the blob file. This method must be called before the program is
@@ -92,6 +93,7 @@ module PEROBS
92
93
  if @f
93
94
  @f.flush
94
95
  @f.flock(File::LOCK_UN)
96
+ @f.fsync
95
97
  @f.close
96
98
  @f = nil
97
99
  end
@@ -103,7 +105,7 @@ module PEROBS
103
105
  # Erase the backing store. This method should only be called when the file
104
106
  # is not currently open.
105
107
  def erase
106
- PEROBS.log.fatal 'Cannot call EquiBlobsFile::erase while it is open' if @f
108
+ @f = nil
107
109
  File.delete(@file_name) if File.exist?(@file_name)
108
110
  reset_counters
109
111
  end
@@ -111,7 +113,10 @@ module PEROBS
111
113
  # Flush out all unwritten data.
112
114
  def sync
113
115
  begin
114
- @f.flush if @f
116
+ if @f
117
+ @f.flush
118
+ @f.fsync
119
+ end
115
120
  rescue IOError => e
116
121
  PEROBS.log.fatal "Cannot sync blob file #{@file_name}: #{e.message}"
117
122
  end
@@ -48,6 +48,7 @@ module PEROBS
48
48
  @db_dir = dir
49
49
  @f = nil
50
50
  @index = BTree.new(@db_dir, 'index', INDEX_BTREE_ORDER)
51
+ @marks = BTree.new(@db_dir, 'marks', INDEX_BTREE_ORDER)
51
52
  @space_list = SpaceTree.new(@db_dir)
52
53
  end
53
54
 
@@ -71,6 +72,7 @@ module PEROBS
71
72
  PEROBS.log.fatal "FlatFile database '#{file_name}' is locked by " +
72
73
  "another process"
73
74
  end
75
+ @f.sync = true
74
76
 
75
77
  begin
76
78
  @index.open(!new_db_created)
@@ -103,6 +105,7 @@ module PEROBS
103
105
  if @f
104
106
  @f.flush
105
107
  @f.flock(File::LOCK_UN)
108
+ @f.fsync
106
109
  @f.close
107
110
  @f = nil
108
111
  end
@@ -112,6 +115,7 @@ module PEROBS
112
115
  def sync
113
116
  begin
114
117
  @f.flush
118
+ @f.fsync
115
119
  rescue IOError => e
116
120
  PEROBS.log.fatal "Cannot sync flat file database: #{e.message}"
117
121
  end
@@ -148,7 +152,7 @@ module PEROBS
148
152
 
149
153
  deleted_ids = []
150
154
  each_blob_header do |pos, header|
151
- if header.is_valid? && !header.is_marked?
155
+ if header.is_valid? && @marks.get(header.id).nil?
152
156
  delete_obj_by_address(pos, header.id)
153
157
  deleted_ids << header.id
154
158
  end
@@ -184,12 +188,12 @@ module PEROBS
184
188
  # performance impact of compression is not compensated by writing
185
189
  # less data to the storage.
186
190
  compressed = false
187
- if raw_obj.length > 256
191
+ if raw_obj.bytesize > 256
188
192
  raw_obj = Zlib.deflate(raw_obj)
189
193
  compressed = true
190
194
  end
191
195
 
192
- addr, length = find_free_blob(raw_obj.length)
196
+ addr, length = find_free_blob(raw_obj.bytesize)
193
197
  begin
194
198
  if length != -1
195
199
  # Just a safeguard so we don't overwrite current data.
@@ -198,8 +202,8 @@ module PEROBS
198
202
  PEROBS.log.fatal "Length in free list (#{length}) and header " +
199
203
  "(#{header.length}) for address #{addr} don't match."
200
204
  end
201
- if raw_obj.length > header.length
202
- PEROBS.log.fatal "Object (#{raw_obj.length}) is longer than " +
205
+ if raw_obj.bytesize > header.length
206
+ PEROBS.log.fatal "Object (#{raw_obj.bytesize}) is longer than " +
203
207
  "blob space (#{header.length})."
204
208
  end
205
209
  if header.is_valid?
@@ -209,25 +213,19 @@ module PEROBS
209
213
  end
210
214
  flags = 1 << FlatFileBlobHeader::VALID_FLAG_BIT
211
215
  flags |= (1 << FlatFileBlobHeader::COMPRESSED_FLAG_BIT) if compressed
212
- if old_addr && old_header.is_marked?
213
- # This method might be called in the middle of an operation that
214
- # uses the mark flag. We must ensure that the flag is carried over
215
- # to the new header.
216
- flags |= (1 << FlatFileBlobHeader::MARK_FLAG_BIT)
217
- end
218
- FlatFileBlobHeader.new(@f, addr, flags, raw_obj.length, id, crc).write
216
+ FlatFileBlobHeader.new(@f, addr, flags, raw_obj.bytesize, id, crc).write
219
217
  @f.write(raw_obj)
220
- if length != -1 && raw_obj.length < length
218
+ if length != -1 && raw_obj.bytesize < length
221
219
  # The new object was not appended and it did not completely fill the
222
220
  # free space. So we have to write a new header to mark the remaining
223
221
  # empty space.
224
- unless length - raw_obj.length >= FlatFileBlobHeader::LENGTH
222
+ unless length - raw_obj.bytesize >= FlatFileBlobHeader::LENGTH
225
223
  PEROBS.log.fatal "Not enough space to append the empty space " +
226
- "header (space: #{length} bytes, object: #{raw_obj.length} " +
224
+ "header (space: #{length} bytes, object: #{raw_obj.bytesize} " +
227
225
  "bytes)."
228
226
  end
229
227
  space_address = @f.pos
230
- space_length = length - FlatFileBlobHeader::LENGTH - raw_obj.length
228
+ space_length = length - FlatFileBlobHeader::LENGTH - raw_obj.bytesize
231
229
  FlatFileBlobHeader.new(@f, space_address, 0, space_length,
232
230
  0, 0).write
233
231
  # Register the new space with the space list.
@@ -272,6 +270,15 @@ module PEROBS
272
270
  nil
273
271
  end
274
272
 
273
+ def search_object(id)
274
+ each_blob_header do |pos, header|
275
+ return read_obj_by_address(pos, id)
276
+ end
277
+
278
+ nil
279
+ end
280
+
281
+
275
282
  # Read the object at the specified address.
276
283
  # @param addr [Integer] Offset in the flat file
277
284
  # @param id [Integer] ID of the data blob
@@ -312,47 +319,19 @@ module PEROBS
312
319
  # Mark the object with the given ID.
313
320
  # @param id [Integer] ID of the object
314
321
  def mark_obj_by_id(id)
315
- if (addr = find_obj_addr_by_id(id))
316
- mark_obj_by_address(addr, id)
317
- end
318
- end
319
-
320
- # Mark the object at the specified address.
321
- # @param addr [Integer] Offset in the file
322
- # @param id [Integer] ID of the object
323
- def mark_obj_by_address(addr, id)
324
- FlatFileBlobHeader.read_at(@f, addr, id).set_mark_flag
322
+ @marks.insert(id, 0)
325
323
  end
326
324
 
327
325
  # Return true if the object with the given ID is marked, false otherwise.
328
326
  # @param id [Integer] ID of the object
329
327
  def is_marked_by_id?(id)
330
- if (addr = find_obj_addr_by_id(id))
331
- header = FlatFileBlobHeader.read_at(@f, addr, id)
332
- return header.is_marked?
333
- end
334
-
335
- false
328
+ !@marks.get(id).nil?
336
329
  end
337
330
 
338
331
  # Clear alls marks.
339
332
  def clear_all_marks
340
- t = Time.now
341
- PEROBS.log.info "Clearing all marks..."
342
-
343
- total_blob_count = 0
344
- marked_blob_count = 0
345
-
346
- each_blob_header do |pos, header|
347
- total_blob_count += 1
348
- if header.is_valid? && header.is_marked?
349
- # Clear all valid and marked blocks.
350
- marked_blob_count += 1
351
- header.clear_mark_flag
352
- end
353
- end
354
- PEROBS.log.info "#{marked_blob_count} marks in #{total_blob_count} " +
355
- "objects cleared in #{Time.now - t} seconds"
333
+ @marks.erase
334
+ @marks.open
356
335
  end
357
336
 
358
337
  # Eliminate all the holes in the file. This is an in-place
@@ -464,7 +443,7 @@ module PEROBS
464
443
  begin
465
444
  @f.seek(pos + FlatFileBlobHeader::LENGTH)
466
445
  buf = @f.read(header.length)
467
- if buf.length != header.length
446
+ if buf.bytesize != header.length
468
447
  PEROBS.log.error "Premature end of file in blob with ID " +
469
448
  "#{header.id}."
470
449
  discard_damaged_blob(header) if repair
@@ -33,7 +33,7 @@ module PEROBS
33
33
  #
34
34
  # 1 Byte: Flags byte.
35
35
  # Bit 0: 0 deleted entry, 1 valid entry
36
- # Bit 1: 0 unmarked, 1 marked
36
+ # Bit 1: 0 reserved, must be 0
37
37
  # Bit 2: 0 uncompressed data, 1 compressed data
38
38
  # Bit 3: 0 current entry, 1 outdated entry
39
39
  # Bit 4 - 7: reserved, must be 0
@@ -50,7 +50,6 @@ module PEROBS
50
50
  # The length of the header in bytes.
51
51
  LENGTH = 21
52
52
  VALID_FLAG_BIT = 0
53
- MARK_FLAG_BIT = 1
54
53
  COMPRESSED_FLAG_BIT = 2
55
54
  OUTDATED_FLAG_BIT = 3
56
55
 
@@ -148,23 +147,6 @@ module PEROBS
148
147
  bit_set?(VALID_FLAG_BIT)
149
148
  end
150
149
 
151
- # Return true if the blob has been marked.
152
- def is_marked?
153
- bit_set?(MARK_FLAG_BIT)
154
- end
155
-
156
- # Set the mark bit.
157
- def set_mark_flag
158
- set_flag(MARK_FLAG_BIT)
159
- write_flags
160
- end
161
-
162
- # Clear the mark bit.
163
- def clear_mark_flag
164
- clear_flag(MARK_FLAG_BIT)
165
- write_flags
166
- end
167
-
168
150
  # Return true if the blob contains compressed data.
169
151
  def is_compressed?
170
152
  bit_set?(COMPRESSED_FLAG_BIT)
@@ -87,6 +87,7 @@ module PEROBS
87
87
  end
88
88
 
89
89
  def FlatFileDB::delete_db(db_name)
90
+ close
90
91
  FileUtils.rm_rf(db_name)
91
92
  end
92
93
 
@@ -142,6 +143,10 @@ module PEROBS
142
143
  end
143
144
  end
144
145
 
146
+ def search_object(id)
147
+ @flat_file.search_object(id)
148
+ end
149
+
145
150
  # This method must be called to initiate the marking process.
146
151
  def clear_marks
147
152
  @flat_file.clear_all_marks
@@ -70,12 +70,14 @@ module PEROBS
70
70
  while retries > 0
71
71
  begin
72
72
  @file = File.open(@file_name, File::RDWR | File::CREAT, 0644)
73
+ @file.sync = true
73
74
 
74
75
  if @file.flock(File::LOCK_EX | File::LOCK_NB)
75
76
  # We have taken the lock. Write the PID into the file and leave it
76
77
  # open.
77
78
  @file.write($$)
78
79
  @file.flush
80
+ @file.fsync
79
81
  @file.truncate(@file.pos)
80
82
  PEROBS.log.debug "Lock file #{@file_name} has been taken for " +
81
83
  "process #{$$}"
@@ -129,6 +131,7 @@ module PEROBS
129
131
 
130
132
  begin
131
133
  @file.flock(File::LOCK_UN)
134
+ @file.fsync
132
135
  @file.close
133
136
  forced_unlock
134
137
  PEROBS.log.debug "Lock file #{@file_name} for PID #{$$} has been " +
data/lib/perobs/Object.rb CHANGED
@@ -49,9 +49,11 @@ module PEROBS
49
49
  attr_reader :attributes
50
50
 
51
51
  # This method can be used to define instance variable for
52
- # PEROBS::Object derived classes.
52
+ # PEROBS::Object derived classes. Persistent attributes always have
53
+ # getter and setter methods defined. So it's essentially equivalent to
54
+ # attr_accessor but additionally declares an attribute as persistent.
53
55
  # @param attributes [Symbol] Name of the instance variable
54
- def po_attr(*attributes)
56
+ def attr_persist(*attributes)
55
57
  attributes.each do |attr_name|
56
58
  unless attr_name.is_a?(Symbol)
57
59
  PEROBS.log.fatal "name must be a symbol but is a " +
@@ -73,6 +75,9 @@ module PEROBS
73
75
  end
74
76
  end
75
77
 
78
+ # This is the deprecated name for the attr_persist method
79
+ alias po_attr attr_persist
80
+
76
81
  end
77
82
 
78
83
  attr_reader :attributes
@@ -108,7 +113,7 @@ module PEROBS
108
113
  # object was saved with an earlier version of the program that did not yet
109
114
  # have the instance variable. If you want to assign another PEROBS object
110
115
  # to the variable you should use the block variant to avoid unnecessary
111
- # creation of PEROBS object that later need to be collected again.
116
+ # creation of PEROBS objects that later need to be collected again.
112
117
  def attr_init(attr, val = nil, &block)
113
118
  if _all_attributes.include?(attr)
114
119
  unless instance_variable_defined?('@' + attr.to_s)
@@ -133,7 +133,8 @@ module PEROBS
133
133
  @store = p.store
134
134
  @_id = p.id
135
135
  @store._register_in_memory(self, @_id)
136
- ObjectSpace.define_finalizer(self, ObjectBase._finalize(@store, @_id))
136
+ ObjectSpace.define_finalizer(
137
+ self, ObjectBase._finalize(@store, @_id, object_id))
137
138
  @_stash_map = nil
138
139
  # Allocate a proxy object for this object. User code should only operate
139
140
  # on this proxy, never on self.
@@ -144,8 +145,8 @@ module PEROBS
144
145
  # is done this way to prevent the Proc object hanging on to a reference to
145
146
  # self which would prevent the object from being collected. This internal
146
147
  # method is not intended for users to call.
147
- def ObjectBase._finalize(store, id)
148
- proc { store._collect(id) }
148
+ def ObjectBase._finalize(store, id, ruby_object_id)
149
+ proc { store._collect(id, ruby_object_id) }
149
150
  end
150
151
 
151
152
  # Library internal method to transfer the Object to a new store.
@@ -158,7 +159,8 @@ module PEROBS
158
159
  # Register the object as in-memory object with the new store.
159
160
  @store._register_in_memory(self, @_id)
160
161
  # Register the finalizer for the new store.
161
- ObjectSpace.define_finalizer(self, ObjectBase._finalize(@store, @_id))
162
+ ObjectSpace.define_finalizer(
163
+ self, ObjectBase._finalize(@store, @_id, object_id))
162
164
  @myself = POXReference.new(@store, @_id)
163
165
  end
164
166