perobs 3.0.1 → 3.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 20f164ff526b984530355e99ab38623fd2b87cea
4
- data.tar.gz: befc7531c92e0dc86c23b058cbc572657f66661f
3
+ metadata.gz: 985e62db6b84f509a3b390ce11ac96f4215de5ea
4
+ data.tar.gz: f6747e42300df5c9b55c1a5bbc6adf9822165b7b
5
5
  SHA512:
6
- metadata.gz: 3b18e2e16719f896d8500852f0c1529ce2a1d9816695df04be2203cd1a10da8b70fe737afa910ec1fed71855eb6534da2b0dd16f8c6428c0ea1ae06a6509b59f
7
- data.tar.gz: d6de8c99f6cffcb245b5468c6f6b5f4b5572c44019f7b9926adeae126a81ad507b815bbefda24b07f4724bc0a11abc0ab3235b304d2ca7180cde3bdc628c0b1e
6
+ metadata.gz: 7ed1499555741de10e6810f62041ffc879a738f488f5ce09860bddae21ed12cf999a8d4303361227a3798f3f389bd6cb9584e71e809b26d0f82fa7cdc6a28544
7
+ data.tar.gz: 2c56287162c7ae376ecdc091e55655fd37e4985e22914a4e21d74f28eb0c3e054ca543fb7092485ea09605b8a0ad4cc33f13dc96fa19ffcdf5137921c5b145ae
data/lib/perobs/Array.rb CHANGED
@@ -81,7 +81,7 @@ module PEROBS
81
81
  # PEROBS users should never call this method or equivalents of derived
82
82
  # methods directly.
83
83
  # @param p [PEROBS::Handle] PEROBS handle
84
- # @param size [Fixnum] The requested size of the Array
84
+ # @param size [Integer] The requested size of the Array
85
85
  # @param default [Any] The default value that is returned when no value is
86
86
  # stored for a specific key.
87
87
  def initialize(p, size = 0, default = nil)
@@ -94,7 +94,7 @@ module PEROBS
94
94
 
95
95
  # Return a list of all object IDs of all persistend objects that this Array
96
96
  # is referencing.
97
- # @return [Array of Fixnum or Bignum] IDs of referenced objects
97
+ # @return [Array of Integer] IDs of referenced objects
98
98
  def _referenced_object_ids
99
99
  @data.each.select do |v|
100
100
  v && v.respond_to?(:is_poxreference?)
@@ -103,7 +103,7 @@ module PEROBS
103
103
 
104
104
  # This method should only be used during store repair operations. It will
105
105
  # delete all references to the given object ID.
106
- # @param id [Fixnum/Bignum] targeted object ID
106
+ # @param id [Integer] targeted object ID
107
107
  def _delete_reference_to_id(id)
108
108
  @data.delete_if do |v|
109
109
  v && v.respond_to?(:is_poxreference?) && v.id == id
@@ -65,7 +65,7 @@ module PEROBS
65
65
  end
66
66
 
67
67
  # Write the given bytes with the given ID into the DB.
68
- # @param id [Fixnum or Bignum] ID
68
+ # @param id [Integer] ID
69
69
  # @param raw [String] sequence of bytes
70
70
  def write_object(id, raw)
71
71
  if @entries.length > @btreedb.max_blob_size
@@ -87,7 +87,7 @@ module PEROBS
87
87
  end
88
88
 
89
89
  # Read the entry for the given ID and return it as bytes.
90
- # @param id [Fixnum or Bignum] ID
90
+ # @param id [Integer] ID
91
91
  # @return [String] sequence of bytes or nil if ID is unknown
92
92
  def read_object(id)
93
93
  return nil unless (index_entry = find(id))
@@ -95,7 +95,7 @@ module PEROBS
95
95
  end
96
96
 
97
97
  # Find the data for the object with given id.
98
- # @param id [Fixnum or Bignum] Object ID
98
+ # @param id [Integer] Object ID
99
99
  # @return [Array] Returns an Array that represents the index entry for the
100
100
  # given object.
101
101
  def find(id)
@@ -109,7 +109,7 @@ module PEROBS
109
109
  end
110
110
 
111
111
  # Set a mark on the entry with the given ID.
112
- # @param id [Fixnum or Bignum] ID of the entry
112
+ # @param id [Integer] ID of the entry
113
113
  def mark(id)
114
114
  found = false
115
115
  @entries.each do |entry|
@@ -129,7 +129,7 @@ module PEROBS
129
129
  end
130
130
 
131
131
  # Check if the entry for a given ID is marked.
132
- # @param id [Fixnum or Bignum] ID of the entry
132
+ # @param id [Integer] ID of the entry
133
133
  # @param ignore_errors [Boolean] If set to true no errors will be raised
134
134
  # for non-existing objects.
135
135
  # @return [TrueClass or FalseClass] true if marked, false otherwise
@@ -205,8 +205,8 @@ module PEROBS
205
205
 
206
206
  # Write a string of bytes into the file at the given address.
207
207
  # @param raw [String] bytes to write
208
- # @param address [Fixnum] offset in the file
209
- # @return [Fixnum] number of bytes written
208
+ # @param address [Integer] offset in the file
209
+ # @return [Integer] number of bytes written
210
210
  def write_to_blobs_file(raw, address)
211
211
  begin
212
212
  File.write(@blobs_file_name, raw, address)
@@ -236,9 +236,9 @@ module PEROBS
236
236
 
237
237
  # Reserve the bytes needed for the specified number of bytes with the
238
238
  # given ID.
239
- # @param id [Fixnum or Bignum] ID of the entry
240
- # @param bytes [Fixnum] number of bytes for this entry
241
- # @return [Fixnum] the start address of the reserved blob
239
+ # @param id [Integer] ID of the entry
240
+ # @param bytes [Integer] number of bytes for this entry
241
+ # @return [Integer] the start address of the reserved blob
242
242
  def reserve_bytes(id, bytes, crc32)
243
243
  # index of first blob after the last seen entry
244
244
  end_of_last_entry = 0
@@ -103,7 +103,7 @@ module PEROBS
103
103
  end
104
104
 
105
105
  # Return true if the object with given ID exists
106
- # @param id [Fixnum or Bignum]
106
+ # @param id [Integer]
107
107
  def include?(id)
108
108
  !(blob = find_blob(id)).nil? && !blob.find(id).nil?
109
109
  end
@@ -143,7 +143,7 @@ module PEROBS
143
143
  end
144
144
 
145
145
  # Load the given object from the filesystem.
146
- # @param id [Fixnum or Bignum] object ID
146
+ # @param id [Integer] object ID
147
147
  # @return [Hash] Object as defined by PEROBS::ObjectBase or nil if ID does
148
148
  # not exist
149
149
  def get_object(id)
@@ -166,13 +166,13 @@ module PEROBS
166
166
  end
167
167
 
168
168
  # Mark an object.
169
- # @param id [Fixnum or Bignum] ID of the object to mark
169
+ # @param id [Integer] ID of the object to mark
170
170
  def mark(id)
171
171
  (blob = find_blob(id)) && blob.mark(id)
172
172
  end
173
173
 
174
174
  # Check if the object is marked.
175
- # @param id [Fixnum or Bignum] ID of the object to check
175
+ # @param id [Integer] ID of the object to check
176
176
  # @param ignore_errors [Boolean] If set to true no errors will be raised
177
177
  # for non-existing objects.
178
178
  def is_marked?(id, ignore_errors = false)
@@ -189,7 +189,7 @@ module PEROBS
189
189
  end
190
190
 
191
191
  # Check if the stored object is syntactically correct.
192
- # @param id [Fixnum/Bignum] Object ID
192
+ # @param id [Integer] Object ID
193
193
  # @param repair [TrueClass/FalseClass] True if an repair attempt should be
194
194
  # made.
195
195
  # @return [TrueClass/FalseClass] True if the object is OK, otherwise
@@ -208,7 +208,7 @@ module PEROBS
208
208
  # Store the given serialized object into the cluster files. This method is
209
209
  # for internal use only!
210
210
  # @param raw [String] Serialized Object as defined by PEROBS::ObjectBase
211
- # @param id [Fixnum or Bignum] Object ID
211
+ # @param id [Integer] Object ID
212
212
  def put_raw_object(raw, id)
213
213
  find_blob(id, true).write_object(id, raw)
214
214
  end
data/lib/perobs/Cache.rb CHANGED
@@ -37,7 +37,7 @@ module PEROBS
37
37
  class Cache
38
38
 
39
39
  # Create a new Cache object.
40
- # @param bits [Fixnum] Number of bits for the cache index. This parameter
40
+ # @param bits [Integer] Number of bits for the cache index. This parameter
41
41
  # heavilty affects the performance and memory consumption of the
42
42
  # cache.
43
43
  def initialize(bits = 16)
@@ -94,7 +94,7 @@ module PEROBS
94
94
  end
95
95
 
96
96
  # Return the PEROBS::Object with the specified ID or nil if not found.
97
- # @param id [Fixnum or Bignum] ID of the cached PEROBS::ObjectBase
97
+ # @param id [Integer] ID of the cached PEROBS::ObjectBase
98
98
  #def object_by_id(id)
99
99
  # idx = id & @mask
100
100
  # # The index is just a hash. We still need to check if the object IDs are
@@ -47,14 +47,14 @@ module PEROBS
47
47
 
48
48
  # Get the ID for a given class.
49
49
  # @param klass [String] Class
50
- # @return [Fixnum] ID. If klass is not yet known a new ID will be
50
+ # @return [Integer] ID. If klass is not yet known a new ID will be
51
51
  # allocated.
52
52
  def class_to_id(klass)
53
53
  @by_class[klass] || new_id(klass)
54
54
  end
55
55
 
56
56
  # Get the klass for a given ID.
57
- # @param id [Fixnum]
57
+ # @param id [Integer]
58
58
  # @return [String] String version of the class
59
59
  def id_to_class(id)
60
60
  @by_id[id]
@@ -85,7 +85,7 @@ module PEROBS
85
85
  end
86
86
 
87
87
  # Return true if the object with given ID exists
88
- # @param id [Fixnum or Bignum]
88
+ # @param id [Integer]
89
89
  def include?(id)
90
90
  !dynamo_get_item(id.to_s).nil?
91
91
  end
@@ -116,7 +116,7 @@ module PEROBS
116
116
  end
117
117
 
118
118
  # Load the given object from the filesystem.
119
- # @param id [Fixnum or Bignum] object ID
119
+ # @param id [Integer] object ID
120
120
  # @return [Hash] Object as defined by PEROBS::ObjectBase or nil if ID does
121
121
  # not exist
122
122
  def get_object(id)
@@ -148,13 +148,13 @@ module PEROBS
148
148
  end
149
149
 
150
150
  # Mark an object.
151
- # @param id [Fixnum or Bignum] ID of the object to mark
151
+ # @param id [Integer] ID of the object to mark
152
152
  def mark(id)
153
153
  dynamo_mark_item(id.to_s, true)
154
154
  end
155
155
 
156
156
  # Check if the object is marked.
157
- # @param id [Fixnum or Bignum] ID of the object to check
157
+ # @param id [Integer] ID of the object to check
158
158
  def is_marked?(id)
159
159
  dynamo_is_marked?(id.to_s)
160
160
  end
@@ -167,7 +167,7 @@ module PEROBS
167
167
  end
168
168
 
169
169
  # Check if the stored object is syntactically correct.
170
- # @param id [Fixnum/Bignum] Object ID
170
+ # @param id [Integer] Object ID
171
171
  # @param repair [TrueClass/FalseClass] True if an repair attempt should be
172
172
  # made.
173
173
  # @return [TrueClass/FalseClass] True if the object is OK, otherwise
@@ -45,13 +45,13 @@ module PEROBS
45
45
  FIRST_SPACE_OFFSET = 3 * 8
46
46
  HEADER_SIZE = 4 * 8
47
47
 
48
- attr_reader :total_entries, :total_spaces, :file_name
49
- attr_accessor :first_entry
48
+ attr_reader :total_entries, :total_spaces, :file_name, :first_entry
50
49
 
51
50
  # Create a new stack file in the given directory with the given file name.
52
51
  # @param dir [String] Directory
53
52
  # @param name [String] File name
54
- # @param entry_bytes [Fixnum] Number of bytes each entry must have
53
+ # @param entry_bytes [Integer] Number of bytes each entry must have
54
+ # @param first_entry_default [Integer] Default address of the first blob
55
55
  def initialize(dir, name, entry_bytes, first_entry_default = 0)
56
56
  @file_name = File.join(dir, name + '.blobs')
57
57
  if entry_bytes < 8
@@ -125,9 +125,16 @@ module PEROBS
125
125
  write_header
126
126
  end
127
127
 
128
+ # Change the address of the first blob.
129
+ # @param address [Integer] New address
130
+ def first_entry=(address)
131
+ @first_entry = address
132
+ write_header
133
+ end
134
+
128
135
  # Return the address of a free blob storage space. Addresses start at 0
129
136
  # and increase linearly.
130
- # @return [Fixnum] address of a free blob space
137
+ # @return [Integer] address of a free blob space
131
138
  def free_address
132
139
  if @first_space == 0
133
140
  # There is currently no free entry. Create a new reserved entry at the
@@ -170,7 +177,7 @@ module PEROBS
170
177
 
171
178
  # Store the given byte blob at the specified address. If the blob space is
172
179
  # already in use the content will be overwritten.
173
- # @param address [Fixnum] Address to store the blob
180
+ # @param address [Integer] Address to store the blob
174
181
  # @param bytes [String] bytes to store
175
182
  def store_blob(address, bytes)
176
183
  unless address >= 0
@@ -217,7 +224,7 @@ module PEROBS
217
224
  end
218
225
 
219
226
  # Retrieve a blob from the given address.
220
- # @param address [Fixnum] Address to store the blob
227
+ # @param address [Integer] Address to store the blob
221
228
  # @return [String] blob bytes
222
229
  def retrieve_blob(address)
223
230
  unless address > 0
@@ -248,7 +255,7 @@ module PEROBS
248
255
  end
249
256
 
250
257
  # Delete the blob at the given address.
251
- # @param address [Fixnum] Address of blob to delete
258
+ # @param address [Integer] Address of blob to delete
252
259
  def delete_blob(address)
253
260
  unless address >= 0
254
261
  PEROBS.log.fatal "Blob address must be larger than 0, " +
@@ -116,6 +116,7 @@ module PEROBS
116
116
  PEROBS.log.fatal "Cannot sync flat file database: #{e.message}"
117
117
  end
118
118
  @index.sync
119
+ @space_list.sync
119
120
  end
120
121
 
121
122
  # Delete the blob for the specified ID.
@@ -195,18 +196,25 @@ module PEROBS
195
196
  header = FlatFileBlobHeader.read_at(@f, addr)
196
197
  if header.length != length
197
198
  PEROBS.log.fatal "Length in free list (#{length}) and header " +
198
- "(#{header.length}) don't match."
199
+ "(#{header.length}) for address #{addr} don't match."
199
200
  end
200
201
  if raw_obj.length > header.length
201
202
  PEROBS.log.fatal "Object (#{raw_obj.length}) is longer than " +
202
203
  "blob space (#{header.length})."
203
204
  end
204
205
  if header.is_valid?
205
- PEROBS.log.fatal "Entry (flags: #{header.flags}) is already used."
206
+ PEROBS.log.fatal "Entry at address #{addr} with flags: " +
207
+ "#{header.flags} is already used for ID #{header.id}."
206
208
  end
207
209
  end
208
210
  flags = 1 << FlatFileBlobHeader::VALID_FLAG_BIT
209
211
  flags |= (1 << FlatFileBlobHeader::COMPRESSED_FLAG_BIT) if compressed
212
+ if old_addr && old_header.is_marked?
213
+ # This method might be called in the middle of an operation that
214
+ # uses the mark flag. We must ensure that the flag is carried over
215
+ # to the new header.
216
+ flags |= (1 << FlatFileBlobHeader::MARK_FLAG_BIT)
217
+ end
210
218
  FlatFileBlobHeader.new(@f, addr, flags, raw_obj.length, id, crc).write
211
219
  @f.write(raw_obj)
212
220
  if length != -1 && raw_obj.length < length
@@ -225,15 +233,19 @@ module PEROBS
225
233
  # Register the new space with the space list.
226
234
  @space_list.add_space(space_address, space_length) if space_length > 0
227
235
  end
236
+
237
+ # Once the blob has been written we can update the index as well.
238
+ @index.insert(id, addr)
239
+
228
240
  if old_addr
229
241
  # If we had an existing object stored for the ID we have to mark
230
242
  # this entry as deleted now.
231
243
  old_header.clear_flags
244
+ # And register the newly freed space with the space list.
245
+ @space_list.add_space(old_addr, old_header.length)
232
246
  else
233
247
  @f.flush
234
248
  end
235
- # Once the blob has been written we can update the index as well.
236
- @index.insert(id, addr)
237
249
  rescue IOError => e
238
250
  PEROBS.log.fatal "Cannot write blob for ID #{id} to FlatFileDB: " +
239
251
  e.message
@@ -549,21 +561,31 @@ module PEROBS
549
561
 
550
562
  each_blob_header do |pos, header|
551
563
  if header.is_valid?
552
- @index.insert(header.id, pos)
564
+ if (duplicate_pos = @index.get(header.id))
565
+ PEROBS.log.error "FlatFile contains multiple blobs for ID " +
566
+ "#{header.id}. First blob is at address #{duplicate_pos}. " +
567
+ "Other blob found at address #{pos}."
568
+ @space_list.add_space(pos, header.length) if header.length > 0
569
+ discard_damaged_blob(header)
570
+ else
571
+ @index.insert(header.id, pos)
572
+ end
553
573
  else
554
574
  @space_list.add_space(pos, header.length) if header.length > 0
555
575
  end
556
576
  end
577
+
578
+ sync
557
579
  end
558
580
 
559
581
  def has_space?(address, size)
560
582
  header = FlatFileBlobHeader.read_at(@f, address)
561
- header.length == size
583
+ !header.is_valid? && header.length == size
562
584
  end
563
585
 
564
586
  def has_id_at?(id, address)
565
587
  header = FlatFileBlobHeader.read_at(@f, address)
566
- header.id == id
588
+ header.is_valid? && header.id == id
567
589
  end
568
590
 
569
591
  def inspect
@@ -59,10 +59,10 @@ module PEROBS
59
59
  # Create a new FlatFileBlobHeader with the given flags, length, id and crc.
60
60
  # @param file [File] the FlatFile that contains the header
61
61
  # @param addr [Integer] the offset address of the header in the file
62
- # @param flags [Fixnum] 8 bit number, see above
63
- # @param length [Fixnum] length of the header in bytes
62
+ # @param flags [Integer] 8 bit number, see above
63
+ # @param length [Integer] length of the header in bytes
64
64
  # @param id [Integer] ID of the blob entry
65
- # @param crc [Fixnum] CRC32 checksum of the blob entry
65
+ # @param crc [Integer] CRC32 checksum of the blob entry
66
66
  def initialize(file, addr, flags, length, id, crc)
67
67
  @file = file
68
68
  @addr = addr
@@ -116,7 +116,8 @@ module PEROBS
116
116
  header = FlatFileBlobHeader.new(file, addr, *buf.unpack(FORMAT))
117
117
  if id && header.id != id
118
118
  PEROBS.log.fatal "Mismatch between FlatFile index and blob file " +
119
- "found for entry with ID #{id}/#{header.id}"
119
+ "found. FlatFile has entry with ID #{header.id} at address " +
120
+ "#{addr}. Index has ID #{id} for this address."
120
121
  end
121
122
 
122
123
  return header
@@ -138,14 +139,8 @@ module PEROBS
138
139
  # @param file [File] The file handle of the blob file.
139
140
  # @param addr [Integer] The address of the header
140
141
  def clear_flags
141
- begin
142
- @file.seek(@addr)
143
- @file.write([ 0 ].pack('C'))
144
- @file.flush
145
- rescue IOError => e
146
- PEROBS.log.fatal "Clearing flags of FlatFileBlobHeader with ID " +
147
- "#{@id} failed: #{e.message}"
148
- end
142
+ @flags = 0
143
+ write_flags
149
144
  end
150
145
 
151
146
  # Return true if the header is for a non-empty blob.
@@ -91,7 +91,7 @@ module PEROBS
91
91
  end
92
92
 
93
93
  # Return true if the object with given ID exists
94
- # @param id [Fixnum or Bignum]
94
+ # @param id [Integer]
95
95
  def include?(id)
96
96
  !@flat_file.find_obj_addr_by_id(id).nil?
97
97
  end
@@ -131,7 +131,7 @@ module PEROBS
131
131
  end
132
132
 
133
133
  # Load the given object from the filesystem.
134
- # @param id [Fixnum or Bignum] object ID
134
+ # @param id [Integer] object ID
135
135
  # @return [Hash] Object as defined by PEROBS::ObjectBase or nil if ID does
136
136
  # not exist
137
137
  def get_object(id)
@@ -155,13 +155,13 @@ module PEROBS
155
155
  end
156
156
 
157
157
  # Mark an object.
158
- # @param id [Fixnum or Bignum] ID of the object to mark
158
+ # @param id [Integer] ID of the object to mark
159
159
  def mark(id)
160
160
  @flat_file.mark_obj_by_id(id)
161
161
  end
162
162
 
163
163
  # Check if the object is marked.
164
- # @param id [Fixnum or Bignum] ID of the object to check
164
+ # @param id [Integer] ID of the object to check
165
165
  # @param ignore_errors [Boolean] If set to true no errors will be raised
166
166
  # for non-existing objects.
167
167
  def is_marked?(id, ignore_errors = false)
@@ -177,7 +177,7 @@ module PEROBS
177
177
  end
178
178
 
179
179
  # Check if the stored object is syntactically correct.
180
- # @param id [Fixnum/Bignum] Object ID
180
+ # @param id [Integer] Object ID
181
181
  # @param repair [TrueClass/FalseClass] True if an repair attempt should be
182
182
  # made.
183
183
  # @return [TrueClass/FalseClass] True if the object is OK, otherwise
@@ -202,7 +202,7 @@ module PEROBS
202
202
  # Store the given serialized object into the cluster files. This method is
203
203
  # for internal use only!
204
204
  # @param raw [String] Serialized Object as defined by PEROBS::ObjectBase
205
- # @param id [Fixnum or Bignum] Object ID
205
+ # @param id [Integer] Object ID
206
206
  def put_raw_object(raw, id)
207
207
  @flat_file.write_obj_by_id(id, raw)
208
208
  end