perobs 4.0.0 → 4.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. checksums.yaml +4 -4
  2. data/lib/perobs.rb +1 -0
  3. data/lib/perobs/Array.rb +66 -19
  4. data/lib/perobs/BTree.rb +83 -12
  5. data/lib/perobs/BTreeBlob.rb +1 -1
  6. data/lib/perobs/BTreeDB.rb +2 -2
  7. data/lib/perobs/BTreeNode.rb +365 -85
  8. data/lib/perobs/BigArray.rb +267 -0
  9. data/lib/perobs/BigArrayNode.rb +998 -0
  10. data/lib/perobs/BigHash.rb +262 -0
  11. data/lib/perobs/BigTree.rb +184 -0
  12. data/lib/perobs/BigTreeNode.rb +873 -0
  13. data/lib/perobs/ConsoleProgressMeter.rb +61 -0
  14. data/lib/perobs/DataBase.rb +4 -3
  15. data/lib/perobs/DynamoDB.rb +57 -15
  16. data/lib/perobs/EquiBlobsFile.rb +143 -51
  17. data/lib/perobs/FNV_Hash_1a_64.rb +54 -0
  18. data/lib/perobs/FlatFile.rb +363 -203
  19. data/lib/perobs/FlatFileBlobHeader.rb +98 -54
  20. data/lib/perobs/FlatFileDB.rb +42 -20
  21. data/lib/perobs/Hash.rb +58 -13
  22. data/lib/perobs/IDList.rb +144 -0
  23. data/lib/perobs/IDListPage.rb +107 -0
  24. data/lib/perobs/IDListPageFile.rb +180 -0
  25. data/lib/perobs/IDListPageRecord.rb +142 -0
  26. data/lib/perobs/Object.rb +18 -15
  27. data/lib/perobs/ObjectBase.rb +38 -4
  28. data/lib/perobs/PersistentObjectCache.rb +53 -67
  29. data/lib/perobs/PersistentObjectCacheLine.rb +24 -12
  30. data/lib/perobs/ProgressMeter.rb +97 -0
  31. data/lib/perobs/SpaceTree.rb +21 -12
  32. data/lib/perobs/SpaceTreeNode.rb +53 -61
  33. data/lib/perobs/Store.rb +71 -32
  34. data/lib/perobs/version.rb +1 -1
  35. data/perobs.gemspec +4 -4
  36. data/test/Array_spec.rb +15 -6
  37. data/test/BTree_spec.rb +5 -2
  38. data/test/BigArray_spec.rb +214 -0
  39. data/test/BigHash_spec.rb +144 -0
  40. data/test/BigTreeNode_spec.rb +153 -0
  41. data/test/BigTree_spec.rb +259 -0
  42. data/test/EquiBlobsFile_spec.rb +105 -1
  43. data/test/FNV_Hash_1a_64_spec.rb +59 -0
  44. data/test/FlatFileDB_spec.rb +63 -14
  45. data/test/Hash_spec.rb +1 -2
  46. data/test/IDList_spec.rb +77 -0
  47. data/test/LegacyDBs/LegacyDB.rb +151 -0
  48. data/test/LegacyDBs/version_3/class_map.json +1 -0
  49. data/test/LegacyDBs/version_3/config.json +1 -0
  50. data/test/LegacyDBs/version_3/database.blobs +0 -0
  51. data/test/LegacyDBs/version_3/database_spaces.blobs +0 -0
  52. data/test/LegacyDBs/version_3/index.blobs +0 -0
  53. data/test/LegacyDBs/version_3/version +1 -0
  54. data/test/LockFile_spec.rb +9 -6
  55. data/test/SpaceTree_spec.rb +4 -1
  56. data/test/Store_spec.rb +290 -199
  57. data/test/spec_helper.rb +9 -4
  58. metadata +47 -10
  59. data/lib/perobs/TreeDB.rb +0 -277
@@ -0,0 +1,61 @@
1
+ # encoding: UTF-8
2
+ #
3
+ # = ConsoleProgressMeter.rb -- Persistent Ruby Object Store
4
+ #
5
+ # Copyright (c) 2018 by Chris Schlaeger <chris@taskjuggler.org>
6
+ #
7
+ # MIT License
8
+ #
9
+ # Permission is hereby granted, free of charge, to any person obtaining
10
+ # a copy of this software and associated documentation files (the
11
+ # "Software"), to deal in the Software without restriction, including
12
+ # without limitation the rights to use, copy, modify, merge, publish,
13
+ # distribute, sublicense, and/or sell copies of the Software, and to
14
+ # permit persons to whom the Software is furnished to do so, subject to
15
+ # the following conditions:
16
+ #
17
+ # The above copyright notice and this permission notice shall be
18
+ # included in all copies or substantial portions of the Software.
19
+ #
20
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
21
+ # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
22
+ # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
23
+ # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
24
+ # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
25
+ # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
26
+ # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
27
+
28
+ require 'perobs/ProgressMeter'
29
+
30
+ module PEROBS
31
+
32
+ class ConsoleProgressMeter < ProgressMeter
33
+
34
+ LINE_LENGTH = 79
35
+
36
+ private
37
+
38
+ def print_bar
39
+ percent = @max_value == 0 ? 100.0 :
40
+ (@current_value.to_f / @max_value) * 100.0
41
+ percent = 0.0 if percent < 0
42
+ percent = 100.0 if percent > 100.0
43
+
44
+ meter = "<#{percent.to_i}%>"
45
+
46
+ bar_length = LINE_LENGTH - @name.chars.length - 3 - meter.chars.length
47
+ left_bar = '*' * (bar_length * percent / 100.0)
48
+ right_bar = ' ' * (bar_length - left_bar.chars.length)
49
+
50
+ print "\r#{@name} [#{left_bar}#{meter}#{right_bar}]"
51
+ end
52
+
53
+ def print_time
54
+ s = "\r#{@name} [#{secsToHMS(@end_time - @start_time)}]"
55
+ puts s + (' ' * (LINE_LENGTH - s.chars.length + 1))
56
+ end
57
+
58
+ end
59
+
60
+ end
61
+
@@ -2,7 +2,7 @@
2
2
  #
3
3
  # = DataBase.rb -- Persistent Ruby Object Store
4
4
  #
5
- # Copyright (c) 2015 by Chris Schlaeger <chris@taskjuggler.org>
5
+ # Copyright (c) 2015, 2018, 2019 by Chris Schlaeger <chris@taskjuggler.org>
6
6
  #
7
7
  # MIT License
8
8
  #
@@ -42,8 +42,9 @@ module PEROBS
42
42
 
43
43
  # Create a new DataBase object. This method must be overwritten by the
44
44
  # deriving classes and then called via their constructor.
45
- def initialize(serializer = :json)
46
- @serializer = serializer
45
+ def initialize(options)
46
+ @serializer = options[:serializer] || :json
47
+ @progressmeter = options[:progressmeter] || ProgressMeter.new
47
48
  @config = {}
48
49
  end
49
50
 
@@ -2,7 +2,8 @@
2
2
  #
3
3
  # = DynamoDB.rb -- Persistent Ruby Object Store
4
4
  #
5
- # Copyright (c) 2015, 2016 by Chris Schlaeger <chris@taskjuggler.org>
5
+ # Copyright (c) 2015, 2016, 2017, 2018
6
+ # by Chris Schlaeger <chris@taskjuggler.org>
6
7
  #
7
8
  # MIT License
8
9
  #
@@ -25,7 +26,7 @@
25
26
  # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
26
27
  # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
27
28
 
28
- require 'aws-sdk'
29
+ require 'aws-sdk-dynamodb'
29
30
 
30
31
  require 'perobs/DataBase'
31
32
  require 'perobs/BTreeBlob'
@@ -35,6 +36,10 @@ module PEROBS
35
36
  # This class implements an Amazon DynamoDB storage engine for PEROBS.
36
37
  class DynamoDB < DataBase
37
38
 
39
+ INTERNAL_ITEMS = %w( config item_counter )
40
+
41
+ attr_reader :item_counter
42
+
38
43
  # Create a new DynamoDB object.
39
44
  # @param db_name [String] name of the DB directory
40
45
  # @param options [Hash] options to customize the behavior. Currently only
@@ -50,7 +55,7 @@ module PEROBS
50
55
  options[:serializer] = :yaml
51
56
  end
52
57
 
53
- super(options[:serializer] || :json)
58
+ super(options)
54
59
 
55
60
  if options.include?(:aws_id) && options.include?(:aws_key)
56
61
  Aws.config[:credentials] = Aws::Credentials.new(options[:aws_id],
@@ -62,12 +67,25 @@ module PEROBS
62
67
 
63
68
  @dynamodb = Aws::DynamoDB::Client.new
64
69
  @table_name = db_name
65
- ensure_table_exists(@table_name)
70
+ @config = nil
71
+ # The number of items currently stored in the DB.
72
+ @item_counter = nil
73
+ if create_table(@table_name)
74
+ @config = { 'serializer' => @serializer }
75
+ put_hash('config', @config)
76
+ @item_counter = 0
77
+ dynamo_put_item('item_counter', @item_counter.to_s)
78
+ else
79
+ @config = get_hash('config')
80
+ if @config['serializer'] != @serializer
81
+ raise ArgumentError, "DynamoDB #{@table_name} was created with " +
82
+ "serializer #{@config['serializer']} but was now opened with " +
83
+ "serializer #{@serializer}."
84
+ end
85
+ @item_counter = dynamo_get_item('item_counter').to_i
86
+ end
66
87
 
67
88
  # Read the existing DB config.
68
- @config = get_hash('config')
69
- check_option('serializer')
70
- put_hash('config', @config)
71
89
  end
72
90
 
73
91
  # Delete the entire database. The database is no longer usable after this
@@ -112,6 +130,13 @@ module PEROBS
112
130
  # Store the given object into the cluster files.
113
131
  # @param obj [Hash] Object as defined by PEROBS::ObjectBase
114
132
  def put_object(obj, id)
133
+ id_str = id.to_s
134
+ unless dynamo_get_item(id_str)
135
+ # The is no object with this ID yet. Increase the item counter.
136
+ @item_counter += 1
137
+ dynamo_put_item('item_counter', @item_counter.to_s)
138
+ end
139
+
115
140
  dynamo_put_item(id.to_s, serialize(obj))
116
141
  end
117
142
 
@@ -128,23 +153,23 @@ module PEROBS
128
153
  each_item do |id|
129
154
  dynamo_mark_item(id, false)
130
155
  end
131
- # Mark the 'config' item so it will not get deleted.
132
- dynamo_mark_item('config')
133
156
  end
134
157
 
135
158
  # Permanently delete all objects that have not been marked. Those are
136
159
  # orphaned and are no longer referenced by any actively used object.
137
- # @return [Array] List of object IDs of the deleted objects.
160
+ # @return [Integer] Count of the deleted objects.
138
161
  def delete_unmarked_objects
139
- deleted_ids = []
162
+ deleted_objects_count = 0
140
163
  each_item do |id|
141
164
  unless dynamo_is_marked?(id)
142
165
  dynamo_delete_item(id)
143
- deleted_ids << id
166
+ deleted_objects_count += 1
167
+ @item_counter -= 1
144
168
  end
145
169
  end
170
+ dynamo_put_item('item_counter', @item_counter.to_s)
146
171
 
147
- deleted_ids
172
+ deleted_objects_count
148
173
  end
149
174
 
150
175
  # Mark an object.
@@ -163,7 +188,17 @@ module PEROBS
163
188
  # @param repair [TrueClass/FalseClass] True if found errors should be
164
189
  # repaired.
165
190
  def check_db(repair = false)
166
- # TODO: See if we can add checks here
191
+ unless (item_counter = dynamo_get_item('item_counter')) &&
192
+ item_counter == @item_counter
193
+ PEROBS.log.error "@item_counter variable (#{@item_counter}) and " +
194
+ "item_counter table entry (#{item_counter}) don't match"
195
+ end
196
+ item_counter = 0
197
+ each_item { item_counter += 1 }
198
+ unless item_counter == @item_counter
199
+ PEROBS.log.error "Table contains #{item_counter} items but " +
200
+ "@item_counter is #{@item_counter}"
201
+ end
167
202
  end
168
203
 
169
204
  # Check if the stored object is syntactically correct.
@@ -185,9 +220,11 @@ module PEROBS
185
220
 
186
221
  private
187
222
 
188
- def ensure_table_exists(table_name)
223
+ def create_table(table_name)
189
224
  begin
190
225
  @dynamodb.describe_table(:table_name => table_name)
226
+ # The table exists already. No need to create it.
227
+ return false
191
228
  rescue Aws::DynamoDB::Errors::ResourceNotFoundException
192
229
  @dynamodb.create_table(
193
230
  :table_name => table_name,
@@ -210,6 +247,8 @@ module PEROBS
210
247
  )
211
248
 
212
249
  @dynamodb.wait_until(:table_exists, table_name: table_name)
250
+ # The table was successfully created.
251
+ return true
213
252
  end
214
253
  end
215
254
 
@@ -251,6 +290,9 @@ module PEROBS
251
290
  break if resp.count <= 0
252
291
 
253
292
  resp.items.each do |item|
293
+ # Skip all internal items
294
+ next if INTERNAL_ITEMS.include?(item['Id'])
295
+
254
296
  yield(item['Id'])
255
297
  end
256
298
 
@@ -2,7 +2,8 @@
2
2
  #
3
3
  # = EquiBlobsFile.rb -- Persistent Ruby Object Store
4
4
  #
5
- # Copyright (c) 2016, 2017 by Chris Schlaeger <chris@taskjuggler.org>
5
+ # Copyright (c) 2016, 2017, 2018, 2019
6
+ # by Chris Schlaeger <chris@taskjuggler.org>
6
7
  #
7
8
  # MIT License
8
9
  #
@@ -26,6 +27,7 @@
26
27
  # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
27
28
 
28
29
  require 'perobs/Log'
30
+ require 'perobs/ProgressMeter'
29
31
 
30
32
  module PEROBS
31
33
 
@@ -37,6 +39,8 @@ module PEROBS
37
39
  # is used to represent an undefined address or nil. The file has a 4 * 8
38
40
  # bytes long header that stores the total entry count, the total space
39
41
  # count, the offset of the first entry and the offset of the first space.
42
+ # The header is followed by a custom entry section. Each entry is also 8
43
+ # bytes long. After the custom entry section the data blobs start.
40
44
  class EquiBlobsFile
41
45
 
42
46
  TOTAL_ENTRIES_OFFSET = 0
@@ -50,15 +54,21 @@ module PEROBS
50
54
  # Create a new stack file in the given directory with the given file name.
51
55
  # @param dir [String] Directory
52
56
  # @param name [String] File name
57
+ # @param progressmeter [ProgressMeter] Reference to a progress meter
58
+ # object
53
59
  # @param entry_bytes [Integer] Number of bytes each entry must have
54
60
  # @param first_entry_default [Integer] Default address of the first blob
55
- def initialize(dir, name, entry_bytes, first_entry_default = 0)
61
+ def initialize(dir, name, progressmeter, entry_bytes,
62
+ first_entry_default = 0)
63
+ @name = name
56
64
  @file_name = File.join(dir, name + '.blobs')
65
+ @progressmeter = progressmeter
57
66
  if entry_bytes < 8
58
67
  PEROBS.log.fatal "EquiBlobsFile entry size must be at least 8"
59
68
  end
60
69
  @entry_bytes = entry_bytes
61
70
  @first_entry_default = first_entry_default
71
+ clear_custom_data
62
72
  reset_counters
63
73
 
64
74
  # The File handle.
@@ -102,6 +112,57 @@ module PEROBS
102
112
  end
103
113
  end
104
114
 
115
+ # In addition to the standard offsets for the first entry and the first
116
+ # space any number of additional data fields can be registered. This must be
117
+ # done right after the object is instanciated and before the open() method
118
+ # is called. Each field represents a 64 bit unsigned integer.
119
+ # @param name [String] The label for this offset
120
+ # @param default_value [Integer] The default value for the offset
121
+ def register_custom_data(name, default_value = 0)
122
+ if @custom_data_labels.include?(name)
123
+ PEROBS.log.fatal "Custom data field #{name} has already been registered"
124
+ end
125
+
126
+ @custom_data_labels << name
127
+ @custom_data_values << default_value
128
+ @custom_data_defaults << default_value
129
+ end
130
+
131
+ # Reset (delete) all custom data labels that have been registered.
132
+ def clear_custom_data
133
+ unless @f.nil?
134
+ PEROBS.log.fatal "clear_custom_data should only be called when " +
135
+ "the file is not opened"
136
+ end
137
+
138
+ @custom_data_labels = []
139
+ @custom_data_values = []
140
+ @custom_data_defaults = []
141
+ end
142
+
143
+ # Set the registered custom data field to the given value.
144
+ # @param name [String] Label of the offset
145
+ # @param value [Integer] Value
146
+ def set_custom_data(name, value)
147
+ unless @custom_data_labels.include?(name)
148
+ PEROBS.log.fatal "Unknown custom data field #{name}"
149
+ end
150
+
151
+ @custom_data_values[@custom_data_labels.index(name)] = value
152
+ write_header if @f
153
+ end
154
+
155
+ # Get the registered custom data field value.
156
+ # @param name [String] Label of the offset
157
+ # @return [Integer] Value of the custom data field
158
+ def get_custom_data(name)
159
+ unless @custom_data_labels.include?(name)
160
+ PEROBS.log.fatal "Unknown custom data field #{name}"
161
+ end
162
+
163
+ @custom_data_values[@custom_data_labels.index(name)]
164
+ end
165
+
105
166
  # Erase the backing store. This method should only be called when the file
106
167
  # is not currently open.
107
168
  def erase
@@ -285,7 +346,7 @@ module PEROBS
285
346
 
286
347
  @first_space = offset
287
348
  @total_spaces += 1
288
- @total_entries -= 1
349
+ @total_entries -= 1 unless marker == 1
289
350
  write_header
290
351
 
291
352
  if offset == @f.size - 1 - @entry_bytes
@@ -298,12 +359,16 @@ module PEROBS
298
359
  # Check the file for logical errors.
299
360
  # @return [Boolean] true of file has no errors, false otherwise.
300
361
  def check
362
+ sync
363
+
301
364
  return false unless check_spaces
302
365
  return false unless check_entries
303
366
 
304
- if @f.size != HEADER_SIZE + (@total_entries + @total_spaces) *
305
- (1 + @entry_bytes)
306
- PEROBS.log.error "Size mismatch in EquiBlobsFile #{@file_name}"
367
+ expected_size = address_to_offset(@total_entries + @total_spaces + 1)
368
+ actual_size = @f.size
369
+ if actual_size != expected_size
370
+ PEROBS.log.error "Size mismatch in EquiBlobsFile #{@file_name}. " +
371
+ "Expected #{expected_size} bytes but found #{actual_size} bytes."
307
372
  return false
308
373
  end
309
374
 
@@ -326,6 +391,9 @@ module PEROBS
326
391
  @first_entry = @first_entry_default
327
392
  # The file offset of the first empty entry.
328
393
  @first_space = 0
394
+
395
+ # Copy default custom values
396
+ @custom_data_values = @custom_data_defaults.dup
329
397
  end
330
398
 
331
399
  def read_header
@@ -333,6 +401,12 @@ module PEROBS
333
401
  @f.seek(0)
334
402
  @total_entries, @total_spaces, @first_entry, @first_space =
335
403
  @f.read(HEADER_SIZE).unpack('QQQQ')
404
+ custom_labels_count = @custom_data_labels.length
405
+ if custom_labels_count > 0
406
+ @custom_data_values =
407
+ @f.read(custom_labels_count * 8).unpack("Q#{custom_labels_count}")
408
+ end
409
+
336
410
  rescue IOError => e
337
411
  PEROBS.log.fatal "Cannot read EquiBlobsFile header: #{e.message}"
338
412
  end
@@ -343,6 +417,10 @@ module PEROBS
343
417
  begin
344
418
  @f.seek(0)
345
419
  @f.write(header_ary.pack('QQQQ'))
420
+ unless @custom_data_values.empty?
421
+ @f.write(@custom_data_values.
422
+ pack("Q#{@custom_data_values.length}"))
423
+ end
346
424
  @f.flush
347
425
  end
348
426
  end
@@ -367,25 +445,31 @@ module PEROBS
367
445
  return false
368
446
  end
369
447
 
448
+ return true if next_offset == 0
449
+
370
450
  total_spaces = 0
371
- begin
372
- while next_offset != 0
373
- # Check that the marker byte is 0
374
- @f.seek(next_offset)
375
- if (marker = read_char) != 0
376
- PEROBS.log.error "Marker byte at address " +
377
- "#{offset_to_address(next_offset)} is #{marker} instead of 0."
378
- return false
379
- end
380
- # Read offset of next empty space
381
- next_offset = read_unsigned_int
451
+ @progressmeter.start("Checking #{@name} spaces list",
452
+ @total_spaces) do |pm|
453
+ begin
454
+ while next_offset != 0
455
+ # Check that the marker byte is 0
456
+ @f.seek(next_offset)
457
+ if (marker = read_char) != 0
458
+ PEROBS.log.error "Marker byte at address " +
459
+ "#{offset_to_address(next_offset)} is #{marker} instead of 0."
460
+ return false
461
+ end
462
+ # Read offset of next empty space
463
+ next_offset = read_unsigned_int
382
464
 
383
- total_spaces += 1
465
+ total_spaces += 1
466
+ pm.update(total_spaces)
467
+ end
468
+ rescue IOError => e
469
+ PEROBS.log.error "Cannot check space list of EquiBlobsFile " +
470
+ "#{@file_name}: #{e.message}"
471
+ return false
384
472
  end
385
- rescue IOError => e
386
- PEROBS.log.error "Cannot check space list of EquiBlobsFile " +
387
- "#{@file_name}: #{e.message}"
388
- return false
389
473
  end
390
474
 
391
475
  unless total_spaces == @total_spaces
@@ -414,36 +498,41 @@ module PEROBS
414
498
  return false
415
499
  end
416
500
 
417
- next_offset = HEADER_SIZE
501
+ next_offset = address_to_offset(1)
418
502
  total_entries = 0
419
503
  total_spaces = 0
420
- begin
421
- @f.seek(next_offset)
422
- while !@f.eof
423
- marker, bytes = @f.read(1 + @entry_bytes).
424
- unpack("C#{1 + @entry_bytes}")
425
- case marker
426
- when 0
427
- total_spaces += 1
428
- when 1
429
- PEROBS.log.error "Entry at address " +
430
- "#{offset_to_address(next_offset)} in EquiBlobsFile " +
431
- "#{@file_name} has reserved marker"
432
- return false
433
- when 2
434
- total_entries += 1
435
- else
436
- PEROBS.log.error "Entry at address " +
437
- "#{offset_to_address(next_offset)} in EquiBlobsFile " +
438
- "#{@file_name} has illegal marker #{marker}"
439
- return false
504
+ @progressmeter.start("Checking #{@name} entries",
505
+ @total_spaces + @total_entries) do |pm|
506
+ begin
507
+ @f.seek(next_offset)
508
+ while !@f.eof
509
+ marker, bytes = @f.read(1 + @entry_bytes).
510
+ unpack("C#{1 + @entry_bytes}")
511
+ case marker
512
+ when 0
513
+ total_spaces += 1
514
+ when 1
515
+ PEROBS.log.error "Entry at address " +
516
+ "#{offset_to_address(next_offset)} in EquiBlobsFile " +
517
+ "#{@file_name} has reserved marker"
518
+ return false
519
+ when 2
520
+ total_entries += 1
521
+ else
522
+ PEROBS.log.error "Entry at address " +
523
+ "#{offset_to_address(next_offset)} in EquiBlobsFile " +
524
+ "#{@file_name} has illegal marker #{marker}"
525
+ return false
526
+ end
527
+ next_offset += 1 + @entry_bytes
440
528
  end
441
- next_offset += 1 + @entry_bytes
529
+
530
+ pm.update(total_spaces + total_entries)
531
+ rescue
532
+ PEROBS.log.error "Cannot check entries of EquiBlobsFile " +
533
+ "#{@file_name}: #{e.message}"
534
+ return false
442
535
  end
443
- rescue
444
- PEROBS.log.error "Cannot check entries of EquiBlobsFile " +
445
- "#{@file_name}: #{e.message}"
446
- return false
447
536
  end
448
537
 
449
538
  unless total_spaces == @total_spaces
@@ -464,7 +553,7 @@ module PEROBS
464
553
 
465
554
  def trim_file
466
555
  offset = @f.size - 1 - @entry_bytes
467
- while offset >= HEADER_SIZE
556
+ while offset >= address_to_offset(1)
468
557
  @f.seek(offset)
469
558
  begin
470
559
  if (marker = read_char) == 0
@@ -548,12 +637,15 @@ module PEROBS
548
637
 
549
638
  # Translate a blob address to the actual offset in the file.
550
639
  def address_to_offset(address)
551
- HEADER_SIZE + (address - 1) * (1 + @entry_bytes)
640
+ # Since address 0 is illegal, we can use address - 1 as index here.
641
+ HEADER_SIZE + @custom_data_labels.length * 8 +
642
+ (address - 1) * (1 + @entry_bytes)
552
643
  end
553
644
 
554
645
  # Translate the file offset to the address of a blob.
555
646
  def offset_to_address(offset)
556
- (offset - HEADER_SIZE) / (1 + @entry_bytes) + 1
647
+ (offset - HEADER_SIZE - @custom_data_labels.length * 8) /
648
+ (1 + @entry_bytes) + 1
557
649
  end
558
650
 
559
651
  def write_char(c)