perobs 4.0.0 → 4.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/perobs.rb +1 -0
- data/lib/perobs/Array.rb +66 -19
- data/lib/perobs/BTree.rb +83 -12
- data/lib/perobs/BTreeBlob.rb +1 -1
- data/lib/perobs/BTreeDB.rb +2 -2
- data/lib/perobs/BTreeNode.rb +365 -85
- data/lib/perobs/BigArray.rb +267 -0
- data/lib/perobs/BigArrayNode.rb +998 -0
- data/lib/perobs/BigHash.rb +262 -0
- data/lib/perobs/BigTree.rb +184 -0
- data/lib/perobs/BigTreeNode.rb +873 -0
- data/lib/perobs/ConsoleProgressMeter.rb +61 -0
- data/lib/perobs/DataBase.rb +4 -3
- data/lib/perobs/DynamoDB.rb +57 -15
- data/lib/perobs/EquiBlobsFile.rb +143 -51
- data/lib/perobs/FNV_Hash_1a_64.rb +54 -0
- data/lib/perobs/FlatFile.rb +363 -203
- data/lib/perobs/FlatFileBlobHeader.rb +98 -54
- data/lib/perobs/FlatFileDB.rb +42 -20
- data/lib/perobs/Hash.rb +58 -13
- data/lib/perobs/IDList.rb +144 -0
- data/lib/perobs/IDListPage.rb +107 -0
- data/lib/perobs/IDListPageFile.rb +180 -0
- data/lib/perobs/IDListPageRecord.rb +142 -0
- data/lib/perobs/Object.rb +18 -15
- data/lib/perobs/ObjectBase.rb +38 -4
- data/lib/perobs/PersistentObjectCache.rb +53 -67
- data/lib/perobs/PersistentObjectCacheLine.rb +24 -12
- data/lib/perobs/ProgressMeter.rb +97 -0
- data/lib/perobs/SpaceTree.rb +21 -12
- data/lib/perobs/SpaceTreeNode.rb +53 -61
- data/lib/perobs/Store.rb +71 -32
- data/lib/perobs/version.rb +1 -1
- data/perobs.gemspec +4 -4
- data/test/Array_spec.rb +15 -6
- data/test/BTree_spec.rb +5 -2
- data/test/BigArray_spec.rb +214 -0
- data/test/BigHash_spec.rb +144 -0
- data/test/BigTreeNode_spec.rb +153 -0
- data/test/BigTree_spec.rb +259 -0
- data/test/EquiBlobsFile_spec.rb +105 -1
- data/test/FNV_Hash_1a_64_spec.rb +59 -0
- data/test/FlatFileDB_spec.rb +63 -14
- data/test/Hash_spec.rb +1 -2
- data/test/IDList_spec.rb +77 -0
- data/test/LegacyDBs/LegacyDB.rb +151 -0
- data/test/LegacyDBs/version_3/class_map.json +1 -0
- data/test/LegacyDBs/version_3/config.json +1 -0
- data/test/LegacyDBs/version_3/database.blobs +0 -0
- data/test/LegacyDBs/version_3/database_spaces.blobs +0 -0
- data/test/LegacyDBs/version_3/index.blobs +0 -0
- data/test/LegacyDBs/version_3/version +1 -0
- data/test/LockFile_spec.rb +9 -6
- data/test/SpaceTree_spec.rb +4 -1
- data/test/Store_spec.rb +290 -199
- data/test/spec_helper.rb +9 -4
- metadata +47 -10
- data/lib/perobs/TreeDB.rb +0 -277
@@ -0,0 +1,61 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
#
|
3
|
+
# = ConsoleProgressMeter.rb -- Persistent Ruby Object Store
|
4
|
+
#
|
5
|
+
# Copyright (c) 2018 by Chris Schlaeger <chris@taskjuggler.org>
|
6
|
+
#
|
7
|
+
# MIT License
|
8
|
+
#
|
9
|
+
# Permission is hereby granted, free of charge, to any person obtaining
|
10
|
+
# a copy of this software and associated documentation files (the
|
11
|
+
# "Software"), to deal in the Software without restriction, including
|
12
|
+
# without limitation the rights to use, copy, modify, merge, publish,
|
13
|
+
# distribute, sublicense, and/or sell copies of the Software, and to
|
14
|
+
# permit persons to whom the Software is furnished to do so, subject to
|
15
|
+
# the following conditions:
|
16
|
+
#
|
17
|
+
# The above copyright notice and this permission notice shall be
|
18
|
+
# included in all copies or substantial portions of the Software.
|
19
|
+
#
|
20
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
21
|
+
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
22
|
+
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
23
|
+
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
24
|
+
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
25
|
+
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
26
|
+
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
27
|
+
|
28
|
+
require 'perobs/ProgressMeter'
|
29
|
+
|
30
|
+
module PEROBS
|
31
|
+
|
32
|
+
class ConsoleProgressMeter < ProgressMeter
|
33
|
+
|
34
|
+
LINE_LENGTH = 79
|
35
|
+
|
36
|
+
private
|
37
|
+
|
38
|
+
def print_bar
|
39
|
+
percent = @max_value == 0 ? 100.0 :
|
40
|
+
(@current_value.to_f / @max_value) * 100.0
|
41
|
+
percent = 0.0 if percent < 0
|
42
|
+
percent = 100.0 if percent > 100.0
|
43
|
+
|
44
|
+
meter = "<#{percent.to_i}%>"
|
45
|
+
|
46
|
+
bar_length = LINE_LENGTH - @name.chars.length - 3 - meter.chars.length
|
47
|
+
left_bar = '*' * (bar_length * percent / 100.0)
|
48
|
+
right_bar = ' ' * (bar_length - left_bar.chars.length)
|
49
|
+
|
50
|
+
print "\r#{@name} [#{left_bar}#{meter}#{right_bar}]"
|
51
|
+
end
|
52
|
+
|
53
|
+
def print_time
|
54
|
+
s = "\r#{@name} [#{secsToHMS(@end_time - @start_time)}]"
|
55
|
+
puts s + (' ' * (LINE_LENGTH - s.chars.length + 1))
|
56
|
+
end
|
57
|
+
|
58
|
+
end
|
59
|
+
|
60
|
+
end
|
61
|
+
|
data/lib/perobs/DataBase.rb
CHANGED
@@ -2,7 +2,7 @@
|
|
2
2
|
#
|
3
3
|
# = DataBase.rb -- Persistent Ruby Object Store
|
4
4
|
#
|
5
|
-
# Copyright (c) 2015 by Chris Schlaeger <chris@taskjuggler.org>
|
5
|
+
# Copyright (c) 2015, 2018, 2019 by Chris Schlaeger <chris@taskjuggler.org>
|
6
6
|
#
|
7
7
|
# MIT License
|
8
8
|
#
|
@@ -42,8 +42,9 @@ module PEROBS
|
|
42
42
|
|
43
43
|
# Create a new DataBase object. This method must be overwritten by the
|
44
44
|
# deriving classes and then called via their constructor.
|
45
|
-
def initialize(
|
46
|
-
@serializer = serializer
|
45
|
+
def initialize(options)
|
46
|
+
@serializer = options[:serializer] || :json
|
47
|
+
@progressmeter = options[:progressmeter] || ProgressMeter.new
|
47
48
|
@config = {}
|
48
49
|
end
|
49
50
|
|
data/lib/perobs/DynamoDB.rb
CHANGED
@@ -2,7 +2,8 @@
|
|
2
2
|
#
|
3
3
|
# = DynamoDB.rb -- Persistent Ruby Object Store
|
4
4
|
#
|
5
|
-
# Copyright (c) 2015, 2016
|
5
|
+
# Copyright (c) 2015, 2016, 2017, 2018
|
6
|
+
# by Chris Schlaeger <chris@taskjuggler.org>
|
6
7
|
#
|
7
8
|
# MIT License
|
8
9
|
#
|
@@ -25,7 +26,7 @@
|
|
25
26
|
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
26
27
|
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
27
28
|
|
28
|
-
require 'aws-sdk'
|
29
|
+
require 'aws-sdk-dynamodb'
|
29
30
|
|
30
31
|
require 'perobs/DataBase'
|
31
32
|
require 'perobs/BTreeBlob'
|
@@ -35,6 +36,10 @@ module PEROBS
|
|
35
36
|
# This class implements an Amazon DynamoDB storage engine for PEROBS.
|
36
37
|
class DynamoDB < DataBase
|
37
38
|
|
39
|
+
INTERNAL_ITEMS = %w( config item_counter )
|
40
|
+
|
41
|
+
attr_reader :item_counter
|
42
|
+
|
38
43
|
# Create a new DynamoDB object.
|
39
44
|
# @param db_name [String] name of the DB directory
|
40
45
|
# @param options [Hash] options to customize the behavior. Currently only
|
@@ -50,7 +55,7 @@ module PEROBS
|
|
50
55
|
options[:serializer] = :yaml
|
51
56
|
end
|
52
57
|
|
53
|
-
super(options
|
58
|
+
super(options)
|
54
59
|
|
55
60
|
if options.include?(:aws_id) && options.include?(:aws_key)
|
56
61
|
Aws.config[:credentials] = Aws::Credentials.new(options[:aws_id],
|
@@ -62,12 +67,25 @@ module PEROBS
|
|
62
67
|
|
63
68
|
@dynamodb = Aws::DynamoDB::Client.new
|
64
69
|
@table_name = db_name
|
65
|
-
|
70
|
+
@config = nil
|
71
|
+
# The number of items currently stored in the DB.
|
72
|
+
@item_counter = nil
|
73
|
+
if create_table(@table_name)
|
74
|
+
@config = { 'serializer' => @serializer }
|
75
|
+
put_hash('config', @config)
|
76
|
+
@item_counter = 0
|
77
|
+
dynamo_put_item('item_counter', @item_counter.to_s)
|
78
|
+
else
|
79
|
+
@config = get_hash('config')
|
80
|
+
if @config['serializer'] != @serializer
|
81
|
+
raise ArgumentError, "DynamoDB #{@table_name} was created with " +
|
82
|
+
"serializer #{@config['serializer']} but was now opened with " +
|
83
|
+
"serializer #{@serializer}."
|
84
|
+
end
|
85
|
+
@item_counter = dynamo_get_item('item_counter').to_i
|
86
|
+
end
|
66
87
|
|
67
88
|
# Read the existing DB config.
|
68
|
-
@config = get_hash('config')
|
69
|
-
check_option('serializer')
|
70
|
-
put_hash('config', @config)
|
71
89
|
end
|
72
90
|
|
73
91
|
# Delete the entire database. The database is no longer usable after this
|
@@ -112,6 +130,13 @@ module PEROBS
|
|
112
130
|
# Store the given object into the cluster files.
|
113
131
|
# @param obj [Hash] Object as defined by PEROBS::ObjectBase
|
114
132
|
def put_object(obj, id)
|
133
|
+
id_str = id.to_s
|
134
|
+
unless dynamo_get_item(id_str)
|
135
|
+
# The is no object with this ID yet. Increase the item counter.
|
136
|
+
@item_counter += 1
|
137
|
+
dynamo_put_item('item_counter', @item_counter.to_s)
|
138
|
+
end
|
139
|
+
|
115
140
|
dynamo_put_item(id.to_s, serialize(obj))
|
116
141
|
end
|
117
142
|
|
@@ -128,23 +153,23 @@ module PEROBS
|
|
128
153
|
each_item do |id|
|
129
154
|
dynamo_mark_item(id, false)
|
130
155
|
end
|
131
|
-
# Mark the 'config' item so it will not get deleted.
|
132
|
-
dynamo_mark_item('config')
|
133
156
|
end
|
134
157
|
|
135
158
|
# Permanently delete all objects that have not been marked. Those are
|
136
159
|
# orphaned and are no longer referenced by any actively used object.
|
137
|
-
# @return [
|
160
|
+
# @return [Integer] Count of the deleted objects.
|
138
161
|
def delete_unmarked_objects
|
139
|
-
|
162
|
+
deleted_objects_count = 0
|
140
163
|
each_item do |id|
|
141
164
|
unless dynamo_is_marked?(id)
|
142
165
|
dynamo_delete_item(id)
|
143
|
-
|
166
|
+
deleted_objects_count += 1
|
167
|
+
@item_counter -= 1
|
144
168
|
end
|
145
169
|
end
|
170
|
+
dynamo_put_item('item_counter', @item_counter.to_s)
|
146
171
|
|
147
|
-
|
172
|
+
deleted_objects_count
|
148
173
|
end
|
149
174
|
|
150
175
|
# Mark an object.
|
@@ -163,7 +188,17 @@ module PEROBS
|
|
163
188
|
# @param repair [TrueClass/FalseClass] True if found errors should be
|
164
189
|
# repaired.
|
165
190
|
def check_db(repair = false)
|
166
|
-
|
191
|
+
unless (item_counter = dynamo_get_item('item_counter')) &&
|
192
|
+
item_counter == @item_counter
|
193
|
+
PEROBS.log.error "@item_counter variable (#{@item_counter}) and " +
|
194
|
+
"item_counter table entry (#{item_counter}) don't match"
|
195
|
+
end
|
196
|
+
item_counter = 0
|
197
|
+
each_item { item_counter += 1 }
|
198
|
+
unless item_counter == @item_counter
|
199
|
+
PEROBS.log.error "Table contains #{item_counter} items but " +
|
200
|
+
"@item_counter is #{@item_counter}"
|
201
|
+
end
|
167
202
|
end
|
168
203
|
|
169
204
|
# Check if the stored object is syntactically correct.
|
@@ -185,9 +220,11 @@ module PEROBS
|
|
185
220
|
|
186
221
|
private
|
187
222
|
|
188
|
-
def
|
223
|
+
def create_table(table_name)
|
189
224
|
begin
|
190
225
|
@dynamodb.describe_table(:table_name => table_name)
|
226
|
+
# The table exists already. No need to create it.
|
227
|
+
return false
|
191
228
|
rescue Aws::DynamoDB::Errors::ResourceNotFoundException
|
192
229
|
@dynamodb.create_table(
|
193
230
|
:table_name => table_name,
|
@@ -210,6 +247,8 @@ module PEROBS
|
|
210
247
|
)
|
211
248
|
|
212
249
|
@dynamodb.wait_until(:table_exists, table_name: table_name)
|
250
|
+
# The table was successfully created.
|
251
|
+
return true
|
213
252
|
end
|
214
253
|
end
|
215
254
|
|
@@ -251,6 +290,9 @@ module PEROBS
|
|
251
290
|
break if resp.count <= 0
|
252
291
|
|
253
292
|
resp.items.each do |item|
|
293
|
+
# Skip all internal items
|
294
|
+
next if INTERNAL_ITEMS.include?(item['Id'])
|
295
|
+
|
254
296
|
yield(item['Id'])
|
255
297
|
end
|
256
298
|
|
data/lib/perobs/EquiBlobsFile.rb
CHANGED
@@ -2,7 +2,8 @@
|
|
2
2
|
#
|
3
3
|
# = EquiBlobsFile.rb -- Persistent Ruby Object Store
|
4
4
|
#
|
5
|
-
# Copyright (c) 2016, 2017
|
5
|
+
# Copyright (c) 2016, 2017, 2018, 2019
|
6
|
+
# by Chris Schlaeger <chris@taskjuggler.org>
|
6
7
|
#
|
7
8
|
# MIT License
|
8
9
|
#
|
@@ -26,6 +27,7 @@
|
|
26
27
|
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
27
28
|
|
28
29
|
require 'perobs/Log'
|
30
|
+
require 'perobs/ProgressMeter'
|
29
31
|
|
30
32
|
module PEROBS
|
31
33
|
|
@@ -37,6 +39,8 @@ module PEROBS
|
|
37
39
|
# is used to represent an undefined address or nil. The file has a 4 * 8
|
38
40
|
# bytes long header that stores the total entry count, the total space
|
39
41
|
# count, the offset of the first entry and the offset of the first space.
|
42
|
+
# The header is followed by a custom entry section. Each entry is also 8
|
43
|
+
# bytes long. After the custom entry section the data blobs start.
|
40
44
|
class EquiBlobsFile
|
41
45
|
|
42
46
|
TOTAL_ENTRIES_OFFSET = 0
|
@@ -50,15 +54,21 @@ module PEROBS
|
|
50
54
|
# Create a new stack file in the given directory with the given file name.
|
51
55
|
# @param dir [String] Directory
|
52
56
|
# @param name [String] File name
|
57
|
+
# @param progressmeter [ProgressMeter] Reference to a progress meter
|
58
|
+
# object
|
53
59
|
# @param entry_bytes [Integer] Number of bytes each entry must have
|
54
60
|
# @param first_entry_default [Integer] Default address of the first blob
|
55
|
-
def initialize(dir, name, entry_bytes,
|
61
|
+
def initialize(dir, name, progressmeter, entry_bytes,
|
62
|
+
first_entry_default = 0)
|
63
|
+
@name = name
|
56
64
|
@file_name = File.join(dir, name + '.blobs')
|
65
|
+
@progressmeter = progressmeter
|
57
66
|
if entry_bytes < 8
|
58
67
|
PEROBS.log.fatal "EquiBlobsFile entry size must be at least 8"
|
59
68
|
end
|
60
69
|
@entry_bytes = entry_bytes
|
61
70
|
@first_entry_default = first_entry_default
|
71
|
+
clear_custom_data
|
62
72
|
reset_counters
|
63
73
|
|
64
74
|
# The File handle.
|
@@ -102,6 +112,57 @@ module PEROBS
|
|
102
112
|
end
|
103
113
|
end
|
104
114
|
|
115
|
+
# In addition to the standard offsets for the first entry and the first
|
116
|
+
# space any number of additional data fields can be registered. This must be
|
117
|
+
# done right after the object is instanciated and before the open() method
|
118
|
+
# is called. Each field represents a 64 bit unsigned integer.
|
119
|
+
# @param name [String] The label for this offset
|
120
|
+
# @param default_value [Integer] The default value for the offset
|
121
|
+
def register_custom_data(name, default_value = 0)
|
122
|
+
if @custom_data_labels.include?(name)
|
123
|
+
PEROBS.log.fatal "Custom data field #{name} has already been registered"
|
124
|
+
end
|
125
|
+
|
126
|
+
@custom_data_labels << name
|
127
|
+
@custom_data_values << default_value
|
128
|
+
@custom_data_defaults << default_value
|
129
|
+
end
|
130
|
+
|
131
|
+
# Reset (delete) all custom data labels that have been registered.
|
132
|
+
def clear_custom_data
|
133
|
+
unless @f.nil?
|
134
|
+
PEROBS.log.fatal "clear_custom_data should only be called when " +
|
135
|
+
"the file is not opened"
|
136
|
+
end
|
137
|
+
|
138
|
+
@custom_data_labels = []
|
139
|
+
@custom_data_values = []
|
140
|
+
@custom_data_defaults = []
|
141
|
+
end
|
142
|
+
|
143
|
+
# Set the registered custom data field to the given value.
|
144
|
+
# @param name [String] Label of the offset
|
145
|
+
# @param value [Integer] Value
|
146
|
+
def set_custom_data(name, value)
|
147
|
+
unless @custom_data_labels.include?(name)
|
148
|
+
PEROBS.log.fatal "Unknown custom data field #{name}"
|
149
|
+
end
|
150
|
+
|
151
|
+
@custom_data_values[@custom_data_labels.index(name)] = value
|
152
|
+
write_header if @f
|
153
|
+
end
|
154
|
+
|
155
|
+
# Get the registered custom data field value.
|
156
|
+
# @param name [String] Label of the offset
|
157
|
+
# @return [Integer] Value of the custom data field
|
158
|
+
def get_custom_data(name)
|
159
|
+
unless @custom_data_labels.include?(name)
|
160
|
+
PEROBS.log.fatal "Unknown custom data field #{name}"
|
161
|
+
end
|
162
|
+
|
163
|
+
@custom_data_values[@custom_data_labels.index(name)]
|
164
|
+
end
|
165
|
+
|
105
166
|
# Erase the backing store. This method should only be called when the file
|
106
167
|
# is not currently open.
|
107
168
|
def erase
|
@@ -285,7 +346,7 @@ module PEROBS
|
|
285
346
|
|
286
347
|
@first_space = offset
|
287
348
|
@total_spaces += 1
|
288
|
-
@total_entries -= 1
|
349
|
+
@total_entries -= 1 unless marker == 1
|
289
350
|
write_header
|
290
351
|
|
291
352
|
if offset == @f.size - 1 - @entry_bytes
|
@@ -298,12 +359,16 @@ module PEROBS
|
|
298
359
|
# Check the file for logical errors.
|
299
360
|
# @return [Boolean] true of file has no errors, false otherwise.
|
300
361
|
def check
|
362
|
+
sync
|
363
|
+
|
301
364
|
return false unless check_spaces
|
302
365
|
return false unless check_entries
|
303
366
|
|
304
|
-
|
305
|
-
|
306
|
-
|
367
|
+
expected_size = address_to_offset(@total_entries + @total_spaces + 1)
|
368
|
+
actual_size = @f.size
|
369
|
+
if actual_size != expected_size
|
370
|
+
PEROBS.log.error "Size mismatch in EquiBlobsFile #{@file_name}. " +
|
371
|
+
"Expected #{expected_size} bytes but found #{actual_size} bytes."
|
307
372
|
return false
|
308
373
|
end
|
309
374
|
|
@@ -326,6 +391,9 @@ module PEROBS
|
|
326
391
|
@first_entry = @first_entry_default
|
327
392
|
# The file offset of the first empty entry.
|
328
393
|
@first_space = 0
|
394
|
+
|
395
|
+
# Copy default custom values
|
396
|
+
@custom_data_values = @custom_data_defaults.dup
|
329
397
|
end
|
330
398
|
|
331
399
|
def read_header
|
@@ -333,6 +401,12 @@ module PEROBS
|
|
333
401
|
@f.seek(0)
|
334
402
|
@total_entries, @total_spaces, @first_entry, @first_space =
|
335
403
|
@f.read(HEADER_SIZE).unpack('QQQQ')
|
404
|
+
custom_labels_count = @custom_data_labels.length
|
405
|
+
if custom_labels_count > 0
|
406
|
+
@custom_data_values =
|
407
|
+
@f.read(custom_labels_count * 8).unpack("Q#{custom_labels_count}")
|
408
|
+
end
|
409
|
+
|
336
410
|
rescue IOError => e
|
337
411
|
PEROBS.log.fatal "Cannot read EquiBlobsFile header: #{e.message}"
|
338
412
|
end
|
@@ -343,6 +417,10 @@ module PEROBS
|
|
343
417
|
begin
|
344
418
|
@f.seek(0)
|
345
419
|
@f.write(header_ary.pack('QQQQ'))
|
420
|
+
unless @custom_data_values.empty?
|
421
|
+
@f.write(@custom_data_values.
|
422
|
+
pack("Q#{@custom_data_values.length}"))
|
423
|
+
end
|
346
424
|
@f.flush
|
347
425
|
end
|
348
426
|
end
|
@@ -367,25 +445,31 @@ module PEROBS
|
|
367
445
|
return false
|
368
446
|
end
|
369
447
|
|
448
|
+
return true if next_offset == 0
|
449
|
+
|
370
450
|
total_spaces = 0
|
371
|
-
|
372
|
-
|
373
|
-
|
374
|
-
|
375
|
-
|
376
|
-
|
377
|
-
|
378
|
-
|
379
|
-
|
380
|
-
|
381
|
-
|
451
|
+
@progressmeter.start("Checking #{@name} spaces list",
|
452
|
+
@total_spaces) do |pm|
|
453
|
+
begin
|
454
|
+
while next_offset != 0
|
455
|
+
# Check that the marker byte is 0
|
456
|
+
@f.seek(next_offset)
|
457
|
+
if (marker = read_char) != 0
|
458
|
+
PEROBS.log.error "Marker byte at address " +
|
459
|
+
"#{offset_to_address(next_offset)} is #{marker} instead of 0."
|
460
|
+
return false
|
461
|
+
end
|
462
|
+
# Read offset of next empty space
|
463
|
+
next_offset = read_unsigned_int
|
382
464
|
|
383
|
-
|
465
|
+
total_spaces += 1
|
466
|
+
pm.update(total_spaces)
|
467
|
+
end
|
468
|
+
rescue IOError => e
|
469
|
+
PEROBS.log.error "Cannot check space list of EquiBlobsFile " +
|
470
|
+
"#{@file_name}: #{e.message}"
|
471
|
+
return false
|
384
472
|
end
|
385
|
-
rescue IOError => e
|
386
|
-
PEROBS.log.error "Cannot check space list of EquiBlobsFile " +
|
387
|
-
"#{@file_name}: #{e.message}"
|
388
|
-
return false
|
389
473
|
end
|
390
474
|
|
391
475
|
unless total_spaces == @total_spaces
|
@@ -414,36 +498,41 @@ module PEROBS
|
|
414
498
|
return false
|
415
499
|
end
|
416
500
|
|
417
|
-
next_offset =
|
501
|
+
next_offset = address_to_offset(1)
|
418
502
|
total_entries = 0
|
419
503
|
total_spaces = 0
|
420
|
-
|
421
|
-
|
422
|
-
|
423
|
-
|
424
|
-
|
425
|
-
|
426
|
-
|
427
|
-
|
428
|
-
|
429
|
-
|
430
|
-
|
431
|
-
"
|
432
|
-
|
433
|
-
|
434
|
-
|
435
|
-
|
436
|
-
|
437
|
-
|
438
|
-
"
|
439
|
-
|
504
|
+
@progressmeter.start("Checking #{@name} entries",
|
505
|
+
@total_spaces + @total_entries) do |pm|
|
506
|
+
begin
|
507
|
+
@f.seek(next_offset)
|
508
|
+
while !@f.eof
|
509
|
+
marker, bytes = @f.read(1 + @entry_bytes).
|
510
|
+
unpack("C#{1 + @entry_bytes}")
|
511
|
+
case marker
|
512
|
+
when 0
|
513
|
+
total_spaces += 1
|
514
|
+
when 1
|
515
|
+
PEROBS.log.error "Entry at address " +
|
516
|
+
"#{offset_to_address(next_offset)} in EquiBlobsFile " +
|
517
|
+
"#{@file_name} has reserved marker"
|
518
|
+
return false
|
519
|
+
when 2
|
520
|
+
total_entries += 1
|
521
|
+
else
|
522
|
+
PEROBS.log.error "Entry at address " +
|
523
|
+
"#{offset_to_address(next_offset)} in EquiBlobsFile " +
|
524
|
+
"#{@file_name} has illegal marker #{marker}"
|
525
|
+
return false
|
526
|
+
end
|
527
|
+
next_offset += 1 + @entry_bytes
|
440
528
|
end
|
441
|
-
|
529
|
+
|
530
|
+
pm.update(total_spaces + total_entries)
|
531
|
+
rescue
|
532
|
+
PEROBS.log.error "Cannot check entries of EquiBlobsFile " +
|
533
|
+
"#{@file_name}: #{e.message}"
|
534
|
+
return false
|
442
535
|
end
|
443
|
-
rescue
|
444
|
-
PEROBS.log.error "Cannot check entries of EquiBlobsFile " +
|
445
|
-
"#{@file_name}: #{e.message}"
|
446
|
-
return false
|
447
536
|
end
|
448
537
|
|
449
538
|
unless total_spaces == @total_spaces
|
@@ -464,7 +553,7 @@ module PEROBS
|
|
464
553
|
|
465
554
|
def trim_file
|
466
555
|
offset = @f.size - 1 - @entry_bytes
|
467
|
-
while offset >=
|
556
|
+
while offset >= address_to_offset(1)
|
468
557
|
@f.seek(offset)
|
469
558
|
begin
|
470
559
|
if (marker = read_char) == 0
|
@@ -548,12 +637,15 @@ module PEROBS
|
|
548
637
|
|
549
638
|
# Translate a blob address to the actual offset in the file.
|
550
639
|
def address_to_offset(address)
|
551
|
-
|
640
|
+
# Since address 0 is illegal, we can use address - 1 as index here.
|
641
|
+
HEADER_SIZE + @custom_data_labels.length * 8 +
|
642
|
+
(address - 1) * (1 + @entry_bytes)
|
552
643
|
end
|
553
644
|
|
554
645
|
# Translate the file offset to the address of a blob.
|
555
646
|
def offset_to_address(offset)
|
556
|
-
(offset - HEADER_SIZE
|
647
|
+
(offset - HEADER_SIZE - @custom_data_labels.length * 8) /
|
648
|
+
(1 + @entry_bytes) + 1
|
557
649
|
end
|
558
650
|
|
559
651
|
def write_char(c)
|