perobs 3.0.1 → 4.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. checksums.yaml +5 -5
  2. data/README.md +19 -18
  3. data/lib/perobs.rb +2 -0
  4. data/lib/perobs/Array.rb +68 -21
  5. data/lib/perobs/BTree.rb +110 -54
  6. data/lib/perobs/BTreeBlob.rb +14 -13
  7. data/lib/perobs/BTreeDB.rb +11 -10
  8. data/lib/perobs/BTreeNode.rb +551 -197
  9. data/lib/perobs/BTreeNodeCache.rb +10 -8
  10. data/lib/perobs/BTreeNodeLink.rb +11 -1
  11. data/lib/perobs/BigArray.rb +285 -0
  12. data/lib/perobs/BigArrayNode.rb +1002 -0
  13. data/lib/perobs/BigHash.rb +246 -0
  14. data/lib/perobs/BigTree.rb +197 -0
  15. data/lib/perobs/BigTreeNode.rb +873 -0
  16. data/lib/perobs/Cache.rb +47 -22
  17. data/lib/perobs/ClassMap.rb +2 -2
  18. data/lib/perobs/ConsoleProgressMeter.rb +61 -0
  19. data/lib/perobs/DataBase.rb +4 -3
  20. data/lib/perobs/DynamoDB.rb +62 -20
  21. data/lib/perobs/EquiBlobsFile.rb +174 -59
  22. data/lib/perobs/FNV_Hash_1a_64.rb +54 -0
  23. data/lib/perobs/FlatFile.rb +536 -242
  24. data/lib/perobs/FlatFileBlobHeader.rb +120 -84
  25. data/lib/perobs/FlatFileDB.rb +58 -27
  26. data/lib/perobs/FuzzyStringMatcher.rb +175 -0
  27. data/lib/perobs/Hash.rb +129 -35
  28. data/lib/perobs/IDList.rb +144 -0
  29. data/lib/perobs/IDListPage.rb +107 -0
  30. data/lib/perobs/IDListPageFile.rb +180 -0
  31. data/lib/perobs/IDListPageRecord.rb +142 -0
  32. data/lib/perobs/LockFile.rb +3 -0
  33. data/lib/perobs/Object.rb +28 -20
  34. data/lib/perobs/ObjectBase.rb +53 -10
  35. data/lib/perobs/PersistentObjectCache.rb +142 -0
  36. data/lib/perobs/PersistentObjectCacheLine.rb +99 -0
  37. data/lib/perobs/ProgressMeter.rb +97 -0
  38. data/lib/perobs/SpaceManager.rb +273 -0
  39. data/lib/perobs/SpaceTree.rb +63 -47
  40. data/lib/perobs/SpaceTreeNode.rb +134 -115
  41. data/lib/perobs/SpaceTreeNodeLink.rb +1 -1
  42. data/lib/perobs/StackFile.rb +1 -1
  43. data/lib/perobs/Store.rb +180 -70
  44. data/lib/perobs/version.rb +1 -1
  45. data/perobs.gemspec +4 -4
  46. data/test/Array_spec.rb +48 -39
  47. data/test/BTreeDB_spec.rb +2 -2
  48. data/test/BTree_spec.rb +50 -1
  49. data/test/BigArray_spec.rb +261 -0
  50. data/test/BigHash_spec.rb +152 -0
  51. data/test/BigTreeNode_spec.rb +153 -0
  52. data/test/BigTree_spec.rb +259 -0
  53. data/test/EquiBlobsFile_spec.rb +105 -5
  54. data/test/FNV_Hash_1a_64_spec.rb +59 -0
  55. data/test/FlatFileDB_spec.rb +199 -15
  56. data/test/FuzzyStringMatcher_spec.rb +261 -0
  57. data/test/Hash_spec.rb +27 -16
  58. data/test/IDList_spec.rb +77 -0
  59. data/test/LegacyDBs/LegacyDB.rb +155 -0
  60. data/test/LegacyDBs/version_3/class_map.json +1 -0
  61. data/test/LegacyDBs/version_3/config.json +1 -0
  62. data/test/LegacyDBs/version_3/database.blobs +0 -0
  63. data/test/LegacyDBs/version_3/database_spaces.blobs +0 -0
  64. data/test/LegacyDBs/version_3/index.blobs +0 -0
  65. data/test/LegacyDBs/version_3/version +1 -0
  66. data/test/LockFile_spec.rb +9 -6
  67. data/test/Object_spec.rb +5 -5
  68. data/test/SpaceManager_spec.rb +176 -0
  69. data/test/SpaceTree_spec.rb +27 -9
  70. data/test/Store_spec.rb +353 -206
  71. data/test/perobs_spec.rb +7 -3
  72. data/test/spec_helper.rb +9 -4
  73. metadata +59 -16
  74. data/lib/perobs/SpaceTreeNodeCache.rb +0 -76
  75. data/lib/perobs/TreeDB.rb +0 -277
data/lib/perobs/Cache.rb CHANGED
@@ -2,7 +2,7 @@
2
2
  #
3
3
  # = Cache.rb -- Persistent Ruby Object Store
4
4
  #
5
- # Copyright (c) 2015, 2016 by Chris Schlaeger <chris@taskjuggler.org>
5
+ # Copyright (c) 2015, 2016, 2019 by Chris Schlaeger <chris@taskjuggler.org>
6
6
  #
7
7
  # MIT License
8
8
  #
@@ -37,7 +37,7 @@ module PEROBS
37
37
  class Cache
38
38
 
39
39
  # Create a new Cache object.
40
- # @param bits [Fixnum] Number of bits for the cache index. This parameter
40
+ # @param bits [Integer] Number of bits for the cache index. This parameter
41
41
  # heavilty affects the performance and memory consumption of the
42
42
  # cache.
43
43
  def initialize(bits = 16)
@@ -66,10 +66,10 @@ module PEROBS
66
66
  def cache_write(obj)
67
67
  # This is just a safety check. It can probably be disabled in the future
68
68
  # to increase performance.
69
- if obj.respond_to?(:is_poxreference?)
70
- # If this condition triggers, we have a bug in the library.
71
- PEROBS.log.fatal "POXReference objects should never be cached"
72
- end
69
+ #if obj.respond_to?(:is_poxreference?)
70
+ # # If this condition triggers, we have a bug in the library.
71
+ # PEROBS.log.fatal "POXReference objects should never be cached"
72
+ #end
73
73
 
74
74
  if @transaction_stack.empty?
75
75
  # We are not in transaction mode.
@@ -93,22 +93,47 @@ module PEROBS
93
93
  end
94
94
  end
95
95
 
96
+ # Evict the object with the given ID from the cache.
97
+ # @param id [Integer] ID of the cached PEROBS::ObjectBase
98
+ # @return [True/False] True if object was stored in the cache. False
99
+ # otherwise.
100
+ def evict(id)
101
+ unless @transaction_stack.empty?
102
+ PEROBS.log.fatal "You cannot evict entries during a transaction."
103
+ end
104
+
105
+ idx = id & @mask
106
+ # The index is just a hash. We still need to check if the object IDs are
107
+ # actually the same before we can return the object.
108
+ if (obj = @writes[idx]) && obj._id == id
109
+ # The object is in the write cache.
110
+ @writes[idx] = nil
111
+ return true
112
+ elsif (obj = @reads[idx]) && obj._id == id
113
+ # The object is in the read cache.
114
+ @reads[idx] = nil
115
+ return true
116
+ end
117
+
118
+ false
119
+ end
120
+
96
121
  # Return the PEROBS::Object with the specified ID or nil if not found.
97
- # @param id [Fixnum or Bignum] ID of the cached PEROBS::ObjectBase
98
- #def object_by_id(id)
99
- # idx = id & @mask
100
- # # The index is just a hash. We still need to check if the object IDs are
101
- # # actually the same before we can return the object.
102
- # if (obj = @writes[idx]) && obj._id == id
103
- # # The object was in the write cache.
104
- # return obj
105
- # elsif (obj = @reads[idx]) && obj._id == id
106
- # # The object was in the read cache.
107
- # return obj
108
- # end
109
-
110
- # nil
111
- #end
122
+ # @param id [Integer] ID of the cached PEROBS::ObjectBase
123
+ def object_by_id(id)
124
+ idx = id & @mask
125
+ # The index is just a hash. We still need to check if the object IDs are
126
+ # actually the same before we can return the object.
127
+ if (obj = @writes[idx]) && obj._id == id
128
+ # The object was in the write cache.
129
+ return obj
130
+ elsif (obj = @reads[idx]) && obj._id == id
131
+ # The object was in the read cache.
132
+ return obj
133
+ end
134
+
135
+ nil
136
+ end
112
137
 
113
138
  # Flush all pending writes to the persistant storage back-end.
114
139
  def flush
@@ -160,7 +185,7 @@ module PEROBS
160
185
  transactions = @transaction_stack.pop
161
186
  # Merge the two lists
162
187
  @transaction_stack.push(@transaction_stack.pop + transactions)
163
- # Ensure that each object is only included once in the list.
188
+ # Ensure that each object ID is only included once in the list.
164
189
  @transaction_stack.last.uniq!
165
190
  end
166
191
  end
@@ -47,14 +47,14 @@ module PEROBS
47
47
 
48
48
  # Get the ID for a given class.
49
49
  # @param klass [String] Class
50
- # @return [Fixnum] ID. If klass is not yet known a new ID will be
50
+ # @return [Integer] ID. If klass is not yet known a new ID will be
51
51
  # allocated.
52
52
  def class_to_id(klass)
53
53
  @by_class[klass] || new_id(klass)
54
54
  end
55
55
 
56
56
  # Get the klass for a given ID.
57
- # @param id [Fixnum]
57
+ # @param id [Integer]
58
58
  # @return [String] String version of the class
59
59
  def id_to_class(id)
60
60
  @by_id[id]
@@ -0,0 +1,61 @@
1
+ # encoding: UTF-8
2
+ #
3
+ # = ConsoleProgressMeter.rb -- Persistent Ruby Object Store
4
+ #
5
+ # Copyright (c) 2018 by Chris Schlaeger <chris@taskjuggler.org>
6
+ #
7
+ # MIT License
8
+ #
9
+ # Permission is hereby granted, free of charge, to any person obtaining
10
+ # a copy of this software and associated documentation files (the
11
+ # "Software"), to deal in the Software without restriction, including
12
+ # without limitation the rights to use, copy, modify, merge, publish,
13
+ # distribute, sublicense, and/or sell copies of the Software, and to
14
+ # permit persons to whom the Software is furnished to do so, subject to
15
+ # the following conditions:
16
+ #
17
+ # The above copyright notice and this permission notice shall be
18
+ # included in all copies or substantial portions of the Software.
19
+ #
20
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
21
+ # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
22
+ # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
23
+ # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
24
+ # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
25
+ # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
26
+ # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
27
+
28
+ require 'perobs/ProgressMeter'
29
+
30
+ module PEROBS
31
+
32
+ class ConsoleProgressMeter < ProgressMeter
33
+
34
+ LINE_LENGTH = 79
35
+
36
+ private
37
+
38
+ def print_bar
39
+ percent = @max_value == 0 ? 100.0 :
40
+ (@current_value.to_f / @max_value) * 100.0
41
+ percent = 0.0 if percent < 0
42
+ percent = 100.0 if percent > 100.0
43
+
44
+ meter = "<#{percent.to_i}%>"
45
+
46
+ bar_length = LINE_LENGTH - @name.chars.length - 3 - meter.chars.length
47
+ left_bar = '*' * (bar_length * percent / 100.0)
48
+ right_bar = ' ' * (bar_length - left_bar.chars.length)
49
+
50
+ print "\r#{@name} [#{left_bar}#{meter}#{right_bar}]"
51
+ end
52
+
53
+ def print_time
54
+ s = "\r#{@name} [#{secsToHMS(@end_time - @start_time)}]"
55
+ puts s + (' ' * (LINE_LENGTH - s.chars.length + 1))
56
+ end
57
+
58
+ end
59
+
60
+ end
61
+
@@ -2,7 +2,7 @@
2
2
  #
3
3
  # = DataBase.rb -- Persistent Ruby Object Store
4
4
  #
5
- # Copyright (c) 2015 by Chris Schlaeger <chris@taskjuggler.org>
5
+ # Copyright (c) 2015, 2018, 2019 by Chris Schlaeger <chris@taskjuggler.org>
6
6
  #
7
7
  # MIT License
8
8
  #
@@ -42,8 +42,9 @@ module PEROBS
42
42
 
43
43
  # Create a new DataBase object. This method must be overwritten by the
44
44
  # deriving classes and then called via their constructor.
45
- def initialize(serializer = :json)
46
- @serializer = serializer
45
+ def initialize(options)
46
+ @serializer = options[:serializer] || :json
47
+ @progressmeter = options[:progressmeter] || ProgressMeter.new
47
48
  @config = {}
48
49
  end
49
50
 
@@ -2,7 +2,8 @@
2
2
  #
3
3
  # = DynamoDB.rb -- Persistent Ruby Object Store
4
4
  #
5
- # Copyright (c) 2015, 2016 by Chris Schlaeger <chris@taskjuggler.org>
5
+ # Copyright (c) 2015, 2016, 2017, 2018
6
+ # by Chris Schlaeger <chris@taskjuggler.org>
6
7
  #
7
8
  # MIT License
8
9
  #
@@ -25,7 +26,7 @@
25
26
  # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
26
27
  # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
27
28
 
28
- require 'aws-sdk-core'
29
+ require 'aws-sdk-dynamodb'
29
30
 
30
31
  require 'perobs/DataBase'
31
32
  require 'perobs/BTreeBlob'
@@ -35,6 +36,10 @@ module PEROBS
35
36
  # This class implements an Amazon DynamoDB storage engine for PEROBS.
36
37
  class DynamoDB < DataBase
37
38
 
39
+ INTERNAL_ITEMS = %w( config item_counter )
40
+
41
+ attr_reader :item_counter
42
+
38
43
  # Create a new DynamoDB object.
39
44
  # @param db_name [String] name of the DB directory
40
45
  # @param options [Hash] options to customize the behavior. Currently only
@@ -50,7 +55,7 @@ module PEROBS
50
55
  options[:serializer] = :yaml
51
56
  end
52
57
 
53
- super(options[:serializer] || :json)
58
+ super(options)
54
59
 
55
60
  if options.include?(:aws_id) && options.include?(:aws_key)
56
61
  Aws.config[:credentials] = Aws::Credentials.new(options[:aws_id],
@@ -62,12 +67,25 @@ module PEROBS
62
67
 
63
68
  @dynamodb = Aws::DynamoDB::Client.new
64
69
  @table_name = db_name
65
- ensure_table_exists(@table_name)
70
+ @config = nil
71
+ # The number of items currently stored in the DB.
72
+ @item_counter = nil
73
+ if create_table(@table_name)
74
+ @config = { 'serializer' => @serializer }
75
+ put_hash('config', @config)
76
+ @item_counter = 0
77
+ dynamo_put_item('item_counter', @item_counter.to_s)
78
+ else
79
+ @config = get_hash('config')
80
+ if @config['serializer'] != @serializer
81
+ raise ArgumentError, "DynamoDB #{@table_name} was created with " +
82
+ "serializer #{@config['serializer']} but was now opened with " +
83
+ "serializer #{@serializer}."
84
+ end
85
+ @item_counter = dynamo_get_item('item_counter').to_i
86
+ end
66
87
 
67
88
  # Read the existing DB config.
68
- @config = get_hash('config')
69
- check_option('serializer')
70
- put_hash('config', @config)
71
89
  end
72
90
 
73
91
  # Delete the entire database. The database is no longer usable after this
@@ -85,7 +103,7 @@ module PEROBS
85
103
  end
86
104
 
87
105
  # Return true if the object with given ID exists
88
- # @param id [Fixnum or Bignum]
106
+ # @param id [Integer]
89
107
  def include?(id)
90
108
  !dynamo_get_item(id.to_s).nil?
91
109
  end
@@ -112,11 +130,18 @@ module PEROBS
112
130
  # Store the given object into the cluster files.
113
131
  # @param obj [Hash] Object as defined by PEROBS::ObjectBase
114
132
  def put_object(obj, id)
133
+ id_str = id.to_s
134
+ unless dynamo_get_item(id_str)
135
+ # The is no object with this ID yet. Increase the item counter.
136
+ @item_counter += 1
137
+ dynamo_put_item('item_counter', @item_counter.to_s)
138
+ end
139
+
115
140
  dynamo_put_item(id.to_s, serialize(obj))
116
141
  end
117
142
 
118
143
  # Load the given object from the filesystem.
119
- # @param id [Fixnum or Bignum] object ID
144
+ # @param id [Integer] object ID
120
145
  # @return [Hash] Object as defined by PEROBS::ObjectBase or nil if ID does
121
146
  # not exist
122
147
  def get_object(id)
@@ -128,33 +153,33 @@ module PEROBS
128
153
  each_item do |id|
129
154
  dynamo_mark_item(id, false)
130
155
  end
131
- # Mark the 'config' item so it will not get deleted.
132
- dynamo_mark_item('config')
133
156
  end
134
157
 
135
158
  # Permanently delete all objects that have not been marked. Those are
136
159
  # orphaned and are no longer referenced by any actively used object.
137
- # @return [Array] List of object IDs of the deleted objects.
160
+ # @return [Integer] Count of the deleted objects.
138
161
  def delete_unmarked_objects
139
- deleted_ids = []
162
+ deleted_objects_count = 0
140
163
  each_item do |id|
141
164
  unless dynamo_is_marked?(id)
142
165
  dynamo_delete_item(id)
143
- deleted_ids << id
166
+ deleted_objects_count += 1
167
+ @item_counter -= 1
144
168
  end
145
169
  end
170
+ dynamo_put_item('item_counter', @item_counter.to_s)
146
171
 
147
- deleted_ids
172
+ deleted_objects_count
148
173
  end
149
174
 
150
175
  # Mark an object.
151
- # @param id [Fixnum or Bignum] ID of the object to mark
176
+ # @param id [Integer] ID of the object to mark
152
177
  def mark(id)
153
178
  dynamo_mark_item(id.to_s, true)
154
179
  end
155
180
 
156
181
  # Check if the object is marked.
157
- # @param id [Fixnum or Bignum] ID of the object to check
182
+ # @param id [Integer] ID of the object to check
158
183
  def is_marked?(id)
159
184
  dynamo_is_marked?(id.to_s)
160
185
  end
@@ -163,11 +188,21 @@ module PEROBS
163
188
  # @param repair [TrueClass/FalseClass] True if found errors should be
164
189
  # repaired.
165
190
  def check_db(repair = false)
166
- # TODO: See if we can add checks here
191
+ unless (item_counter = dynamo_get_item('item_counter')) &&
192
+ item_counter == @item_counter
193
+ PEROBS.log.error "@item_counter variable (#{@item_counter}) and " +
194
+ "item_counter table entry (#{item_counter}) don't match"
195
+ end
196
+ item_counter = 0
197
+ each_item { item_counter += 1 }
198
+ unless item_counter == @item_counter
199
+ PEROBS.log.error "Table contains #{item_counter} items but " +
200
+ "@item_counter is #{@item_counter}"
201
+ end
167
202
  end
168
203
 
169
204
  # Check if the stored object is syntactically correct.
170
- # @param id [Fixnum/Bignum] Object ID
205
+ # @param id [Integer] Object ID
171
206
  # @param repair [TrueClass/FalseClass] True if an repair attempt should be
172
207
  # made.
173
208
  # @return [TrueClass/FalseClass] True if the object is OK, otherwise
@@ -185,9 +220,11 @@ module PEROBS
185
220
 
186
221
  private
187
222
 
188
- def ensure_table_exists(table_name)
223
+ def create_table(table_name)
189
224
  begin
190
225
  @dynamodb.describe_table(:table_name => table_name)
226
+ # The table exists already. No need to create it.
227
+ return false
191
228
  rescue Aws::DynamoDB::Errors::ResourceNotFoundException
192
229
  @dynamodb.create_table(
193
230
  :table_name => table_name,
@@ -210,6 +247,8 @@ module PEROBS
210
247
  )
211
248
 
212
249
  @dynamodb.wait_until(:table_exists, table_name: table_name)
250
+ # The table was successfully created.
251
+ return true
213
252
  end
214
253
  end
215
254
 
@@ -251,6 +290,9 @@ module PEROBS
251
290
  break if resp.count <= 0
252
291
 
253
292
  resp.items.each do |item|
293
+ # Skip all internal items
294
+ next if INTERNAL_ITEMS.include?(item['Id'])
295
+
254
296
  yield(item['Id'])
255
297
  end
256
298
 
@@ -2,7 +2,8 @@
2
2
  #
3
3
  # = EquiBlobsFile.rb -- Persistent Ruby Object Store
4
4
  #
5
- # Copyright (c) 2016, 2017 by Chris Schlaeger <chris@taskjuggler.org>
5
+ # Copyright (c) 2016, 2017, 2018, 2019
6
+ # by Chris Schlaeger <chris@taskjuggler.org>
6
7
  #
7
8
  # MIT License
8
9
  #
@@ -26,6 +27,7 @@
26
27
  # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
27
28
 
28
29
  require 'perobs/Log'
30
+ require 'perobs/ProgressMeter'
29
31
 
30
32
  module PEROBS
31
33
 
@@ -37,6 +39,11 @@ module PEROBS
37
39
  # is used to represent an undefined address or nil. The file has a 4 * 8
38
40
  # bytes long header that stores the total entry count, the total space
39
41
  # count, the offset of the first entry and the offset of the first space.
42
+ # The header is followed by a custom entry section. Each entry is also 8
43
+ # bytes long. After the custom entry section the data blobs start. Each data
44
+ # blob starts with a mark byte that indicates if the blob is valid data (2),
45
+ # a free space (0) or reseved space (1). Then it is followed by @entry_bytes
46
+ # number of bytes for the data blob.
40
47
  class EquiBlobsFile
41
48
 
42
49
  TOTAL_ENTRIES_OFFSET = 0
@@ -45,20 +52,26 @@ module PEROBS
45
52
  FIRST_SPACE_OFFSET = 3 * 8
46
53
  HEADER_SIZE = 4 * 8
47
54
 
48
- attr_reader :total_entries, :total_spaces, :file_name
49
- attr_accessor :first_entry
55
+ attr_reader :total_entries, :total_spaces, :file_name, :first_entry
50
56
 
51
57
  # Create a new stack file in the given directory with the given file name.
52
58
  # @param dir [String] Directory
53
59
  # @param name [String] File name
54
- # @param entry_bytes [Fixnum] Number of bytes each entry must have
55
- def initialize(dir, name, entry_bytes, first_entry_default = 0)
60
+ # @param progressmeter [ProgressMeter] Reference to a progress meter
61
+ # object
62
+ # @param entry_bytes [Integer] Number of bytes each entry must have
63
+ # @param first_entry_default [Integer] Default address of the first blob
64
+ def initialize(dir, name, progressmeter, entry_bytes,
65
+ first_entry_default = 0)
66
+ @name = name
56
67
  @file_name = File.join(dir, name + '.blobs')
68
+ @progressmeter = progressmeter
57
69
  if entry_bytes < 8
58
70
  PEROBS.log.fatal "EquiBlobsFile entry size must be at least 8"
59
71
  end
60
72
  @entry_bytes = entry_bytes
61
73
  @first_entry_default = first_entry_default
74
+ clear_custom_data
62
75
  reset_counters
63
76
 
64
77
  # The File handle.
@@ -83,6 +96,7 @@ module PEROBS
83
96
  unless @f.flock(File::LOCK_NB | File::LOCK_EX)
84
97
  PEROBS.log.fatal 'Database blob file is locked by another process'
85
98
  end
99
+ @f.sync = true
86
100
  end
87
101
 
88
102
  # Close the blob file. This method must be called before the program is
@@ -92,6 +106,7 @@ module PEROBS
92
106
  if @f
93
107
  @f.flush
94
108
  @f.flock(File::LOCK_UN)
109
+ @f.fsync
95
110
  @f.close
96
111
  @f = nil
97
112
  end
@@ -100,10 +115,61 @@ module PEROBS
100
115
  end
101
116
  end
102
117
 
118
+ # In addition to the standard offsets for the first entry and the first
119
+ # space any number of additional data fields can be registered. This must be
120
+ # done right after the object is instanciated and before the open() method
121
+ # is called. Each field represents a 64 bit unsigned integer.
122
+ # @param name [String] The label for this offset
123
+ # @param default_value [Integer] The default value for the offset
124
+ def register_custom_data(name, default_value = 0)
125
+ if @custom_data_labels.include?(name)
126
+ PEROBS.log.fatal "Custom data field #{name} has already been registered"
127
+ end
128
+
129
+ @custom_data_labels << name
130
+ @custom_data_values << default_value
131
+ @custom_data_defaults << default_value
132
+ end
133
+
134
+ # Reset (delete) all custom data labels that have been registered.
135
+ def clear_custom_data
136
+ unless @f.nil?
137
+ PEROBS.log.fatal "clear_custom_data should only be called when " +
138
+ "the file is not opened"
139
+ end
140
+
141
+ @custom_data_labels = []
142
+ @custom_data_values = []
143
+ @custom_data_defaults = []
144
+ end
145
+
146
+ # Set the registered custom data field to the given value.
147
+ # @param name [String] Label of the offset
148
+ # @param value [Integer] Value
149
+ def set_custom_data(name, value)
150
+ unless @custom_data_labels.include?(name)
151
+ PEROBS.log.fatal "Unknown custom data field #{name}"
152
+ end
153
+
154
+ @custom_data_values[@custom_data_labels.index(name)] = value
155
+ write_header if @f
156
+ end
157
+
158
+ # Get the registered custom data field value.
159
+ # @param name [String] Label of the offset
160
+ # @return [Integer] Value of the custom data field
161
+ def get_custom_data(name)
162
+ unless @custom_data_labels.include?(name)
163
+ PEROBS.log.fatal "Unknown custom data field #{name}"
164
+ end
165
+
166
+ @custom_data_values[@custom_data_labels.index(name)]
167
+ end
168
+
103
169
  # Erase the backing store. This method should only be called when the file
104
170
  # is not currently open.
105
171
  def erase
106
- PEROBS.log.fatal 'Cannot call EquiBlobsFile::erase while it is open' if @f
172
+ @f = nil
107
173
  File.delete(@file_name) if File.exist?(@file_name)
108
174
  reset_counters
109
175
  end
@@ -111,7 +177,10 @@ module PEROBS
111
177
  # Flush out all unwritten data.
112
178
  def sync
113
179
  begin
114
- @f.flush if @f
180
+ if @f
181
+ @f.flush
182
+ @f.fsync
183
+ end
115
184
  rescue IOError => e
116
185
  PEROBS.log.fatal "Cannot sync blob file #{@file_name}: #{e.message}"
117
186
  end
@@ -125,9 +194,16 @@ module PEROBS
125
194
  write_header
126
195
  end
127
196
 
197
+ # Change the address of the first blob.
198
+ # @param address [Integer] New address
199
+ def first_entry=(address)
200
+ @first_entry = address
201
+ write_header
202
+ end
203
+
128
204
  # Return the address of a free blob storage space. Addresses start at 0
129
205
  # and increase linearly.
130
- # @return [Fixnum] address of a free blob space
206
+ # @return [Integer] address of a free blob space
131
207
  def free_address
132
208
  if @first_space == 0
133
209
  # There is currently no free entry. Create a new reserved entry at the
@@ -170,7 +246,7 @@ module PEROBS
170
246
 
171
247
  # Store the given byte blob at the specified address. If the blob space is
172
248
  # already in use the content will be overwritten.
173
- # @param address [Fixnum] Address to store the blob
249
+ # @param address [Integer] Address to store the blob
174
250
  # @param bytes [String] bytes to store
175
251
  def store_blob(address, bytes)
176
252
  unless address >= 0
@@ -217,7 +293,7 @@ module PEROBS
217
293
  end
218
294
 
219
295
  # Retrieve a blob from the given address.
220
- # @param address [Fixnum] Address to store the blob
296
+ # @param address [Integer] Address to store the blob
221
297
  # @return [String] blob bytes
222
298
  def retrieve_blob(address)
223
299
  unless address > 0
@@ -248,7 +324,7 @@ module PEROBS
248
324
  end
249
325
 
250
326
  # Delete the blob at the given address.
251
- # @param address [Fixnum] Address of blob to delete
327
+ # @param address [Integer] Address of blob to delete
252
328
  def delete_blob(address)
253
329
  unless address >= 0
254
330
  PEROBS.log.fatal "Blob address must be larger than 0, " +
@@ -273,7 +349,7 @@ module PEROBS
273
349
 
274
350
  @first_space = offset
275
351
  @total_spaces += 1
276
- @total_entries -= 1
352
+ @total_entries -= 1 unless marker == 1
277
353
  write_header
278
354
 
279
355
  if offset == @f.size - 1 - @entry_bytes
@@ -286,12 +362,16 @@ module PEROBS
286
362
  # Check the file for logical errors.
287
363
  # @return [Boolean] true of file has no errors, false otherwise.
288
364
  def check
365
+ sync
366
+
289
367
  return false unless check_spaces
290
368
  return false unless check_entries
291
369
 
292
- if @f.size != HEADER_SIZE + (@total_entries + @total_spaces) *
293
- (1 + @entry_bytes)
294
- PEROBS.log.error "Size mismatch in EquiBlobsFile #{@file_name}"
370
+ expected_size = address_to_offset(@total_entries + @total_spaces + 1)
371
+ actual_size = @f.size
372
+ if actual_size != expected_size
373
+ PEROBS.log.error "Size mismatch in EquiBlobsFile #{@file_name}. " +
374
+ "Expected #{expected_size} bytes but found #{actual_size} bytes."
295
375
  return false
296
376
  end
297
377
 
@@ -314,6 +394,9 @@ module PEROBS
314
394
  @first_entry = @first_entry_default
315
395
  # The file offset of the first empty entry.
316
396
  @first_space = 0
397
+
398
+ # Copy default custom values
399
+ @custom_data_values = @custom_data_defaults.dup
317
400
  end
318
401
 
319
402
  def read_header
@@ -321,6 +404,12 @@ module PEROBS
321
404
  @f.seek(0)
322
405
  @total_entries, @total_spaces, @first_entry, @first_space =
323
406
  @f.read(HEADER_SIZE).unpack('QQQQ')
407
+ custom_labels_count = @custom_data_labels.length
408
+ if custom_labels_count > 0
409
+ @custom_data_values =
410
+ @f.read(custom_labels_count * 8).unpack("Q#{custom_labels_count}")
411
+ end
412
+
324
413
  rescue IOError => e
325
414
  PEROBS.log.fatal "Cannot read EquiBlobsFile header: #{e.message}"
326
415
  end
@@ -331,6 +420,10 @@ module PEROBS
331
420
  begin
332
421
  @f.seek(0)
333
422
  @f.write(header_ary.pack('QQQQ'))
423
+ unless @custom_data_values.empty?
424
+ @f.write(@custom_data_values.
425
+ pack("Q#{@custom_data_values.length}"))
426
+ end
334
427
  @f.flush
335
428
  end
336
429
  end
@@ -355,25 +448,31 @@ module PEROBS
355
448
  return false
356
449
  end
357
450
 
451
+ return true if next_offset == 0
452
+
358
453
  total_spaces = 0
359
- begin
360
- while next_offset != 0
361
- # Check that the marker byte is 0
362
- @f.seek(next_offset)
363
- if (marker = read_char) != 0
364
- PEROBS.log.error "Marker byte at address " +
365
- "#{offset_to_address(next_offset)} is #{marker} instead of 0."
366
- return false
367
- end
368
- # Read offset of next empty space
369
- next_offset = read_unsigned_int
454
+ @progressmeter.start("Checking #{@name} spaces list",
455
+ @total_spaces) do |pm|
456
+ begin
457
+ while next_offset != 0
458
+ # Check that the marker byte is 0
459
+ @f.seek(next_offset)
460
+ if (marker = read_char) != 0
461
+ PEROBS.log.error "Marker byte at address " +
462
+ "#{offset_to_address(next_offset)} is #{marker} instead of 0."
463
+ return false
464
+ end
465
+ # Read offset of next empty space
466
+ next_offset = read_unsigned_int
370
467
 
371
- total_spaces += 1
468
+ total_spaces += 1
469
+ pm.update(total_spaces)
470
+ end
471
+ rescue IOError => e
472
+ PEROBS.log.error "Cannot check space list of EquiBlobsFile " +
473
+ "#{@file_name}: #{e.message}"
474
+ return false
372
475
  end
373
- rescue IOError => e
374
- PEROBS.log.error "Cannot check space list of EquiBlobsFile " +
375
- "#{@file_name}: #{e.message}"
376
- return false
377
476
  end
378
477
 
379
478
  unless total_spaces == @total_spaces
@@ -402,35 +501,48 @@ module PEROBS
402
501
  return false
403
502
  end
404
503
 
405
- next_offset = HEADER_SIZE
504
+ next_offset = address_to_offset(1)
406
505
  total_entries = 0
407
506
  total_spaces = 0
408
- begin
409
- @f.seek(next_offset)
410
- while !@f.eof
411
- marker, bytes = @f.read(1 + @entry_bytes).
412
- unpack("C#{1 + @entry_bytes}")
413
- case marker
414
- when 0
415
- total_spaces += 1
416
- when 1
417
- PEROBS.log.error "Entry at address " +
418
- "#{offset_to_address(next_offset)} in EquiBlobsFile " +
419
- "#{@file_name} has reserved marker"
420
- return false
421
- when 2
422
- total_entries += 1
423
- else
424
- PEROBS.log.error "Entry at address " +
425
- "#{offset_to_address(next_offset)} in EquiBlobsFile " +
426
- "#{@file_name} has illegal marker #{marker}"
427
- return false
507
+ last_entry_is_space = false
508
+ @progressmeter.start("Checking #{@name} entries",
509
+ @total_spaces + @total_entries) do |pm|
510
+ begin
511
+ @f.seek(next_offset)
512
+ while !@f.eof
513
+ marker, bytes = @f.read(1 + @entry_bytes).
514
+ unpack("C#{1 + @entry_bytes}")
515
+ case marker
516
+ when 0
517
+ total_spaces += 1
518
+ last_entry_is_space = true
519
+ when 1
520
+ PEROBS.log.error "Entry at address " +
521
+ "#{offset_to_address(next_offset)} in EquiBlobsFile " +
522
+ "#{@file_name} has reserved marker"
523
+ return false
524
+ when 2
525
+ total_entries += 1
526
+ last_entry_is_space = false
527
+ else
528
+ PEROBS.log.error "Entry at address " +
529
+ "#{offset_to_address(next_offset)} in EquiBlobsFile " +
530
+ "#{@file_name} has illegal marker #{marker}"
531
+ return false
532
+ end
533
+ next_offset += 1 + @entry_bytes
428
534
  end
429
- next_offset += 1 + @entry_bytes
535
+
536
+ pm.update(total_spaces + total_entries)
537
+ rescue
538
+ PEROBS.log.error "Cannot check entries of EquiBlobsFile " +
539
+ "#{@file_name}: #{e.message}"
540
+ return false
430
541
  end
431
- rescue
432
- PEROBS.log.error "Cannot check entries of EquiBlobsFile " +
433
- "#{@file_name}: #{e.message}"
542
+ end
543
+
544
+ if last_entry_is_space
545
+ PEROBS.log.error "EquiBlobsFile #{@file_name} is not properly trimmed"
434
546
  return false
435
547
  end
436
548
 
@@ -452,7 +564,7 @@ module PEROBS
452
564
 
453
565
  def trim_file
454
566
  offset = @f.size - 1 - @entry_bytes
455
- while offset >= HEADER_SIZE
567
+ while offset >= address_to_offset(1)
456
568
  @f.seek(offset)
457
569
  begin
458
570
  if (marker = read_char) == 0
@@ -536,12 +648,15 @@ module PEROBS
536
648
 
537
649
  # Translate a blob address to the actual offset in the file.
538
650
  def address_to_offset(address)
539
- HEADER_SIZE + (address - 1) * (1 + @entry_bytes)
651
+ # Since address 0 is illegal, we can use address - 1 as index here.
652
+ HEADER_SIZE + @custom_data_labels.length * 8 +
653
+ (address - 1) * (1 + @entry_bytes)
540
654
  end
541
655
 
542
656
  # Translate the file offset to the address of a blob.
543
657
  def offset_to_address(offset)
544
- (offset - HEADER_SIZE) / (1 + @entry_bytes) + 1
658
+ (offset - HEADER_SIZE - @custom_data_labels.length * 8) /
659
+ (1 + @entry_bytes) + 1
545
660
  end
546
661
 
547
662
  def write_char(c)