perobs 3.0.1 → 4.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/README.md +19 -18
- data/lib/perobs.rb +2 -0
- data/lib/perobs/Array.rb +68 -21
- data/lib/perobs/BTree.rb +110 -54
- data/lib/perobs/BTreeBlob.rb +14 -13
- data/lib/perobs/BTreeDB.rb +11 -10
- data/lib/perobs/BTreeNode.rb +551 -197
- data/lib/perobs/BTreeNodeCache.rb +10 -8
- data/lib/perobs/BTreeNodeLink.rb +11 -1
- data/lib/perobs/BigArray.rb +285 -0
- data/lib/perobs/BigArrayNode.rb +1002 -0
- data/lib/perobs/BigHash.rb +246 -0
- data/lib/perobs/BigTree.rb +197 -0
- data/lib/perobs/BigTreeNode.rb +873 -0
- data/lib/perobs/Cache.rb +47 -22
- data/lib/perobs/ClassMap.rb +2 -2
- data/lib/perobs/ConsoleProgressMeter.rb +61 -0
- data/lib/perobs/DataBase.rb +4 -3
- data/lib/perobs/DynamoDB.rb +62 -20
- data/lib/perobs/EquiBlobsFile.rb +174 -59
- data/lib/perobs/FNV_Hash_1a_64.rb +54 -0
- data/lib/perobs/FlatFile.rb +536 -242
- data/lib/perobs/FlatFileBlobHeader.rb +120 -84
- data/lib/perobs/FlatFileDB.rb +58 -27
- data/lib/perobs/FuzzyStringMatcher.rb +175 -0
- data/lib/perobs/Hash.rb +129 -35
- data/lib/perobs/IDList.rb +144 -0
- data/lib/perobs/IDListPage.rb +107 -0
- data/lib/perobs/IDListPageFile.rb +180 -0
- data/lib/perobs/IDListPageRecord.rb +142 -0
- data/lib/perobs/LockFile.rb +3 -0
- data/lib/perobs/Object.rb +28 -20
- data/lib/perobs/ObjectBase.rb +53 -10
- data/lib/perobs/PersistentObjectCache.rb +142 -0
- data/lib/perobs/PersistentObjectCacheLine.rb +99 -0
- data/lib/perobs/ProgressMeter.rb +97 -0
- data/lib/perobs/SpaceManager.rb +273 -0
- data/lib/perobs/SpaceTree.rb +63 -47
- data/lib/perobs/SpaceTreeNode.rb +134 -115
- data/lib/perobs/SpaceTreeNodeLink.rb +1 -1
- data/lib/perobs/StackFile.rb +1 -1
- data/lib/perobs/Store.rb +180 -70
- data/lib/perobs/version.rb +1 -1
- data/perobs.gemspec +4 -4
- data/test/Array_spec.rb +48 -39
- data/test/BTreeDB_spec.rb +2 -2
- data/test/BTree_spec.rb +50 -1
- data/test/BigArray_spec.rb +261 -0
- data/test/BigHash_spec.rb +152 -0
- data/test/BigTreeNode_spec.rb +153 -0
- data/test/BigTree_spec.rb +259 -0
- data/test/EquiBlobsFile_spec.rb +105 -5
- data/test/FNV_Hash_1a_64_spec.rb +59 -0
- data/test/FlatFileDB_spec.rb +199 -15
- data/test/FuzzyStringMatcher_spec.rb +261 -0
- data/test/Hash_spec.rb +27 -16
- data/test/IDList_spec.rb +77 -0
- data/test/LegacyDBs/LegacyDB.rb +155 -0
- data/test/LegacyDBs/version_3/class_map.json +1 -0
- data/test/LegacyDBs/version_3/config.json +1 -0
- data/test/LegacyDBs/version_3/database.blobs +0 -0
- data/test/LegacyDBs/version_3/database_spaces.blobs +0 -0
- data/test/LegacyDBs/version_3/index.blobs +0 -0
- data/test/LegacyDBs/version_3/version +1 -0
- data/test/LockFile_spec.rb +9 -6
- data/test/Object_spec.rb +5 -5
- data/test/SpaceManager_spec.rb +176 -0
- data/test/SpaceTree_spec.rb +27 -9
- data/test/Store_spec.rb +353 -206
- data/test/perobs_spec.rb +7 -3
- data/test/spec_helper.rb +9 -4
- metadata +59 -16
- data/lib/perobs/SpaceTreeNodeCache.rb +0 -76
- data/lib/perobs/TreeDB.rb +0 -277
data/lib/perobs/Cache.rb
CHANGED
@@ -2,7 +2,7 @@
|
|
2
2
|
#
|
3
3
|
# = Cache.rb -- Persistent Ruby Object Store
|
4
4
|
#
|
5
|
-
# Copyright (c) 2015, 2016 by Chris Schlaeger <chris@taskjuggler.org>
|
5
|
+
# Copyright (c) 2015, 2016, 2019 by Chris Schlaeger <chris@taskjuggler.org>
|
6
6
|
#
|
7
7
|
# MIT License
|
8
8
|
#
|
@@ -37,7 +37,7 @@ module PEROBS
|
|
37
37
|
class Cache
|
38
38
|
|
39
39
|
# Create a new Cache object.
|
40
|
-
# @param bits [
|
40
|
+
# @param bits [Integer] Number of bits for the cache index. This parameter
|
41
41
|
# heavilty affects the performance and memory consumption of the
|
42
42
|
# cache.
|
43
43
|
def initialize(bits = 16)
|
@@ -66,10 +66,10 @@ module PEROBS
|
|
66
66
|
def cache_write(obj)
|
67
67
|
# This is just a safety check. It can probably be disabled in the future
|
68
68
|
# to increase performance.
|
69
|
-
if obj.respond_to?(:is_poxreference?)
|
70
|
-
|
71
|
-
|
72
|
-
end
|
69
|
+
#if obj.respond_to?(:is_poxreference?)
|
70
|
+
# # If this condition triggers, we have a bug in the library.
|
71
|
+
# PEROBS.log.fatal "POXReference objects should never be cached"
|
72
|
+
#end
|
73
73
|
|
74
74
|
if @transaction_stack.empty?
|
75
75
|
# We are not in transaction mode.
|
@@ -93,22 +93,47 @@ module PEROBS
|
|
93
93
|
end
|
94
94
|
end
|
95
95
|
|
96
|
+
# Evict the object with the given ID from the cache.
|
97
|
+
# @param id [Integer] ID of the cached PEROBS::ObjectBase
|
98
|
+
# @return [True/False] True if object was stored in the cache. False
|
99
|
+
# otherwise.
|
100
|
+
def evict(id)
|
101
|
+
unless @transaction_stack.empty?
|
102
|
+
PEROBS.log.fatal "You cannot evict entries during a transaction."
|
103
|
+
end
|
104
|
+
|
105
|
+
idx = id & @mask
|
106
|
+
# The index is just a hash. We still need to check if the object IDs are
|
107
|
+
# actually the same before we can return the object.
|
108
|
+
if (obj = @writes[idx]) && obj._id == id
|
109
|
+
# The object is in the write cache.
|
110
|
+
@writes[idx] = nil
|
111
|
+
return true
|
112
|
+
elsif (obj = @reads[idx]) && obj._id == id
|
113
|
+
# The object is in the read cache.
|
114
|
+
@reads[idx] = nil
|
115
|
+
return true
|
116
|
+
end
|
117
|
+
|
118
|
+
false
|
119
|
+
end
|
120
|
+
|
96
121
|
# Return the PEROBS::Object with the specified ID or nil if not found.
|
97
|
-
# @param id [
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
122
|
+
# @param id [Integer] ID of the cached PEROBS::ObjectBase
|
123
|
+
def object_by_id(id)
|
124
|
+
idx = id & @mask
|
125
|
+
# The index is just a hash. We still need to check if the object IDs are
|
126
|
+
# actually the same before we can return the object.
|
127
|
+
if (obj = @writes[idx]) && obj._id == id
|
128
|
+
# The object was in the write cache.
|
129
|
+
return obj
|
130
|
+
elsif (obj = @reads[idx]) && obj._id == id
|
131
|
+
# The object was in the read cache.
|
132
|
+
return obj
|
133
|
+
end
|
134
|
+
|
135
|
+
nil
|
136
|
+
end
|
112
137
|
|
113
138
|
# Flush all pending writes to the persistant storage back-end.
|
114
139
|
def flush
|
@@ -160,7 +185,7 @@ module PEROBS
|
|
160
185
|
transactions = @transaction_stack.pop
|
161
186
|
# Merge the two lists
|
162
187
|
@transaction_stack.push(@transaction_stack.pop + transactions)
|
163
|
-
# Ensure that each object is only included once in the list.
|
188
|
+
# Ensure that each object ID is only included once in the list.
|
164
189
|
@transaction_stack.last.uniq!
|
165
190
|
end
|
166
191
|
end
|
data/lib/perobs/ClassMap.rb
CHANGED
@@ -47,14 +47,14 @@ module PEROBS
|
|
47
47
|
|
48
48
|
# Get the ID for a given class.
|
49
49
|
# @param klass [String] Class
|
50
|
-
# @return [
|
50
|
+
# @return [Integer] ID. If klass is not yet known a new ID will be
|
51
51
|
# allocated.
|
52
52
|
def class_to_id(klass)
|
53
53
|
@by_class[klass] || new_id(klass)
|
54
54
|
end
|
55
55
|
|
56
56
|
# Get the klass for a given ID.
|
57
|
-
# @param id [
|
57
|
+
# @param id [Integer]
|
58
58
|
# @return [String] String version of the class
|
59
59
|
def id_to_class(id)
|
60
60
|
@by_id[id]
|
@@ -0,0 +1,61 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
#
|
3
|
+
# = ConsoleProgressMeter.rb -- Persistent Ruby Object Store
|
4
|
+
#
|
5
|
+
# Copyright (c) 2018 by Chris Schlaeger <chris@taskjuggler.org>
|
6
|
+
#
|
7
|
+
# MIT License
|
8
|
+
#
|
9
|
+
# Permission is hereby granted, free of charge, to any person obtaining
|
10
|
+
# a copy of this software and associated documentation files (the
|
11
|
+
# "Software"), to deal in the Software without restriction, including
|
12
|
+
# without limitation the rights to use, copy, modify, merge, publish,
|
13
|
+
# distribute, sublicense, and/or sell copies of the Software, and to
|
14
|
+
# permit persons to whom the Software is furnished to do so, subject to
|
15
|
+
# the following conditions:
|
16
|
+
#
|
17
|
+
# The above copyright notice and this permission notice shall be
|
18
|
+
# included in all copies or substantial portions of the Software.
|
19
|
+
#
|
20
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
21
|
+
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
22
|
+
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
23
|
+
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
24
|
+
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
25
|
+
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
26
|
+
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
27
|
+
|
28
|
+
require 'perobs/ProgressMeter'
|
29
|
+
|
30
|
+
module PEROBS
|
31
|
+
|
32
|
+
class ConsoleProgressMeter < ProgressMeter
|
33
|
+
|
34
|
+
LINE_LENGTH = 79
|
35
|
+
|
36
|
+
private
|
37
|
+
|
38
|
+
def print_bar
|
39
|
+
percent = @max_value == 0 ? 100.0 :
|
40
|
+
(@current_value.to_f / @max_value) * 100.0
|
41
|
+
percent = 0.0 if percent < 0
|
42
|
+
percent = 100.0 if percent > 100.0
|
43
|
+
|
44
|
+
meter = "<#{percent.to_i}%>"
|
45
|
+
|
46
|
+
bar_length = LINE_LENGTH - @name.chars.length - 3 - meter.chars.length
|
47
|
+
left_bar = '*' * (bar_length * percent / 100.0)
|
48
|
+
right_bar = ' ' * (bar_length - left_bar.chars.length)
|
49
|
+
|
50
|
+
print "\r#{@name} [#{left_bar}#{meter}#{right_bar}]"
|
51
|
+
end
|
52
|
+
|
53
|
+
def print_time
|
54
|
+
s = "\r#{@name} [#{secsToHMS(@end_time - @start_time)}]"
|
55
|
+
puts s + (' ' * (LINE_LENGTH - s.chars.length + 1))
|
56
|
+
end
|
57
|
+
|
58
|
+
end
|
59
|
+
|
60
|
+
end
|
61
|
+
|
data/lib/perobs/DataBase.rb
CHANGED
@@ -2,7 +2,7 @@
|
|
2
2
|
#
|
3
3
|
# = DataBase.rb -- Persistent Ruby Object Store
|
4
4
|
#
|
5
|
-
# Copyright (c) 2015 by Chris Schlaeger <chris@taskjuggler.org>
|
5
|
+
# Copyright (c) 2015, 2018, 2019 by Chris Schlaeger <chris@taskjuggler.org>
|
6
6
|
#
|
7
7
|
# MIT License
|
8
8
|
#
|
@@ -42,8 +42,9 @@ module PEROBS
|
|
42
42
|
|
43
43
|
# Create a new DataBase object. This method must be overwritten by the
|
44
44
|
# deriving classes and then called via their constructor.
|
45
|
-
def initialize(
|
46
|
-
@serializer = serializer
|
45
|
+
def initialize(options)
|
46
|
+
@serializer = options[:serializer] || :json
|
47
|
+
@progressmeter = options[:progressmeter] || ProgressMeter.new
|
47
48
|
@config = {}
|
48
49
|
end
|
49
50
|
|
data/lib/perobs/DynamoDB.rb
CHANGED
@@ -2,7 +2,8 @@
|
|
2
2
|
#
|
3
3
|
# = DynamoDB.rb -- Persistent Ruby Object Store
|
4
4
|
#
|
5
|
-
# Copyright (c) 2015, 2016
|
5
|
+
# Copyright (c) 2015, 2016, 2017, 2018
|
6
|
+
# by Chris Schlaeger <chris@taskjuggler.org>
|
6
7
|
#
|
7
8
|
# MIT License
|
8
9
|
#
|
@@ -25,7 +26,7 @@
|
|
25
26
|
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
26
27
|
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
27
28
|
|
28
|
-
require 'aws-sdk-
|
29
|
+
require 'aws-sdk-dynamodb'
|
29
30
|
|
30
31
|
require 'perobs/DataBase'
|
31
32
|
require 'perobs/BTreeBlob'
|
@@ -35,6 +36,10 @@ module PEROBS
|
|
35
36
|
# This class implements an Amazon DynamoDB storage engine for PEROBS.
|
36
37
|
class DynamoDB < DataBase
|
37
38
|
|
39
|
+
INTERNAL_ITEMS = %w( config item_counter )
|
40
|
+
|
41
|
+
attr_reader :item_counter
|
42
|
+
|
38
43
|
# Create a new DynamoDB object.
|
39
44
|
# @param db_name [String] name of the DB directory
|
40
45
|
# @param options [Hash] options to customize the behavior. Currently only
|
@@ -50,7 +55,7 @@ module PEROBS
|
|
50
55
|
options[:serializer] = :yaml
|
51
56
|
end
|
52
57
|
|
53
|
-
super(options
|
58
|
+
super(options)
|
54
59
|
|
55
60
|
if options.include?(:aws_id) && options.include?(:aws_key)
|
56
61
|
Aws.config[:credentials] = Aws::Credentials.new(options[:aws_id],
|
@@ -62,12 +67,25 @@ module PEROBS
|
|
62
67
|
|
63
68
|
@dynamodb = Aws::DynamoDB::Client.new
|
64
69
|
@table_name = db_name
|
65
|
-
|
70
|
+
@config = nil
|
71
|
+
# The number of items currently stored in the DB.
|
72
|
+
@item_counter = nil
|
73
|
+
if create_table(@table_name)
|
74
|
+
@config = { 'serializer' => @serializer }
|
75
|
+
put_hash('config', @config)
|
76
|
+
@item_counter = 0
|
77
|
+
dynamo_put_item('item_counter', @item_counter.to_s)
|
78
|
+
else
|
79
|
+
@config = get_hash('config')
|
80
|
+
if @config['serializer'] != @serializer
|
81
|
+
raise ArgumentError, "DynamoDB #{@table_name} was created with " +
|
82
|
+
"serializer #{@config['serializer']} but was now opened with " +
|
83
|
+
"serializer #{@serializer}."
|
84
|
+
end
|
85
|
+
@item_counter = dynamo_get_item('item_counter').to_i
|
86
|
+
end
|
66
87
|
|
67
88
|
# Read the existing DB config.
|
68
|
-
@config = get_hash('config')
|
69
|
-
check_option('serializer')
|
70
|
-
put_hash('config', @config)
|
71
89
|
end
|
72
90
|
|
73
91
|
# Delete the entire database. The database is no longer usable after this
|
@@ -85,7 +103,7 @@ module PEROBS
|
|
85
103
|
end
|
86
104
|
|
87
105
|
# Return true if the object with given ID exists
|
88
|
-
# @param id [
|
106
|
+
# @param id [Integer]
|
89
107
|
def include?(id)
|
90
108
|
!dynamo_get_item(id.to_s).nil?
|
91
109
|
end
|
@@ -112,11 +130,18 @@ module PEROBS
|
|
112
130
|
# Store the given object into the cluster files.
|
113
131
|
# @param obj [Hash] Object as defined by PEROBS::ObjectBase
|
114
132
|
def put_object(obj, id)
|
133
|
+
id_str = id.to_s
|
134
|
+
unless dynamo_get_item(id_str)
|
135
|
+
# The is no object with this ID yet. Increase the item counter.
|
136
|
+
@item_counter += 1
|
137
|
+
dynamo_put_item('item_counter', @item_counter.to_s)
|
138
|
+
end
|
139
|
+
|
115
140
|
dynamo_put_item(id.to_s, serialize(obj))
|
116
141
|
end
|
117
142
|
|
118
143
|
# Load the given object from the filesystem.
|
119
|
-
# @param id [
|
144
|
+
# @param id [Integer] object ID
|
120
145
|
# @return [Hash] Object as defined by PEROBS::ObjectBase or nil if ID does
|
121
146
|
# not exist
|
122
147
|
def get_object(id)
|
@@ -128,33 +153,33 @@ module PEROBS
|
|
128
153
|
each_item do |id|
|
129
154
|
dynamo_mark_item(id, false)
|
130
155
|
end
|
131
|
-
# Mark the 'config' item so it will not get deleted.
|
132
|
-
dynamo_mark_item('config')
|
133
156
|
end
|
134
157
|
|
135
158
|
# Permanently delete all objects that have not been marked. Those are
|
136
159
|
# orphaned and are no longer referenced by any actively used object.
|
137
|
-
# @return [
|
160
|
+
# @return [Integer] Count of the deleted objects.
|
138
161
|
def delete_unmarked_objects
|
139
|
-
|
162
|
+
deleted_objects_count = 0
|
140
163
|
each_item do |id|
|
141
164
|
unless dynamo_is_marked?(id)
|
142
165
|
dynamo_delete_item(id)
|
143
|
-
|
166
|
+
deleted_objects_count += 1
|
167
|
+
@item_counter -= 1
|
144
168
|
end
|
145
169
|
end
|
170
|
+
dynamo_put_item('item_counter', @item_counter.to_s)
|
146
171
|
|
147
|
-
|
172
|
+
deleted_objects_count
|
148
173
|
end
|
149
174
|
|
150
175
|
# Mark an object.
|
151
|
-
# @param id [
|
176
|
+
# @param id [Integer] ID of the object to mark
|
152
177
|
def mark(id)
|
153
178
|
dynamo_mark_item(id.to_s, true)
|
154
179
|
end
|
155
180
|
|
156
181
|
# Check if the object is marked.
|
157
|
-
# @param id [
|
182
|
+
# @param id [Integer] ID of the object to check
|
158
183
|
def is_marked?(id)
|
159
184
|
dynamo_is_marked?(id.to_s)
|
160
185
|
end
|
@@ -163,11 +188,21 @@ module PEROBS
|
|
163
188
|
# @param repair [TrueClass/FalseClass] True if found errors should be
|
164
189
|
# repaired.
|
165
190
|
def check_db(repair = false)
|
166
|
-
|
191
|
+
unless (item_counter = dynamo_get_item('item_counter')) &&
|
192
|
+
item_counter == @item_counter
|
193
|
+
PEROBS.log.error "@item_counter variable (#{@item_counter}) and " +
|
194
|
+
"item_counter table entry (#{item_counter}) don't match"
|
195
|
+
end
|
196
|
+
item_counter = 0
|
197
|
+
each_item { item_counter += 1 }
|
198
|
+
unless item_counter == @item_counter
|
199
|
+
PEROBS.log.error "Table contains #{item_counter} items but " +
|
200
|
+
"@item_counter is #{@item_counter}"
|
201
|
+
end
|
167
202
|
end
|
168
203
|
|
169
204
|
# Check if the stored object is syntactically correct.
|
170
|
-
# @param id [
|
205
|
+
# @param id [Integer] Object ID
|
171
206
|
# @param repair [TrueClass/FalseClass] True if an repair attempt should be
|
172
207
|
# made.
|
173
208
|
# @return [TrueClass/FalseClass] True if the object is OK, otherwise
|
@@ -185,9 +220,11 @@ module PEROBS
|
|
185
220
|
|
186
221
|
private
|
187
222
|
|
188
|
-
def
|
223
|
+
def create_table(table_name)
|
189
224
|
begin
|
190
225
|
@dynamodb.describe_table(:table_name => table_name)
|
226
|
+
# The table exists already. No need to create it.
|
227
|
+
return false
|
191
228
|
rescue Aws::DynamoDB::Errors::ResourceNotFoundException
|
192
229
|
@dynamodb.create_table(
|
193
230
|
:table_name => table_name,
|
@@ -210,6 +247,8 @@ module PEROBS
|
|
210
247
|
)
|
211
248
|
|
212
249
|
@dynamodb.wait_until(:table_exists, table_name: table_name)
|
250
|
+
# The table was successfully created.
|
251
|
+
return true
|
213
252
|
end
|
214
253
|
end
|
215
254
|
|
@@ -251,6 +290,9 @@ module PEROBS
|
|
251
290
|
break if resp.count <= 0
|
252
291
|
|
253
292
|
resp.items.each do |item|
|
293
|
+
# Skip all internal items
|
294
|
+
next if INTERNAL_ITEMS.include?(item['Id'])
|
295
|
+
|
254
296
|
yield(item['Id'])
|
255
297
|
end
|
256
298
|
|
data/lib/perobs/EquiBlobsFile.rb
CHANGED
@@ -2,7 +2,8 @@
|
|
2
2
|
#
|
3
3
|
# = EquiBlobsFile.rb -- Persistent Ruby Object Store
|
4
4
|
#
|
5
|
-
# Copyright (c) 2016, 2017
|
5
|
+
# Copyright (c) 2016, 2017, 2018, 2019
|
6
|
+
# by Chris Schlaeger <chris@taskjuggler.org>
|
6
7
|
#
|
7
8
|
# MIT License
|
8
9
|
#
|
@@ -26,6 +27,7 @@
|
|
26
27
|
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
27
28
|
|
28
29
|
require 'perobs/Log'
|
30
|
+
require 'perobs/ProgressMeter'
|
29
31
|
|
30
32
|
module PEROBS
|
31
33
|
|
@@ -37,6 +39,11 @@ module PEROBS
|
|
37
39
|
# is used to represent an undefined address or nil. The file has a 4 * 8
|
38
40
|
# bytes long header that stores the total entry count, the total space
|
39
41
|
# count, the offset of the first entry and the offset of the first space.
|
42
|
+
# The header is followed by a custom entry section. Each entry is also 8
|
43
|
+
# bytes long. After the custom entry section the data blobs start. Each data
|
44
|
+
# blob starts with a mark byte that indicates if the blob is valid data (2),
|
45
|
+
# a free space (0) or reseved space (1). Then it is followed by @entry_bytes
|
46
|
+
# number of bytes for the data blob.
|
40
47
|
class EquiBlobsFile
|
41
48
|
|
42
49
|
TOTAL_ENTRIES_OFFSET = 0
|
@@ -45,20 +52,26 @@ module PEROBS
|
|
45
52
|
FIRST_SPACE_OFFSET = 3 * 8
|
46
53
|
HEADER_SIZE = 4 * 8
|
47
54
|
|
48
|
-
attr_reader :total_entries, :total_spaces, :file_name
|
49
|
-
attr_accessor :first_entry
|
55
|
+
attr_reader :total_entries, :total_spaces, :file_name, :first_entry
|
50
56
|
|
51
57
|
# Create a new stack file in the given directory with the given file name.
|
52
58
|
# @param dir [String] Directory
|
53
59
|
# @param name [String] File name
|
54
|
-
# @param
|
55
|
-
|
60
|
+
# @param progressmeter [ProgressMeter] Reference to a progress meter
|
61
|
+
# object
|
62
|
+
# @param entry_bytes [Integer] Number of bytes each entry must have
|
63
|
+
# @param first_entry_default [Integer] Default address of the first blob
|
64
|
+
def initialize(dir, name, progressmeter, entry_bytes,
|
65
|
+
first_entry_default = 0)
|
66
|
+
@name = name
|
56
67
|
@file_name = File.join(dir, name + '.blobs')
|
68
|
+
@progressmeter = progressmeter
|
57
69
|
if entry_bytes < 8
|
58
70
|
PEROBS.log.fatal "EquiBlobsFile entry size must be at least 8"
|
59
71
|
end
|
60
72
|
@entry_bytes = entry_bytes
|
61
73
|
@first_entry_default = first_entry_default
|
74
|
+
clear_custom_data
|
62
75
|
reset_counters
|
63
76
|
|
64
77
|
# The File handle.
|
@@ -83,6 +96,7 @@ module PEROBS
|
|
83
96
|
unless @f.flock(File::LOCK_NB | File::LOCK_EX)
|
84
97
|
PEROBS.log.fatal 'Database blob file is locked by another process'
|
85
98
|
end
|
99
|
+
@f.sync = true
|
86
100
|
end
|
87
101
|
|
88
102
|
# Close the blob file. This method must be called before the program is
|
@@ -92,6 +106,7 @@ module PEROBS
|
|
92
106
|
if @f
|
93
107
|
@f.flush
|
94
108
|
@f.flock(File::LOCK_UN)
|
109
|
+
@f.fsync
|
95
110
|
@f.close
|
96
111
|
@f = nil
|
97
112
|
end
|
@@ -100,10 +115,61 @@ module PEROBS
|
|
100
115
|
end
|
101
116
|
end
|
102
117
|
|
118
|
+
# In addition to the standard offsets for the first entry and the first
|
119
|
+
# space any number of additional data fields can be registered. This must be
|
120
|
+
# done right after the object is instanciated and before the open() method
|
121
|
+
# is called. Each field represents a 64 bit unsigned integer.
|
122
|
+
# @param name [String] The label for this offset
|
123
|
+
# @param default_value [Integer] The default value for the offset
|
124
|
+
def register_custom_data(name, default_value = 0)
|
125
|
+
if @custom_data_labels.include?(name)
|
126
|
+
PEROBS.log.fatal "Custom data field #{name} has already been registered"
|
127
|
+
end
|
128
|
+
|
129
|
+
@custom_data_labels << name
|
130
|
+
@custom_data_values << default_value
|
131
|
+
@custom_data_defaults << default_value
|
132
|
+
end
|
133
|
+
|
134
|
+
# Reset (delete) all custom data labels that have been registered.
|
135
|
+
def clear_custom_data
|
136
|
+
unless @f.nil?
|
137
|
+
PEROBS.log.fatal "clear_custom_data should only be called when " +
|
138
|
+
"the file is not opened"
|
139
|
+
end
|
140
|
+
|
141
|
+
@custom_data_labels = []
|
142
|
+
@custom_data_values = []
|
143
|
+
@custom_data_defaults = []
|
144
|
+
end
|
145
|
+
|
146
|
+
# Set the registered custom data field to the given value.
|
147
|
+
# @param name [String] Label of the offset
|
148
|
+
# @param value [Integer] Value
|
149
|
+
def set_custom_data(name, value)
|
150
|
+
unless @custom_data_labels.include?(name)
|
151
|
+
PEROBS.log.fatal "Unknown custom data field #{name}"
|
152
|
+
end
|
153
|
+
|
154
|
+
@custom_data_values[@custom_data_labels.index(name)] = value
|
155
|
+
write_header if @f
|
156
|
+
end
|
157
|
+
|
158
|
+
# Get the registered custom data field value.
|
159
|
+
# @param name [String] Label of the offset
|
160
|
+
# @return [Integer] Value of the custom data field
|
161
|
+
def get_custom_data(name)
|
162
|
+
unless @custom_data_labels.include?(name)
|
163
|
+
PEROBS.log.fatal "Unknown custom data field #{name}"
|
164
|
+
end
|
165
|
+
|
166
|
+
@custom_data_values[@custom_data_labels.index(name)]
|
167
|
+
end
|
168
|
+
|
103
169
|
# Erase the backing store. This method should only be called when the file
|
104
170
|
# is not currently open.
|
105
171
|
def erase
|
106
|
-
|
172
|
+
@f = nil
|
107
173
|
File.delete(@file_name) if File.exist?(@file_name)
|
108
174
|
reset_counters
|
109
175
|
end
|
@@ -111,7 +177,10 @@ module PEROBS
|
|
111
177
|
# Flush out all unwritten data.
|
112
178
|
def sync
|
113
179
|
begin
|
114
|
-
|
180
|
+
if @f
|
181
|
+
@f.flush
|
182
|
+
@f.fsync
|
183
|
+
end
|
115
184
|
rescue IOError => e
|
116
185
|
PEROBS.log.fatal "Cannot sync blob file #{@file_name}: #{e.message}"
|
117
186
|
end
|
@@ -125,9 +194,16 @@ module PEROBS
|
|
125
194
|
write_header
|
126
195
|
end
|
127
196
|
|
197
|
+
# Change the address of the first blob.
|
198
|
+
# @param address [Integer] New address
|
199
|
+
def first_entry=(address)
|
200
|
+
@first_entry = address
|
201
|
+
write_header
|
202
|
+
end
|
203
|
+
|
128
204
|
# Return the address of a free blob storage space. Addresses start at 0
|
129
205
|
# and increase linearly.
|
130
|
-
# @return [
|
206
|
+
# @return [Integer] address of a free blob space
|
131
207
|
def free_address
|
132
208
|
if @first_space == 0
|
133
209
|
# There is currently no free entry. Create a new reserved entry at the
|
@@ -170,7 +246,7 @@ module PEROBS
|
|
170
246
|
|
171
247
|
# Store the given byte blob at the specified address. If the blob space is
|
172
248
|
# already in use the content will be overwritten.
|
173
|
-
# @param address [
|
249
|
+
# @param address [Integer] Address to store the blob
|
174
250
|
# @param bytes [String] bytes to store
|
175
251
|
def store_blob(address, bytes)
|
176
252
|
unless address >= 0
|
@@ -217,7 +293,7 @@ module PEROBS
|
|
217
293
|
end
|
218
294
|
|
219
295
|
# Retrieve a blob from the given address.
|
220
|
-
# @param address [
|
296
|
+
# @param address [Integer] Address to store the blob
|
221
297
|
# @return [String] blob bytes
|
222
298
|
def retrieve_blob(address)
|
223
299
|
unless address > 0
|
@@ -248,7 +324,7 @@ module PEROBS
|
|
248
324
|
end
|
249
325
|
|
250
326
|
# Delete the blob at the given address.
|
251
|
-
# @param address [
|
327
|
+
# @param address [Integer] Address of blob to delete
|
252
328
|
def delete_blob(address)
|
253
329
|
unless address >= 0
|
254
330
|
PEROBS.log.fatal "Blob address must be larger than 0, " +
|
@@ -273,7 +349,7 @@ module PEROBS
|
|
273
349
|
|
274
350
|
@first_space = offset
|
275
351
|
@total_spaces += 1
|
276
|
-
@total_entries -= 1
|
352
|
+
@total_entries -= 1 unless marker == 1
|
277
353
|
write_header
|
278
354
|
|
279
355
|
if offset == @f.size - 1 - @entry_bytes
|
@@ -286,12 +362,16 @@ module PEROBS
|
|
286
362
|
# Check the file for logical errors.
|
287
363
|
# @return [Boolean] true of file has no errors, false otherwise.
|
288
364
|
def check
|
365
|
+
sync
|
366
|
+
|
289
367
|
return false unless check_spaces
|
290
368
|
return false unless check_entries
|
291
369
|
|
292
|
-
|
293
|
-
|
294
|
-
|
370
|
+
expected_size = address_to_offset(@total_entries + @total_spaces + 1)
|
371
|
+
actual_size = @f.size
|
372
|
+
if actual_size != expected_size
|
373
|
+
PEROBS.log.error "Size mismatch in EquiBlobsFile #{@file_name}. " +
|
374
|
+
"Expected #{expected_size} bytes but found #{actual_size} bytes."
|
295
375
|
return false
|
296
376
|
end
|
297
377
|
|
@@ -314,6 +394,9 @@ module PEROBS
|
|
314
394
|
@first_entry = @first_entry_default
|
315
395
|
# The file offset of the first empty entry.
|
316
396
|
@first_space = 0
|
397
|
+
|
398
|
+
# Copy default custom values
|
399
|
+
@custom_data_values = @custom_data_defaults.dup
|
317
400
|
end
|
318
401
|
|
319
402
|
def read_header
|
@@ -321,6 +404,12 @@ module PEROBS
|
|
321
404
|
@f.seek(0)
|
322
405
|
@total_entries, @total_spaces, @first_entry, @first_space =
|
323
406
|
@f.read(HEADER_SIZE).unpack('QQQQ')
|
407
|
+
custom_labels_count = @custom_data_labels.length
|
408
|
+
if custom_labels_count > 0
|
409
|
+
@custom_data_values =
|
410
|
+
@f.read(custom_labels_count * 8).unpack("Q#{custom_labels_count}")
|
411
|
+
end
|
412
|
+
|
324
413
|
rescue IOError => e
|
325
414
|
PEROBS.log.fatal "Cannot read EquiBlobsFile header: #{e.message}"
|
326
415
|
end
|
@@ -331,6 +420,10 @@ module PEROBS
|
|
331
420
|
begin
|
332
421
|
@f.seek(0)
|
333
422
|
@f.write(header_ary.pack('QQQQ'))
|
423
|
+
unless @custom_data_values.empty?
|
424
|
+
@f.write(@custom_data_values.
|
425
|
+
pack("Q#{@custom_data_values.length}"))
|
426
|
+
end
|
334
427
|
@f.flush
|
335
428
|
end
|
336
429
|
end
|
@@ -355,25 +448,31 @@ module PEROBS
|
|
355
448
|
return false
|
356
449
|
end
|
357
450
|
|
451
|
+
return true if next_offset == 0
|
452
|
+
|
358
453
|
total_spaces = 0
|
359
|
-
|
360
|
-
|
361
|
-
|
362
|
-
|
363
|
-
|
364
|
-
|
365
|
-
|
366
|
-
|
367
|
-
|
368
|
-
|
369
|
-
|
454
|
+
@progressmeter.start("Checking #{@name} spaces list",
|
455
|
+
@total_spaces) do |pm|
|
456
|
+
begin
|
457
|
+
while next_offset != 0
|
458
|
+
# Check that the marker byte is 0
|
459
|
+
@f.seek(next_offset)
|
460
|
+
if (marker = read_char) != 0
|
461
|
+
PEROBS.log.error "Marker byte at address " +
|
462
|
+
"#{offset_to_address(next_offset)} is #{marker} instead of 0."
|
463
|
+
return false
|
464
|
+
end
|
465
|
+
# Read offset of next empty space
|
466
|
+
next_offset = read_unsigned_int
|
370
467
|
|
371
|
-
|
468
|
+
total_spaces += 1
|
469
|
+
pm.update(total_spaces)
|
470
|
+
end
|
471
|
+
rescue IOError => e
|
472
|
+
PEROBS.log.error "Cannot check space list of EquiBlobsFile " +
|
473
|
+
"#{@file_name}: #{e.message}"
|
474
|
+
return false
|
372
475
|
end
|
373
|
-
rescue IOError => e
|
374
|
-
PEROBS.log.error "Cannot check space list of EquiBlobsFile " +
|
375
|
-
"#{@file_name}: #{e.message}"
|
376
|
-
return false
|
377
476
|
end
|
378
477
|
|
379
478
|
unless total_spaces == @total_spaces
|
@@ -402,35 +501,48 @@ module PEROBS
|
|
402
501
|
return false
|
403
502
|
end
|
404
503
|
|
405
|
-
next_offset =
|
504
|
+
next_offset = address_to_offset(1)
|
406
505
|
total_entries = 0
|
407
506
|
total_spaces = 0
|
408
|
-
|
409
|
-
|
410
|
-
|
411
|
-
|
412
|
-
|
413
|
-
|
414
|
-
|
415
|
-
|
416
|
-
|
417
|
-
|
418
|
-
|
419
|
-
|
420
|
-
|
421
|
-
|
422
|
-
|
423
|
-
|
424
|
-
|
425
|
-
|
426
|
-
|
427
|
-
|
507
|
+
last_entry_is_space = false
|
508
|
+
@progressmeter.start("Checking #{@name} entries",
|
509
|
+
@total_spaces + @total_entries) do |pm|
|
510
|
+
begin
|
511
|
+
@f.seek(next_offset)
|
512
|
+
while !@f.eof
|
513
|
+
marker, bytes = @f.read(1 + @entry_bytes).
|
514
|
+
unpack("C#{1 + @entry_bytes}")
|
515
|
+
case marker
|
516
|
+
when 0
|
517
|
+
total_spaces += 1
|
518
|
+
last_entry_is_space = true
|
519
|
+
when 1
|
520
|
+
PEROBS.log.error "Entry at address " +
|
521
|
+
"#{offset_to_address(next_offset)} in EquiBlobsFile " +
|
522
|
+
"#{@file_name} has reserved marker"
|
523
|
+
return false
|
524
|
+
when 2
|
525
|
+
total_entries += 1
|
526
|
+
last_entry_is_space = false
|
527
|
+
else
|
528
|
+
PEROBS.log.error "Entry at address " +
|
529
|
+
"#{offset_to_address(next_offset)} in EquiBlobsFile " +
|
530
|
+
"#{@file_name} has illegal marker #{marker}"
|
531
|
+
return false
|
532
|
+
end
|
533
|
+
next_offset += 1 + @entry_bytes
|
428
534
|
end
|
429
|
-
|
535
|
+
|
536
|
+
pm.update(total_spaces + total_entries)
|
537
|
+
rescue
|
538
|
+
PEROBS.log.error "Cannot check entries of EquiBlobsFile " +
|
539
|
+
"#{@file_name}: #{e.message}"
|
540
|
+
return false
|
430
541
|
end
|
431
|
-
|
432
|
-
|
433
|
-
|
542
|
+
end
|
543
|
+
|
544
|
+
if last_entry_is_space
|
545
|
+
PEROBS.log.error "EquiBlobsFile #{@file_name} is not properly trimmed"
|
434
546
|
return false
|
435
547
|
end
|
436
548
|
|
@@ -452,7 +564,7 @@ module PEROBS
|
|
452
564
|
|
453
565
|
def trim_file
|
454
566
|
offset = @f.size - 1 - @entry_bytes
|
455
|
-
while offset >=
|
567
|
+
while offset >= address_to_offset(1)
|
456
568
|
@f.seek(offset)
|
457
569
|
begin
|
458
570
|
if (marker = read_char) == 0
|
@@ -536,12 +648,15 @@ module PEROBS
|
|
536
648
|
|
537
649
|
# Translate a blob address to the actual offset in the file.
|
538
650
|
def address_to_offset(address)
|
539
|
-
|
651
|
+
# Since address 0 is illegal, we can use address - 1 as index here.
|
652
|
+
HEADER_SIZE + @custom_data_labels.length * 8 +
|
653
|
+
(address - 1) * (1 + @entry_bytes)
|
540
654
|
end
|
541
655
|
|
542
656
|
# Translate the file offset to the address of a blob.
|
543
657
|
def offset_to_address(offset)
|
544
|
-
(offset - HEADER_SIZE
|
658
|
+
(offset - HEADER_SIZE - @custom_data_labels.length * 8) /
|
659
|
+
(1 + @entry_bytes) + 1
|
545
660
|
end
|
546
661
|
|
547
662
|
def write_char(c)
|