perobs 2.0.1 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +69 -27
- data/lib/perobs/Array.rb +15 -1
- data/lib/perobs/BTreeBlob.rb +13 -2
- data/lib/perobs/BTreeDB.rb +5 -2
- data/lib/perobs/Cache.rb +2 -2
- data/lib/perobs/DynamoDB.rb +9 -2
- data/lib/perobs/Hash.rb +19 -2
- data/lib/perobs/Object.rb +10 -22
- data/lib/perobs/ObjectBase.rb +34 -12
- data/lib/perobs/Store.rb +113 -62
- data/lib/perobs/version.rb +1 -1
- data/tasks/changelog.rake +1 -1
- data/test/Array_spec.rb +19 -0
- data/test/Hash_spec.rb +11 -0
- data/test/Object_spec.rb +17 -0
- data/test/Store_spec.rb +20 -5
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 50292da8f34e68427887d224c56dc2070242fc27
|
4
|
+
data.tar.gz: f3f41eca90b32ebefeeae05eaf655e99130a3af8
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 2ed683b2e8ae0cd9a6bc4531a8f70f6d2f0d98dc5f3f4947ea64f24d715e3c5d050317d6c29ab481820f25cb302e4d8c1e67ca5cf8048c0e1354d732489aef94
|
7
|
+
data.tar.gz: 2dce8dff389bc974126a88e9fdd5d042f5071e13c6560a2eeab63b05ae536ac209dfa4cddbcccac0e4a2332d46c82ea27894cc0e3a7de1c32b016f515920fbd6
|
data/README.md
CHANGED
@@ -3,48 +3,77 @@
|
|
3
3
|
PEROBS is a library that provides a persistent object store for Ruby
|
4
4
|
objects. Objects of your classes can be made persistent by deriving
|
5
5
|
them from PEROBS::Object. They will be in memory when needed and
|
6
|
-
transparently stored into a persistent storage.
|
7
|
-
filesystem based storage is supported, but back-ends for key/value
|
8
|
-
databases can be easily added.
|
6
|
+
transparently stored into a persistent storage.
|
9
7
|
|
10
8
|
This library is ideal for Ruby applications that work on huge, mostly
|
11
9
|
constant data sets and usually handle a small subset of the data at a
|
12
10
|
time. To ensure data consistency of a larger data set, you can use
|
13
|
-
transactions to make modifications of multiple objects
|
11
|
+
transactions to make modifications of multiple objects atomicaly.
|
14
12
|
Transactions can be nested and are aborted when an exception is
|
15
13
|
raised.
|
16
14
|
|
17
15
|
## Usage
|
18
16
|
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
persistend back-end when not using transactions.
|
23
|
-
|
24
|
-
Persistent objects must be created by deriving your class from
|
25
|
-
PEROBS::Object. Only instance variables that are declared via
|
26
|
-
po_attr will be persistent. All objects that are stored in persistent
|
27
|
-
instance variables must provide a to_json() method that generates JSON
|
28
|
-
syntax that can be also parsed into their original object again. It is
|
29
|
-
required that references to other objects are all going to persistent
|
30
|
-
objects again.
|
31
|
-
|
32
|
-
There are currently 3 kinds of persistent objects available:
|
17
|
+
The objects that you want to persist must be of a class that has been
|
18
|
+
derived from PEROBS::BaseObject. PEROBS already provides 3 such
|
19
|
+
classes:
|
33
20
|
|
34
21
|
* PEROBS::Object is the base class for all your classes that should be
|
35
|
-
persistent.
|
22
|
+
persistent. You can determine which instance variables should be
|
23
|
+
persisted and what default values should be used.
|
36
24
|
|
37
25
|
* PEROBS::Array provides an interface similar to the built-in Array class
|
38
|
-
but its objects are automatically
|
26
|
+
but its objects are automatically persisted.
|
39
27
|
|
40
28
|
* PEROBS::Hash provides an interface similar to the built-in Hash
|
41
|
-
class but its objects are automatically
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
29
|
+
class but its objects are automatically persisted.
|
30
|
+
|
31
|
+
When you derive your own class from PEROBS::Object you need to
|
32
|
+
specify which instance variables should be persistent. By using
|
33
|
+
po_attr you can provide a list of symbols that describe the instance
|
34
|
+
variables to persist. This will also create getter and setter methods
|
35
|
+
for these instance varables. You can use attr_init in the constructor
|
36
|
+
to define default values for your persistent objects.
|
37
|
+
|
38
|
+
To start off you must create at least one PEROBS::Store object that
|
39
|
+
owns your persistent objects. The store provides the persistent
|
40
|
+
database. A persistent object is tied to the creating store for its
|
41
|
+
whole lifetime. By default, PEROBS::Store uses an on-disk database in the
|
42
|
+
directory you specify. But you can use key/value databases as well.
|
43
|
+
Currently only Amazon DynamoDB is supported. You can create your own
|
44
|
+
key/value database wrapper with little effort.
|
45
|
+
|
46
|
+
When creating the store you can also specify the serializer to use.
|
47
|
+
The serializer controls how your data is converted to be stored in the
|
48
|
+
database. The default serializer (JSON), you can only use the subset
|
49
|
+
of Ruby types that JSON supports. See http://www.json.org/ for
|
50
|
+
details. Alternatively, you can use Marshal or YAML which support
|
51
|
+
almost every Ruby data type. YAML is much slower than JSON and Marshal
|
52
|
+
is not guaranteed to be compatible between Ruby versions.
|
53
|
+
|
54
|
+
Once you have created a store you can assign objects to it. All
|
55
|
+
persistent objects must be created with @store.new(). This is
|
56
|
+
necessary as you will only deal with proxy objects in your code.
|
57
|
+
Except for the member methods, you will never deal with the objects
|
58
|
+
directly. Instead @store.new() returns a POXReference object that acts
|
59
|
+
as a transparent proxy. This proxy is needed as your code never knows
|
60
|
+
if the actual object is really loaded into the memory or not. PEROBS
|
61
|
+
will handle this transparently for you.
|
62
|
+
|
63
|
+
A build-in cache keeps access latencies to recently used objects low
|
64
|
+
and lazily flushes modified objects into the persistend back-end when
|
65
|
+
not using transactions. It also features a garbage collector that
|
66
|
+
removes all objects that are no longer in use.
|
67
|
+
|
68
|
+
So what does 'in use' mean? You can assign a few objects to the store
|
69
|
+
directly. The store acts like a hash. These root objects can then
|
70
|
+
reference other persistent objects and so on. The garbage collector
|
71
|
+
will find all objects that are reachable from the root objects and
|
72
|
+
discards all other from the database. You have to invoke the garbage
|
73
|
+
collector manually with Store.gc(). Depending on the size of your
|
74
|
+
database it can take some time. It is recommended that you don't use
|
75
|
+
persistend objects for temporary objects in your code. Every created
|
76
|
+
object will end up in the database end needs to be garbage collected.
|
48
77
|
|
49
78
|
Here is an example how to use PEROBS. Let's define a class that models
|
50
79
|
a person with their family relations.
|
@@ -106,6 +135,19 @@ accesses, you must manually mark the instance as modified by calling
|
|
106
135
|
Object::mark_as_modified(). If that is forgotten, the change will
|
107
136
|
reside in memory but might not be persisted into the database.
|
108
137
|
|
138
|
+
### Use of proxy objects
|
139
|
+
|
140
|
+
Your code should never deal with the persistent objects directly. The
|
141
|
+
PEROBS API takes care that you will always get a proxy object. The
|
142
|
+
only exception to this rule is the code in the instance methods. By
|
143
|
+
design, this code operates on the real object. The only caveat here is
|
144
|
+
the use of self(). If you pass the result of self() to another object
|
145
|
+
you will leak a PEROBS::ObjectBase derived object into your data
|
146
|
+
structures. PEROBS will watch for this and will throw an exception
|
147
|
+
when it detects such objects. Just remember to use myself() instead of
|
148
|
+
self() if you want to pass a reference to the current persistent
|
149
|
+
object to another object.
|
150
|
+
|
109
151
|
## Installation
|
110
152
|
|
111
153
|
Add this line to your application's Gemfile:
|
data/lib/perobs/Array.rb
CHANGED
@@ -119,7 +119,21 @@ module PEROBS
|
|
119
119
|
|
120
120
|
def _serialize
|
121
121
|
@data.map do |v|
|
122
|
-
v.respond_to?(:is_poxreference?)
|
122
|
+
if v.respond_to?(:is_poxreference?)
|
123
|
+
POReference.new(v.id)
|
124
|
+
else
|
125
|
+
# Outside of the PEROBS library all PEROBS::ObjectBase derived
|
126
|
+
# objects should not be used directly. The library only exposes them
|
127
|
+
# via POXReference proxy objects.
|
128
|
+
if v.is_a?(ObjectBase)
|
129
|
+
raise RuntimeError, 'A PEROBS::ObjectBase object escaped! ' +
|
130
|
+
"It is stored in a PEROBS::Array at index #{@data.index(v)}. " +
|
131
|
+
'Have you used self() instead of myself() to' +
|
132
|
+
"get the reference of this PEROBS object?\n" +
|
133
|
+
v.inspect
|
134
|
+
end
|
135
|
+
v
|
136
|
+
end
|
123
137
|
end
|
124
138
|
end
|
125
139
|
|
data/lib/perobs/BTreeBlob.rb
CHANGED
@@ -2,7 +2,7 @@
|
|
2
2
|
#
|
3
3
|
# = BTreeBlob.rb -- Persistent Ruby Object Store
|
4
4
|
#
|
5
|
-
# Copyright (c) 2015 by Chris Schlaeger <chris@taskjuggler.org>
|
5
|
+
# Copyright (c) 2015, 2016 by Chris Schlaeger <chris@taskjuggler.org>
|
6
6
|
#
|
7
7
|
# MIT License
|
8
8
|
#
|
@@ -137,12 +137,23 @@ module PEROBS
|
|
137
137
|
end
|
138
138
|
|
139
139
|
# Remove all entries from the index that have not been marked.
|
140
|
+
# @return [Array] List of deleted object IDs.
|
140
141
|
def delete_unmarked_entries
|
142
|
+
deleted_ids = []
|
141
143
|
# First remove the entry from the hash table.
|
142
|
-
@entries_by_id.delete_if
|
144
|
+
@entries_by_id.delete_if do |id, e|
|
145
|
+
if e[MARKED] == 0
|
146
|
+
deleted_ids << id
|
147
|
+
true
|
148
|
+
else
|
149
|
+
false
|
150
|
+
end
|
151
|
+
end
|
143
152
|
# Then delete the entry itself.
|
144
153
|
@entries.delete_if { |e| e[MARKED] == 0 }
|
145
154
|
write_index
|
155
|
+
|
156
|
+
deleted_ids
|
146
157
|
end
|
147
158
|
|
148
159
|
# Run a basic consistency check.
|
data/lib/perobs/BTreeDB.rb
CHANGED
@@ -2,7 +2,7 @@
|
|
2
2
|
#
|
3
3
|
# = BTreeDB.rb -- Persistent Ruby Object Store
|
4
4
|
#
|
5
|
-
# Copyright (c) 2015 by Chris Schlaeger <chris@taskjuggler.org>
|
5
|
+
# Copyright (c) 2015, 2016 by Chris Schlaeger <chris@taskjuggler.org>
|
6
6
|
#
|
7
7
|
# MIT License
|
8
8
|
#
|
@@ -158,8 +158,11 @@ module PEROBS
|
|
158
158
|
|
159
159
|
# Permanently delete all objects that have not been marked. Those are
|
160
160
|
# orphaned and are no longer referenced by any actively used object.
|
161
|
+
# @return [Array] List of IDs that have been removed from the DB.
|
161
162
|
def delete_unmarked_objects
|
162
|
-
|
163
|
+
deleted_ids = []
|
164
|
+
each_blob { |blob| deleted_ids += blob.delete_unmarked_entries }
|
165
|
+
deleted_ids
|
163
166
|
end
|
164
167
|
|
165
168
|
# Mark an object.
|
data/lib/perobs/Cache.rb
CHANGED
@@ -2,7 +2,7 @@
|
|
2
2
|
#
|
3
3
|
# = Cache.rb -- Persistent Ruby Object Store
|
4
4
|
#
|
5
|
-
# Copyright (c) 2015 by Chris Schlaeger <chris@taskjuggler.org>
|
5
|
+
# Copyright (c) 2015, 2016 by Chris Schlaeger <chris@taskjuggler.org>
|
6
6
|
#
|
7
7
|
# MIT License
|
8
8
|
#
|
@@ -31,7 +31,7 @@ module PEROBS
|
|
31
31
|
|
32
32
|
# The Cache provides two functions for the PEROBS Store. It keeps some
|
33
33
|
# amount of objects in memory to substantially reduce read access latencies.
|
34
|
-
# It
|
34
|
+
# It also stores a list of objects that haven't been synced to the
|
35
35
|
# permanent store yet to accelerate object writes.
|
36
36
|
class Cache
|
37
37
|
|
data/lib/perobs/DynamoDB.rb
CHANGED
@@ -2,7 +2,7 @@
|
|
2
2
|
#
|
3
3
|
# = DynamoDB.rb -- Persistent Ruby Object Store
|
4
4
|
#
|
5
|
-
# Copyright (c) 2015 by Chris Schlaeger <chris@taskjuggler.org>
|
5
|
+
# Copyright (c) 2015, 2016 by Chris Schlaeger <chris@taskjuggler.org>
|
6
6
|
#
|
7
7
|
# MIT License
|
8
8
|
#
|
@@ -134,10 +134,17 @@ module PEROBS
|
|
134
134
|
|
135
135
|
# Permanently delete all objects that have not been marked. Those are
|
136
136
|
# orphaned and are no longer referenced by any actively used object.
|
137
|
+
# @return [Array] List of object IDs of the deleted objects.
|
137
138
|
def delete_unmarked_objects
|
139
|
+
deleted_ids = []
|
138
140
|
each_item do |id|
|
139
|
-
|
141
|
+
unless dynamo_is_marked?(id)
|
142
|
+
dynamo_delete_item(id)
|
143
|
+
deleted_ids << id
|
144
|
+
end
|
140
145
|
end
|
146
|
+
|
147
|
+
deleted_ids
|
141
148
|
end
|
142
149
|
|
143
150
|
# Mark an object.
|
data/lib/perobs/Hash.rb
CHANGED
@@ -118,8 +118,25 @@ module PEROBS
|
|
118
118
|
|
119
119
|
def _serialize
|
120
120
|
data = {}
|
121
|
-
|
122
|
-
|
121
|
+
|
122
|
+
@data.each do |k, v|
|
123
|
+
if v.respond_to?(:is_poxreference?)
|
124
|
+
data[k] = POReference.new(v.id)
|
125
|
+
else
|
126
|
+
# Outside of the PEROBS library all PEROBS::ObjectBase derived
|
127
|
+
# objects should not be used directly. The library only exposes them
|
128
|
+
# via POXReference proxy objects.
|
129
|
+
if v.is_a?(ObjectBase)
|
130
|
+
raise RuntimeError, 'A PEROBS::ObjectBase object escaped! ' +
|
131
|
+
"It is stored in a PEROBS::Hash with key #{k.inspect}. " +
|
132
|
+
'Have you used self() instead of myself() to' +
|
133
|
+
"get the reference of this PEROBS object?\n" +
|
134
|
+
v.inspect
|
135
|
+
end
|
136
|
+
data[k] = v
|
137
|
+
end
|
138
|
+
end
|
139
|
+
|
123
140
|
data
|
124
141
|
end
|
125
142
|
|
data/lib/perobs/Object.rb
CHANGED
@@ -175,13 +175,6 @@ module PEROBS
|
|
175
175
|
_all_attributes.each do |attr|
|
176
176
|
ivar = ('@' + attr.to_s).to_sym
|
177
177
|
value = instance_variable_get(ivar)
|
178
|
-
#if (value = instance_variable_get(ivar)).is_a?(ObjectBase)
|
179
|
-
# raise ArgumentError, "The instance variable #{ivar} contains a " +
|
180
|
-
# "reference to a PEROBS::ObjectBase object! " +
|
181
|
-
# "This is not allowed. You must use the " +
|
182
|
-
# "accessor method to assign a reference to " +
|
183
|
-
# "another PEROBS object."
|
184
|
-
#end
|
185
178
|
attributes[attr.to_s] = value.respond_to?(:is_poxreference?) ?
|
186
179
|
POReference.new(value.id) : value
|
187
180
|
end
|
@@ -189,33 +182,28 @@ module PEROBS
|
|
189
182
|
end
|
190
183
|
|
191
184
|
def _set(attr, val)
|
192
|
-
|
193
|
-
if !val.respond_to?(:is_poxreference?) && val.is_a?(ObjectBase)
|
185
|
+
if val.is_a?(ObjectBase)
|
194
186
|
# References to other PEROBS::Objects must be handled somewhat
|
195
187
|
# special.
|
196
188
|
if @store != val.store
|
197
189
|
raise ArgumentError, 'The referenced object is not part of this store'
|
198
190
|
end
|
199
|
-
|
200
|
-
|
201
|
-
|
202
|
-
|
203
|
-
|
204
|
-
|
191
|
+
unless val.respond_to?(:is_poxreference?)
|
192
|
+
raise ArgumentError, 'A PEROBS::ObjectBase object escaped! ' +
|
193
|
+
'Have you used self() instead of myself() to' +
|
194
|
+
'get the reference of the PEROBS object that ' +
|
195
|
+
'you are trying to assign here?'
|
196
|
+
end
|
205
197
|
end
|
198
|
+
instance_variable_set(('@' + attr.to_s).to_sym, val)
|
206
199
|
# Let the store know that we have a modified object.
|
207
|
-
|
200
|
+
mark_as_modified
|
208
201
|
|
209
202
|
val
|
210
203
|
end
|
211
204
|
|
212
205
|
def _get(attr)
|
213
|
-
|
214
|
-
if value.respond_to?(:is_poxreference?)
|
215
|
-
@store.object_by_id(value.id)
|
216
|
-
else
|
217
|
-
value
|
218
|
-
end
|
206
|
+
instance_variable_get(('@' + attr.to_s).to_sym)
|
219
207
|
end
|
220
208
|
|
221
209
|
def _all_attributes
|
data/lib/perobs/ObjectBase.rb
CHANGED
@@ -2,7 +2,7 @@
|
|
2
2
|
#
|
3
3
|
# = ObjectBase.rb -- Persistent Ruby Object Store
|
4
4
|
#
|
5
|
-
# Copyright (c) 2015 by Chris Schlaeger <chris@taskjuggler.org>
|
5
|
+
# Copyright (c) 2015, 2016 by Chris Schlaeger <chris@taskjuggler.org>
|
6
6
|
#
|
7
7
|
# MIT License
|
8
8
|
#
|
@@ -34,7 +34,7 @@ module PEROBS
|
|
34
34
|
# since it's no longer referenced once it has been evicted from the
|
35
35
|
# PEROBS::Store cache. The POXReference objects function as a transparent
|
36
36
|
# proxy for the objects they are referencing.
|
37
|
-
class POXReference
|
37
|
+
class POXReference < BasicObject
|
38
38
|
|
39
39
|
attr_reader :store, :id
|
40
40
|
|
@@ -47,12 +47,13 @@ module PEROBS
|
|
47
47
|
# Proxy all calls to unknown methods to the referenced object.
|
48
48
|
def method_missing(method_sym, *args, &block)
|
49
49
|
unless (obj = _referenced_object)
|
50
|
-
raise ::RuntimeError,
|
51
|
-
"ID #{@id} found in
|
50
|
+
::Kernel.raise ::RuntimeError,
|
51
|
+
"Internal consistency error. No object with ID #{@id} found in " +
|
52
|
+
'the store.'
|
52
53
|
end
|
53
54
|
if obj.respond_to?(:is_poxreference?)
|
54
|
-
raise ::RuntimeError,
|
55
|
-
"POXReference that references a POXReference found"
|
55
|
+
::Kernel.raise ::RuntimeError,
|
56
|
+
"POXReference that references a POXReference found."
|
56
57
|
end
|
57
58
|
obj.send(method_sym, *args, &block)
|
58
59
|
end
|
@@ -86,6 +87,18 @@ module PEROBS
|
|
86
87
|
_referenced_object == obj
|
87
88
|
end
|
88
89
|
|
90
|
+
# BasicObject provides a equal?() method that prevents method_missing from
|
91
|
+
# being called. So we have to pass the call manually to the referenced
|
92
|
+
# object.
|
93
|
+
# @param obj object to compare this object with.
|
94
|
+
def equal?(obj)
|
95
|
+
if obj.respond_to?(:is_poxreference?)
|
96
|
+
_referenced_object.equal?(obj._referenced_object)
|
97
|
+
else
|
98
|
+
_referenced_object.equal?(obj)
|
99
|
+
end
|
100
|
+
end
|
101
|
+
|
89
102
|
# Shortcut to access the _id() method of the referenced object.
|
90
103
|
def _id
|
91
104
|
@id
|
@@ -102,7 +115,7 @@ module PEROBS
|
|
102
115
|
# common to all classes of persistent objects.
|
103
116
|
class ObjectBase
|
104
117
|
|
105
|
-
attr_reader :_id, :store
|
118
|
+
attr_reader :_id, :store, :myself
|
106
119
|
|
107
120
|
# New PEROBS objects must always be created by calling # Store.new().
|
108
121
|
# PEROBS users should never call this method or equivalents of derived
|
@@ -110,17 +123,29 @@ module PEROBS
|
|
110
123
|
def initialize(store)
|
111
124
|
@store = store
|
112
125
|
unless @store.object_creation_in_progress
|
113
|
-
raise ::RuntimeError,
|
126
|
+
::Kernel.raise ::RuntimeError,
|
114
127
|
"All PEROBS objects must exclusively be created by calling " +
|
115
128
|
"Store.new(). Never call the object constructor directly."
|
116
129
|
end
|
117
130
|
@_id = @store.db.new_id
|
131
|
+
ObjectSpace.define_finalizer(self, ObjectBase._finalize(@store, @_id))
|
118
132
|
@_stash_map = nil
|
133
|
+
# Allocate a proxy object for this object. User code should only operate
|
134
|
+
# on this proxy, never on self.
|
135
|
+
@myself = POXReference.new(@store, @_id)
|
119
136
|
|
120
137
|
# Let the store know that we have a modified object.
|
121
138
|
@store.cache.cache_write(self)
|
122
139
|
end
|
123
140
|
|
141
|
+
# This method generates the destructor for the objects of this class. It
|
142
|
+
# is done this way to prevent the Proc object hanging on to a reference to
|
143
|
+
# self which would prevent the object from being collected. This internal
|
144
|
+
# method is not intended for users to call.
|
145
|
+
def ObjectBase._finalize(store, id)
|
146
|
+
proc { store._collect(id) }
|
147
|
+
end
|
148
|
+
|
124
149
|
public
|
125
150
|
|
126
151
|
# This method can be overloaded by derived classes to do some massaging on
|
@@ -157,10 +182,7 @@ module PEROBS
|
|
157
182
|
|
158
183
|
klass = store.class_map.id_to_class(db_obj['class_id'])
|
159
184
|
# Call the constructor of the specified class.
|
160
|
-
obj = store.
|
161
|
-
# The object gets created with a new ID by default. We need to restore
|
162
|
-
# the old one.
|
163
|
-
obj._change_id(id)
|
185
|
+
obj = store._construct_po(Object.const_get(klass), id)
|
164
186
|
obj._deserialize(db_obj['data'])
|
165
187
|
obj.post_restore
|
166
188
|
|
data/lib/perobs/Store.rb
CHANGED
@@ -2,7 +2,7 @@
|
|
2
2
|
#
|
3
3
|
# = Store.rb -- Persistent Ruby Object Store
|
4
4
|
#
|
5
|
-
# Copyright (c) 2015 by Chris Schlaeger <chris@taskjuggler.org>
|
5
|
+
# Copyright (c) 2015, 2016 by Chris Schlaeger <chris@taskjuggler.org>
|
6
6
|
#
|
7
7
|
# MIT License
|
8
8
|
#
|
@@ -26,6 +26,7 @@
|
|
26
26
|
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
27
27
|
|
28
28
|
require 'set'
|
29
|
+
require 'weakref'
|
29
30
|
|
30
31
|
require 'perobs/Cache'
|
31
32
|
require 'perobs/ClassMap'
|
@@ -128,16 +129,18 @@ module PEROBS
|
|
128
129
|
# the Store.new() call by PEROBS users.
|
129
130
|
@object_creation_in_progress = false
|
130
131
|
|
132
|
+
# List of PEROBS objects that are currently available as Ruby objects
|
133
|
+
# hashed by their ID.
|
134
|
+
@in_memory_objects = {}
|
135
|
+
|
131
136
|
# The Cache reduces read and write latencies by keeping a subset of the
|
132
137
|
# objects in memory.
|
133
138
|
@cache = Cache.new(options[:cache_bits] || 16)
|
134
139
|
|
135
140
|
# The named (global) objects IDs hashed by their name
|
136
141
|
unless (@root_objects = object_by_id(0))
|
137
|
-
@root_objects = construct_po(Hash)
|
138
|
-
|
139
142
|
# The root object hash always has the object ID 0.
|
140
|
-
@root_objects
|
143
|
+
@root_objects = _construct_po(Hash, 0)
|
141
144
|
# The ID change removes it from the write cache. We need to add it
|
142
145
|
# again.
|
143
146
|
@cache.cache_write(@root_objects)
|
@@ -152,26 +155,39 @@ module PEROBS
|
|
152
155
|
# constructor of the specified class.
|
153
156
|
# @return [POXReference] A reference to the newly created object.
|
154
157
|
def new(klass, *args)
|
155
|
-
|
158
|
+
_construct_po(klass, nil, *args).myself
|
156
159
|
end
|
157
160
|
|
158
161
|
# For library internal use only!
|
159
|
-
|
162
|
+
# This method will create a new PEROBS object.
|
163
|
+
# @param klass [BasicObject] Class of the object to create
|
164
|
+
# @param id [Fixnum, Bignum or nil] Requested object ID or nil
|
165
|
+
# @param *args [Array] Arguments to pass to the object constructor.
|
166
|
+
# @return [BasicObject] Newly constructed PEROBS object
|
167
|
+
def _construct_po(klass, id, *args)
|
168
|
+
unless klass.is_a?(BasicObject)
|
169
|
+
raise ArgumentError, "#{klass} is not a BasicObject derivative"
|
170
|
+
end
|
160
171
|
@object_creation_in_progress = true
|
161
172
|
obj = klass.new(self, *args)
|
162
173
|
@object_creation_in_progress = false
|
174
|
+
# If a specific object ID was requested we need to set it now.
|
175
|
+
obj._change_id(id) if id
|
176
|
+
# Add the new object to the in-memory list. We only store a weak
|
177
|
+
# reference to the object so it can be garbage collected. When this
|
178
|
+
# happens the object finalizer is triggered and calls _forget() to
|
179
|
+
# remove the object from this hash again.
|
180
|
+
@in_memory_objects[obj._id] = WeakRef.new(obj)
|
163
181
|
obj
|
164
182
|
end
|
165
183
|
|
166
|
-
|
167
184
|
# Delete the entire store. The store is no longer usable after this
|
168
185
|
# method was called.
|
169
186
|
def delete_store
|
170
187
|
@db.delete_database
|
171
|
-
@class_map = @cache = @root_objects = nil
|
188
|
+
@db = @class_map = @cache = @root_objects = nil
|
172
189
|
end
|
173
190
|
|
174
|
-
|
175
191
|
# Store the provided object under the given name. Use this to make the
|
176
192
|
# object a root or top-level object (think global variable). Each store
|
177
193
|
# should have at least one root object. Objects that are not directly or
|
@@ -187,13 +203,10 @@ module PEROBS
|
|
187
203
|
return nil
|
188
204
|
end
|
189
205
|
|
190
|
-
if obj.respond_to?(:is_poxreference?)
|
191
|
-
obj = obj._referenced_object
|
192
|
-
end
|
193
206
|
# We only allow derivatives of PEROBS::Object to be stored in the
|
194
207
|
# store.
|
195
208
|
unless obj.is_a?(ObjectBase)
|
196
|
-
raise ArgumentError,
|
209
|
+
raise ArgumentError, 'Object must be of class PEROBS::Object but ' +
|
197
210
|
"is of class #{obj.class}"
|
198
211
|
end
|
199
212
|
|
@@ -203,8 +216,6 @@ module PEROBS
|
|
203
216
|
|
204
217
|
# Store the name and mark the name list as modified.
|
205
218
|
@root_objects[name] = obj._id
|
206
|
-
# Add the object to the in-memory storage list.
|
207
|
-
@cache.cache_write(obj)
|
208
219
|
|
209
220
|
obj
|
210
221
|
end
|
@@ -217,7 +228,7 @@ module PEROBS
|
|
217
228
|
# Return nil if there is no object with that name.
|
218
229
|
return nil unless (id = @root_objects[name])
|
219
230
|
|
220
|
-
|
231
|
+
POXReference.new(self, id)
|
221
232
|
end
|
222
233
|
|
223
234
|
# Flush out all modified objects to disk and shrink the in-memory list if
|
@@ -233,6 +244,7 @@ module PEROBS
|
|
233
244
|
# from the back-end storage. The garbage collector is not invoked
|
234
245
|
# automatically. Depending on your usage pattern, you need to call this
|
235
246
|
# method periodically.
|
247
|
+
# @return [Fixnum] The number of collected objects
|
236
248
|
def gc
|
237
249
|
sync
|
238
250
|
mark
|
@@ -243,21 +255,27 @@ module PEROBS
|
|
243
255
|
# public API and should never be called by outside users. It's purely
|
244
256
|
# intended for internal use.
|
245
257
|
def object_by_id(id)
|
246
|
-
if (obj = @
|
258
|
+
if (obj = @in_memory_objects[id])
|
247
259
|
# We have the object in memory so we can just return it.
|
248
|
-
|
249
|
-
|
250
|
-
|
251
|
-
|
252
|
-
#
|
253
|
-
|
254
|
-
# Add the object to the in-memory storage list.
|
255
|
-
@cache.cache_read(obj)
|
256
|
-
|
257
|
-
return obj
|
260
|
+
begin
|
261
|
+
return obj.__getobj__
|
262
|
+
rescue WeakRef::RefError
|
263
|
+
# Due to a race condition the object can still be in the
|
264
|
+
# @in_memory_objects list but has been collected already by the Ruby
|
265
|
+
# GC. In that case we need to load it again.
|
258
266
|
end
|
259
267
|
end
|
260
268
|
|
269
|
+
# We don't have the object in memory. Let's find it in the storage.
|
270
|
+
if @db.include?(id)
|
271
|
+
# Great, object found. Read it into memory and return it.
|
272
|
+
obj = ObjectBase::read(self, id)
|
273
|
+
# Add the object to the in-memory storage list.
|
274
|
+
@cache.cache_read(obj)
|
275
|
+
|
276
|
+
return obj
|
277
|
+
end
|
278
|
+
|
261
279
|
# The requested object does not exist. Return nil.
|
262
280
|
nil
|
263
281
|
end
|
@@ -268,47 +286,24 @@ module PEROBS
|
|
268
286
|
# unreadable object is found, the reference will simply be deleted.
|
269
287
|
# @param repair [TrueClass/FalseClass] true if a repair attempt should be
|
270
288
|
# made.
|
289
|
+
# @return [Fixnum] The number of references to bad objects found.
|
271
290
|
def check(repair = true)
|
291
|
+
# All objects must have in-db version.
|
292
|
+
sync
|
272
293
|
# Run basic consistency checks first.
|
273
294
|
@db.check_db(repair)
|
274
295
|
|
296
|
+
# We will use the mark to mark all objects that we have checked already.
|
297
|
+
# Before we start, we need to clear all marks.
|
275
298
|
@db.clear_marks
|
276
|
-
|
277
|
-
|
278
|
-
# First we check the top-level objects. They are only added to the
|
279
|
-
# working set if they are OK.
|
299
|
+
|
300
|
+
errors = 0
|
280
301
|
@root_objects.each do |name, id|
|
281
|
-
|
282
|
-
stack << id
|
283
|
-
end
|
302
|
+
errors += check_object(id, repair)
|
284
303
|
end
|
285
|
-
|
286
|
-
# Delete any top-level object that is defective.
|
287
|
-
stack.each { |id| @root_objects.delete(id) }
|
288
|
-
# The remaining top-level objects are the initial working set.
|
289
|
-
stack = @root_objects.values
|
290
|
-
else
|
291
|
-
# The initial working set must only be OK objects.
|
292
|
-
stack = @root_objects.values - stack
|
293
|
-
end
|
294
|
-
stack.each { |id| @db.mark(id) }
|
304
|
+
@root_objects.delete_if { |name, id| !@db.check(id, false) }
|
295
305
|
|
296
|
-
|
297
|
-
id = stack.pop
|
298
|
-
(obj = object_by_id(id))._referenced_object_ids.each do |id|
|
299
|
-
# Add all found references that have passed the check to the working
|
300
|
-
# list for the next iterations.
|
301
|
-
if @db.check(id, repair)
|
302
|
-
unless @db.is_marked?(id)
|
303
|
-
stack << id
|
304
|
-
@db.mark(id)
|
305
|
-
end
|
306
|
-
elsif repair
|
307
|
-
# Remove references to bad objects.
|
308
|
-
obj._delete_reference_to_id(id)
|
309
|
-
end
|
310
|
-
end
|
311
|
-
end
|
306
|
+
errors
|
312
307
|
end
|
313
308
|
|
314
309
|
# This method will execute the provided block as an atomic transaction
|
@@ -339,7 +334,7 @@ module PEROBS
|
|
339
334
|
while !stack.empty?
|
340
335
|
# Get an object index from the stack.
|
341
336
|
obj = object_by_id(id = stack.pop)
|
342
|
-
yield(
|
337
|
+
yield(POXReference.new(self, id)) if block_given?
|
343
338
|
obj._referenced_object_ids.each do |id|
|
344
339
|
unless @db.is_marked?(id)
|
345
340
|
@db.mark(id)
|
@@ -355,6 +350,24 @@ module PEROBS
|
|
355
350
|
@class_map.rename(rename_map)
|
356
351
|
end
|
357
352
|
|
353
|
+
# Remove the object from the in-memory list. This is an internal method
|
354
|
+
# and should never be called from user code.
|
355
|
+
# @param id [Fixnum or Bignum] Object ID of object to remove from the list
|
356
|
+
def _collect(id, ignore_errors = false)
|
357
|
+
unless ignore_errors || @in_memory_objects.include?(id)
|
358
|
+
raise RuntimeError, "Object with id #{id} is currently not in memory"
|
359
|
+
end
|
360
|
+
@in_memory_objects.delete(id)
|
361
|
+
end
|
362
|
+
|
363
|
+
# This method returns a Hash with some statistics about this store.
|
364
|
+
def statistics
|
365
|
+
{
|
366
|
+
:in_memory_objects => @in_memory_objects.length,
|
367
|
+
:root_objects => 0 #@root_objects.length
|
368
|
+
}
|
369
|
+
end
|
370
|
+
|
358
371
|
private
|
359
372
|
|
360
373
|
# Mark phase of a mark-and-sweep garbage collector. It will mark all
|
@@ -368,8 +381,46 @@ module PEROBS
|
|
368
381
|
# Sweep phase of a mark-and-sweep garbage collector. It will remove all
|
369
382
|
# unmarked objects from the store.
|
370
383
|
def sweep
|
371
|
-
@db.delete_unmarked_objects
|
384
|
+
cntr = @db.delete_unmarked_objects.length
|
372
385
|
@cache.reset
|
386
|
+
cntr
|
387
|
+
end
|
388
|
+
|
389
|
+
# Check the object with the given start_id and all other objects that are
|
390
|
+
# somehow reachable from the start object.
|
391
|
+
# @param start_id [Fixnum or Bignum] ID of the top-level object to start
|
392
|
+
# with
|
393
|
+
# @param repair [Boolean] Delete refernces to broken objects if true
|
394
|
+
# @return [Fixnum] The number of references to bad objects.
|
395
|
+
def check_object(start_id, repair)
|
396
|
+
errors = 0
|
397
|
+
@db.mark(start_id)
|
398
|
+
# The todo list holds a touple for each object that still needs to be
|
399
|
+
# checked. The first item is the referring object and the second is the
|
400
|
+
# ID of the object to check.
|
401
|
+
todo_list = [ [ nil, start_id ] ]
|
402
|
+
|
403
|
+
while !todo_list.empty?
|
404
|
+
# Get the next PEROBS object to check
|
405
|
+
ref_obj, id = todo_list.pop
|
406
|
+
|
407
|
+
if (obj = object_by_id(id)) && @db.check(id, repair)
|
408
|
+
# The object exists and is OK. Mark is as checked.
|
409
|
+
@db.mark(id)
|
410
|
+
# Now look at all other objects referenced by this object.
|
411
|
+
obj._referenced_object_ids.each do |refd_id|
|
412
|
+
# Push them onto the todo list unless they have been marked
|
413
|
+
# already.
|
414
|
+
todo_list << [ obj, refd_id ] unless @db.is_marked?(refd_id)
|
415
|
+
end
|
416
|
+
else
|
417
|
+
# Remove references to bad objects.
|
418
|
+
ref_obj._delete_reference_to_id(id) if ref_obj && repair
|
419
|
+
errors += 1
|
420
|
+
end
|
421
|
+
end
|
422
|
+
|
423
|
+
errors
|
373
424
|
end
|
374
425
|
|
375
426
|
end
|
data/lib/perobs/version.rb
CHANGED
data/tasks/changelog.rake
CHANGED
@@ -155,7 +155,7 @@ task :changelog do
|
|
155
155
|
def getReleaseVersions
|
156
156
|
# Get list of release tags from Git repository
|
157
157
|
releaseVersions = `git tag`.split("\n").map { |r| r.chomp }.
|
158
|
-
delete_if { |r| ! (/
|
158
|
+
delete_if { |r| ! (/v\d+\.\d+\.\d+/ =~ r) }.
|
159
159
|
sort{ |a, b| compareTags(a, b) }
|
160
160
|
releaseVersions << 'HEAD'
|
161
161
|
end
|
data/test/Array_spec.rb
CHANGED
@@ -40,6 +40,10 @@ class PO < PEROBS::Object
|
|
40
40
|
_set(:name, name)
|
41
41
|
end
|
42
42
|
|
43
|
+
def get_self
|
44
|
+
self # Never do this in real user code!
|
45
|
+
end
|
46
|
+
|
43
47
|
end
|
44
48
|
|
45
49
|
describe PEROBS::Array do
|
@@ -205,4 +209,19 @@ describe PEROBS::Array do
|
|
205
209
|
pcheck { expect(a).to eq([ 1, 2, 3, 4, 5, nil ]) }
|
206
210
|
end
|
207
211
|
|
212
|
+
it 'should only provide POXReference objects' do
|
213
|
+
a = cpa([ @store.new(PO), @store.new(PO) ])
|
214
|
+
expect(a[0].respond_to?(:is_poxreference?)).to be true
|
215
|
+
a.each do |a|
|
216
|
+
expect(a.respond_to?(:is_poxreference?)).to be true
|
217
|
+
end
|
218
|
+
end
|
219
|
+
|
220
|
+
it 'should catch a leaked PEROBS::ObjectBase object' do
|
221
|
+
@store['a'] = a = @store.new(PEROBS::Array)
|
222
|
+
o = @store.new(PO)
|
223
|
+
a[0] = o.get_self
|
224
|
+
expect { @store.sync }.to raise_error(RuntimeError)
|
225
|
+
end
|
226
|
+
|
208
227
|
end
|
data/test/Hash_spec.rb
CHANGED
@@ -41,6 +41,10 @@ class PO < PEROBS::Object
|
|
41
41
|
@name = name
|
42
42
|
end
|
43
43
|
|
44
|
+
def get_self
|
45
|
+
self # Never do this in real user code!
|
46
|
+
end
|
47
|
+
|
44
48
|
end
|
45
49
|
|
46
50
|
describe PEROBS::Hash do
|
@@ -154,4 +158,11 @@ describe PEROBS::Hash do
|
|
154
158
|
pcheck { expect(h2).to eq(hb) }
|
155
159
|
end
|
156
160
|
|
161
|
+
it 'should catch a leaked PEROBS::ObjectBase object' do
|
162
|
+
@store['a'] = a = @store.new(PEROBS::Hash)
|
163
|
+
o = @store.new(PO)
|
164
|
+
a['a'] = o.get_self
|
165
|
+
expect { @store.sync }.to raise_error(RuntimeError)
|
166
|
+
end
|
167
|
+
|
157
168
|
end
|
data/test/Object_spec.rb
CHANGED
@@ -51,6 +51,10 @@ class O2 < PEROBS::Object
|
|
51
51
|
@a3.a1
|
52
52
|
end
|
53
53
|
|
54
|
+
def get_self
|
55
|
+
self # Never do this in real user code!
|
56
|
+
end
|
57
|
+
|
54
58
|
end
|
55
59
|
|
56
60
|
class O3 < PEROBS::Object
|
@@ -125,6 +129,19 @@ describe PEROBS::Store do
|
|
125
129
|
expect(o2.a3_deref).to eq('a1')
|
126
130
|
end
|
127
131
|
|
132
|
+
it 'should always return a POXReference for a PEROBS object' do
|
133
|
+
@store['o1'] = o1 = @store.new(O1)
|
134
|
+
o1.a1 = @store.new(O2)
|
135
|
+
expect(@store['o1'].respond_to?(:is_poxreference?)).to be true
|
136
|
+
expect(o1.a1.respond_to?(:is_poxreference?)).to be true
|
137
|
+
end
|
138
|
+
|
139
|
+
it 'should catch a leaked PEROBS::ObjectBase object' do
|
140
|
+
@store['a'] = a = @store.new(O1)
|
141
|
+
o = @store.new(O2)
|
142
|
+
expect { a.a1 = o.get_self }.to raise_error(ArgumentError)
|
143
|
+
end
|
144
|
+
|
128
145
|
it 'should raise an error when no attributes are defined' do
|
129
146
|
@store['o3'] = @store.new(O3)
|
130
147
|
expect { @store.sync }.to raise_error(StandardError)
|
data/test/Store_spec.rb
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
# encoding: UTF-8
|
2
2
|
#
|
3
|
-
# Copyright (c) 2015 by Chris Schlaeger <chris@taskjuggler.org>
|
3
|
+
# Copyright (c) 2015, 2016 by Chris Schlaeger <chris@taskjuggler.org>
|
4
4
|
#
|
5
5
|
# MIT License
|
6
6
|
#
|
@@ -218,19 +218,23 @@ describe PEROBS::Store do
|
|
218
218
|
p1.related = p2
|
219
219
|
p2.related = p1
|
220
220
|
p0.related = p1
|
221
|
-
@store.
|
222
|
-
@store.gc
|
221
|
+
expect(@store.check).to eq(0)
|
222
|
+
expect(@store.gc).to eq(0)
|
223
|
+
p0 = p1 = p2 = nil
|
224
|
+
GC.start
|
223
225
|
@store = PEROBS::Store.new(@db_file)
|
224
226
|
expect(@store['person0']._id).to eq(id0)
|
225
227
|
expect(@store['person0'].related._id).to eq(id1)
|
226
228
|
expect(@store['person0'].related.related._id).to eq(id2)
|
227
229
|
|
228
230
|
@store['person0'].related = nil
|
229
|
-
@store.gc
|
231
|
+
expect(@store.gc).to eq(2)
|
232
|
+
GC.start
|
230
233
|
expect(@store.object_by_id(id1)).to be_nil
|
231
234
|
expect(@store.object_by_id(id2)).to be_nil
|
232
235
|
|
233
236
|
@store = PEROBS::Store.new(@db_file)
|
237
|
+
expect(@store.check).to eq(0)
|
234
238
|
expect(@store.object_by_id(id1)).to be_nil
|
235
239
|
expect(@store.object_by_id(id2)).to be_nil
|
236
240
|
end
|
@@ -392,6 +396,17 @@ describe PEROBS::Store do
|
|
392
396
|
expect(@store['person2']).to be_nil
|
393
397
|
end
|
394
398
|
|
399
|
+
it 'should track in-memory objects properly' do
|
400
|
+
@store = PEROBS::Store.new(@db_file)
|
401
|
+
@store['person'] = @store.new(Person)
|
402
|
+
# We have the root hash and the Person object.
|
403
|
+
expect(@store.statistics[:in_memory_objects]).to eq(2)
|
404
|
+
@store.sync
|
405
|
+
GC.start
|
406
|
+
# Now the Person should be gone from memory.
|
407
|
+
expect(@store.statistics[:in_memory_objects]).to eq(1)
|
408
|
+
end
|
409
|
+
|
395
410
|
it 'should survive a real world usage test' do
|
396
411
|
options = { :engine => PEROBS::BTreeDB, :dir_bits => 4 }
|
397
412
|
@store = PEROBS::Store.new(@db_file, options)
|
@@ -437,7 +452,7 @@ describe PEROBS::Store do
|
|
437
452
|
when 6
|
438
453
|
if rand(50) == 0
|
439
454
|
@store.sync
|
440
|
-
@store.check(false)
|
455
|
+
expect(@store.check(false)).to eq(0)
|
441
456
|
end
|
442
457
|
when 7
|
443
458
|
index = rand(i)
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: perobs
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.0
|
4
|
+
version: 2.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Chris Schlaeger
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-01-
|
11
|
+
date: 2016-01-09 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|