perobs 2.0.1 → 2.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +69 -27
- data/lib/perobs/Array.rb +15 -1
- data/lib/perobs/BTreeBlob.rb +13 -2
- data/lib/perobs/BTreeDB.rb +5 -2
- data/lib/perobs/Cache.rb +2 -2
- data/lib/perobs/DynamoDB.rb +9 -2
- data/lib/perobs/Hash.rb +19 -2
- data/lib/perobs/Object.rb +10 -22
- data/lib/perobs/ObjectBase.rb +34 -12
- data/lib/perobs/Store.rb +113 -62
- data/lib/perobs/version.rb +1 -1
- data/tasks/changelog.rake +1 -1
- data/test/Array_spec.rb +19 -0
- data/test/Hash_spec.rb +11 -0
- data/test/Object_spec.rb +17 -0
- data/test/Store_spec.rb +20 -5
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 50292da8f34e68427887d224c56dc2070242fc27
|
4
|
+
data.tar.gz: f3f41eca90b32ebefeeae05eaf655e99130a3af8
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 2ed683b2e8ae0cd9a6bc4531a8f70f6d2f0d98dc5f3f4947ea64f24d715e3c5d050317d6c29ab481820f25cb302e4d8c1e67ca5cf8048c0e1354d732489aef94
|
7
|
+
data.tar.gz: 2dce8dff389bc974126a88e9fdd5d042f5071e13c6560a2eeab63b05ae536ac209dfa4cddbcccac0e4a2332d46c82ea27894cc0e3a7de1c32b016f515920fbd6
|
data/README.md
CHANGED
@@ -3,48 +3,77 @@
|
|
3
3
|
PEROBS is a library that provides a persistent object store for Ruby
|
4
4
|
objects. Objects of your classes can be made persistent by deriving
|
5
5
|
them from PEROBS::Object. They will be in memory when needed and
|
6
|
-
transparently stored into a persistent storage.
|
7
|
-
filesystem based storage is supported, but back-ends for key/value
|
8
|
-
databases can be easily added.
|
6
|
+
transparently stored into a persistent storage.
|
9
7
|
|
10
8
|
This library is ideal for Ruby applications that work on huge, mostly
|
11
9
|
constant data sets and usually handle a small subset of the data at a
|
12
10
|
time. To ensure data consistency of a larger data set, you can use
|
13
|
-
transactions to make modifications of multiple objects
|
11
|
+
transactions to make modifications of multiple objects atomicaly.
|
14
12
|
Transactions can be nested and are aborted when an exception is
|
15
13
|
raised.
|
16
14
|
|
17
15
|
## Usage
|
18
16
|
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
persistend back-end when not using transactions.
|
23
|
-
|
24
|
-
Persistent objects must be created by deriving your class from
|
25
|
-
PEROBS::Object. Only instance variables that are declared via
|
26
|
-
po_attr will be persistent. All objects that are stored in persistent
|
27
|
-
instance variables must provide a to_json() method that generates JSON
|
28
|
-
syntax that can be also parsed into their original object again. It is
|
29
|
-
required that references to other objects are all going to persistent
|
30
|
-
objects again.
|
31
|
-
|
32
|
-
There are currently 3 kinds of persistent objects available:
|
17
|
+
The objects that you want to persist must be of a class that has been
|
18
|
+
derived from PEROBS::BaseObject. PEROBS already provides 3 such
|
19
|
+
classes:
|
33
20
|
|
34
21
|
* PEROBS::Object is the base class for all your classes that should be
|
35
|
-
persistent.
|
22
|
+
persistent. You can determine which instance variables should be
|
23
|
+
persisted and what default values should be used.
|
36
24
|
|
37
25
|
* PEROBS::Array provides an interface similar to the built-in Array class
|
38
|
-
but its objects are automatically
|
26
|
+
but its objects are automatically persisted.
|
39
27
|
|
40
28
|
* PEROBS::Hash provides an interface similar to the built-in Hash
|
41
|
-
class but its objects are automatically
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
29
|
+
class but its objects are automatically persisted.
|
30
|
+
|
31
|
+
When you derive your own class from PEROBS::Object you need to
|
32
|
+
specify which instance variables should be persistent. By using
|
33
|
+
po_attr you can provide a list of symbols that describe the instance
|
34
|
+
variables to persist. This will also create getter and setter methods
|
35
|
+
for these instance varables. You can use attr_init in the constructor
|
36
|
+
to define default values for your persistent objects.
|
37
|
+
|
38
|
+
To start off you must create at least one PEROBS::Store object that
|
39
|
+
owns your persistent objects. The store provides the persistent
|
40
|
+
database. A persistent object is tied to the creating store for its
|
41
|
+
whole lifetime. By default, PEROBS::Store uses an on-disk database in the
|
42
|
+
directory you specify. But you can use key/value databases as well.
|
43
|
+
Currently only Amazon DynamoDB is supported. You can create your own
|
44
|
+
key/value database wrapper with little effort.
|
45
|
+
|
46
|
+
When creating the store you can also specify the serializer to use.
|
47
|
+
The serializer controls how your data is converted to be stored in the
|
48
|
+
database. The default serializer (JSON), you can only use the subset
|
49
|
+
of Ruby types that JSON supports. See http://www.json.org/ for
|
50
|
+
details. Alternatively, you can use Marshal or YAML which support
|
51
|
+
almost every Ruby data type. YAML is much slower than JSON and Marshal
|
52
|
+
is not guaranteed to be compatible between Ruby versions.
|
53
|
+
|
54
|
+
Once you have created a store you can assign objects to it. All
|
55
|
+
persistent objects must be created with @store.new(). This is
|
56
|
+
necessary as you will only deal with proxy objects in your code.
|
57
|
+
Except for the member methods, you will never deal with the objects
|
58
|
+
directly. Instead @store.new() returns a POXReference object that acts
|
59
|
+
as a transparent proxy. This proxy is needed as your code never knows
|
60
|
+
if the actual object is really loaded into the memory or not. PEROBS
|
61
|
+
will handle this transparently for you.
|
62
|
+
|
63
|
+
A build-in cache keeps access latencies to recently used objects low
|
64
|
+
and lazily flushes modified objects into the persistend back-end when
|
65
|
+
not using transactions. It also features a garbage collector that
|
66
|
+
removes all objects that are no longer in use.
|
67
|
+
|
68
|
+
So what does 'in use' mean? You can assign a few objects to the store
|
69
|
+
directly. The store acts like a hash. These root objects can then
|
70
|
+
reference other persistent objects and so on. The garbage collector
|
71
|
+
will find all objects that are reachable from the root objects and
|
72
|
+
discards all other from the database. You have to invoke the garbage
|
73
|
+
collector manually with Store.gc(). Depending on the size of your
|
74
|
+
database it can take some time. It is recommended that you don't use
|
75
|
+
persistend objects for temporary objects in your code. Every created
|
76
|
+
object will end up in the database end needs to be garbage collected.
|
48
77
|
|
49
78
|
Here is an example how to use PEROBS. Let's define a class that models
|
50
79
|
a person with their family relations.
|
@@ -106,6 +135,19 @@ accesses, you must manually mark the instance as modified by calling
|
|
106
135
|
Object::mark_as_modified(). If that is forgotten, the change will
|
107
136
|
reside in memory but might not be persisted into the database.
|
108
137
|
|
138
|
+
### Use of proxy objects
|
139
|
+
|
140
|
+
Your code should never deal with the persistent objects directly. The
|
141
|
+
PEROBS API takes care that you will always get a proxy object. The
|
142
|
+
only exception to this rule is the code in the instance methods. By
|
143
|
+
design, this code operates on the real object. The only caveat here is
|
144
|
+
the use of self(). If you pass the result of self() to another object
|
145
|
+
you will leak a PEROBS::ObjectBase derived object into your data
|
146
|
+
structures. PEROBS will watch for this and will throw an exception
|
147
|
+
when it detects such objects. Just remember to use myself() instead of
|
148
|
+
self() if you want to pass a reference to the current persistent
|
149
|
+
object to another object.
|
150
|
+
|
109
151
|
## Installation
|
110
152
|
|
111
153
|
Add this line to your application's Gemfile:
|
data/lib/perobs/Array.rb
CHANGED
@@ -119,7 +119,21 @@ module PEROBS
|
|
119
119
|
|
120
120
|
def _serialize
|
121
121
|
@data.map do |v|
|
122
|
-
v.respond_to?(:is_poxreference?)
|
122
|
+
if v.respond_to?(:is_poxreference?)
|
123
|
+
POReference.new(v.id)
|
124
|
+
else
|
125
|
+
# Outside of the PEROBS library all PEROBS::ObjectBase derived
|
126
|
+
# objects should not be used directly. The library only exposes them
|
127
|
+
# via POXReference proxy objects.
|
128
|
+
if v.is_a?(ObjectBase)
|
129
|
+
raise RuntimeError, 'A PEROBS::ObjectBase object escaped! ' +
|
130
|
+
"It is stored in a PEROBS::Array at index #{@data.index(v)}. " +
|
131
|
+
'Have you used self() instead of myself() to' +
|
132
|
+
"get the reference of this PEROBS object?\n" +
|
133
|
+
v.inspect
|
134
|
+
end
|
135
|
+
v
|
136
|
+
end
|
123
137
|
end
|
124
138
|
end
|
125
139
|
|
data/lib/perobs/BTreeBlob.rb
CHANGED
@@ -2,7 +2,7 @@
|
|
2
2
|
#
|
3
3
|
# = BTreeBlob.rb -- Persistent Ruby Object Store
|
4
4
|
#
|
5
|
-
# Copyright (c) 2015 by Chris Schlaeger <chris@taskjuggler.org>
|
5
|
+
# Copyright (c) 2015, 2016 by Chris Schlaeger <chris@taskjuggler.org>
|
6
6
|
#
|
7
7
|
# MIT License
|
8
8
|
#
|
@@ -137,12 +137,23 @@ module PEROBS
|
|
137
137
|
end
|
138
138
|
|
139
139
|
# Remove all entries from the index that have not been marked.
|
140
|
+
# @return [Array] List of deleted object IDs.
|
140
141
|
def delete_unmarked_entries
|
142
|
+
deleted_ids = []
|
141
143
|
# First remove the entry from the hash table.
|
142
|
-
@entries_by_id.delete_if
|
144
|
+
@entries_by_id.delete_if do |id, e|
|
145
|
+
if e[MARKED] == 0
|
146
|
+
deleted_ids << id
|
147
|
+
true
|
148
|
+
else
|
149
|
+
false
|
150
|
+
end
|
151
|
+
end
|
143
152
|
# Then delete the entry itself.
|
144
153
|
@entries.delete_if { |e| e[MARKED] == 0 }
|
145
154
|
write_index
|
155
|
+
|
156
|
+
deleted_ids
|
146
157
|
end
|
147
158
|
|
148
159
|
# Run a basic consistency check.
|
data/lib/perobs/BTreeDB.rb
CHANGED
@@ -2,7 +2,7 @@
|
|
2
2
|
#
|
3
3
|
# = BTreeDB.rb -- Persistent Ruby Object Store
|
4
4
|
#
|
5
|
-
# Copyright (c) 2015 by Chris Schlaeger <chris@taskjuggler.org>
|
5
|
+
# Copyright (c) 2015, 2016 by Chris Schlaeger <chris@taskjuggler.org>
|
6
6
|
#
|
7
7
|
# MIT License
|
8
8
|
#
|
@@ -158,8 +158,11 @@ module PEROBS
|
|
158
158
|
|
159
159
|
# Permanently delete all objects that have not been marked. Those are
|
160
160
|
# orphaned and are no longer referenced by any actively used object.
|
161
|
+
# @return [Array] List of IDs that have been removed from the DB.
|
161
162
|
def delete_unmarked_objects
|
162
|
-
|
163
|
+
deleted_ids = []
|
164
|
+
each_blob { |blob| deleted_ids += blob.delete_unmarked_entries }
|
165
|
+
deleted_ids
|
163
166
|
end
|
164
167
|
|
165
168
|
# Mark an object.
|
data/lib/perobs/Cache.rb
CHANGED
@@ -2,7 +2,7 @@
|
|
2
2
|
#
|
3
3
|
# = Cache.rb -- Persistent Ruby Object Store
|
4
4
|
#
|
5
|
-
# Copyright (c) 2015 by Chris Schlaeger <chris@taskjuggler.org>
|
5
|
+
# Copyright (c) 2015, 2016 by Chris Schlaeger <chris@taskjuggler.org>
|
6
6
|
#
|
7
7
|
# MIT License
|
8
8
|
#
|
@@ -31,7 +31,7 @@ module PEROBS
|
|
31
31
|
|
32
32
|
# The Cache provides two functions for the PEROBS Store. It keeps some
|
33
33
|
# amount of objects in memory to substantially reduce read access latencies.
|
34
|
-
# It
|
34
|
+
# It also stores a list of objects that haven't been synced to the
|
35
35
|
# permanent store yet to accelerate object writes.
|
36
36
|
class Cache
|
37
37
|
|
data/lib/perobs/DynamoDB.rb
CHANGED
@@ -2,7 +2,7 @@
|
|
2
2
|
#
|
3
3
|
# = DynamoDB.rb -- Persistent Ruby Object Store
|
4
4
|
#
|
5
|
-
# Copyright (c) 2015 by Chris Schlaeger <chris@taskjuggler.org>
|
5
|
+
# Copyright (c) 2015, 2016 by Chris Schlaeger <chris@taskjuggler.org>
|
6
6
|
#
|
7
7
|
# MIT License
|
8
8
|
#
|
@@ -134,10 +134,17 @@ module PEROBS
|
|
134
134
|
|
135
135
|
# Permanently delete all objects that have not been marked. Those are
|
136
136
|
# orphaned and are no longer referenced by any actively used object.
|
137
|
+
# @return [Array] List of object IDs of the deleted objects.
|
137
138
|
def delete_unmarked_objects
|
139
|
+
deleted_ids = []
|
138
140
|
each_item do |id|
|
139
|
-
|
141
|
+
unless dynamo_is_marked?(id)
|
142
|
+
dynamo_delete_item(id)
|
143
|
+
deleted_ids << id
|
144
|
+
end
|
140
145
|
end
|
146
|
+
|
147
|
+
deleted_ids
|
141
148
|
end
|
142
149
|
|
143
150
|
# Mark an object.
|
data/lib/perobs/Hash.rb
CHANGED
@@ -118,8 +118,25 @@ module PEROBS
|
|
118
118
|
|
119
119
|
def _serialize
|
120
120
|
data = {}
|
121
|
-
|
122
|
-
|
121
|
+
|
122
|
+
@data.each do |k, v|
|
123
|
+
if v.respond_to?(:is_poxreference?)
|
124
|
+
data[k] = POReference.new(v.id)
|
125
|
+
else
|
126
|
+
# Outside of the PEROBS library all PEROBS::ObjectBase derived
|
127
|
+
# objects should not be used directly. The library only exposes them
|
128
|
+
# via POXReference proxy objects.
|
129
|
+
if v.is_a?(ObjectBase)
|
130
|
+
raise RuntimeError, 'A PEROBS::ObjectBase object escaped! ' +
|
131
|
+
"It is stored in a PEROBS::Hash with key #{k.inspect}. " +
|
132
|
+
'Have you used self() instead of myself() to' +
|
133
|
+
"get the reference of this PEROBS object?\n" +
|
134
|
+
v.inspect
|
135
|
+
end
|
136
|
+
data[k] = v
|
137
|
+
end
|
138
|
+
end
|
139
|
+
|
123
140
|
data
|
124
141
|
end
|
125
142
|
|
data/lib/perobs/Object.rb
CHANGED
@@ -175,13 +175,6 @@ module PEROBS
|
|
175
175
|
_all_attributes.each do |attr|
|
176
176
|
ivar = ('@' + attr.to_s).to_sym
|
177
177
|
value = instance_variable_get(ivar)
|
178
|
-
#if (value = instance_variable_get(ivar)).is_a?(ObjectBase)
|
179
|
-
# raise ArgumentError, "The instance variable #{ivar} contains a " +
|
180
|
-
# "reference to a PEROBS::ObjectBase object! " +
|
181
|
-
# "This is not allowed. You must use the " +
|
182
|
-
# "accessor method to assign a reference to " +
|
183
|
-
# "another PEROBS object."
|
184
|
-
#end
|
185
178
|
attributes[attr.to_s] = value.respond_to?(:is_poxreference?) ?
|
186
179
|
POReference.new(value.id) : value
|
187
180
|
end
|
@@ -189,33 +182,28 @@ module PEROBS
|
|
189
182
|
end
|
190
183
|
|
191
184
|
def _set(attr, val)
|
192
|
-
|
193
|
-
if !val.respond_to?(:is_poxreference?) && val.is_a?(ObjectBase)
|
185
|
+
if val.is_a?(ObjectBase)
|
194
186
|
# References to other PEROBS::Objects must be handled somewhat
|
195
187
|
# special.
|
196
188
|
if @store != val.store
|
197
189
|
raise ArgumentError, 'The referenced object is not part of this store'
|
198
190
|
end
|
199
|
-
|
200
|
-
|
201
|
-
|
202
|
-
|
203
|
-
|
204
|
-
|
191
|
+
unless val.respond_to?(:is_poxreference?)
|
192
|
+
raise ArgumentError, 'A PEROBS::ObjectBase object escaped! ' +
|
193
|
+
'Have you used self() instead of myself() to' +
|
194
|
+
'get the reference of the PEROBS object that ' +
|
195
|
+
'you are trying to assign here?'
|
196
|
+
end
|
205
197
|
end
|
198
|
+
instance_variable_set(('@' + attr.to_s).to_sym, val)
|
206
199
|
# Let the store know that we have a modified object.
|
207
|
-
|
200
|
+
mark_as_modified
|
208
201
|
|
209
202
|
val
|
210
203
|
end
|
211
204
|
|
212
205
|
def _get(attr)
|
213
|
-
|
214
|
-
if value.respond_to?(:is_poxreference?)
|
215
|
-
@store.object_by_id(value.id)
|
216
|
-
else
|
217
|
-
value
|
218
|
-
end
|
206
|
+
instance_variable_get(('@' + attr.to_s).to_sym)
|
219
207
|
end
|
220
208
|
|
221
209
|
def _all_attributes
|
data/lib/perobs/ObjectBase.rb
CHANGED
@@ -2,7 +2,7 @@
|
|
2
2
|
#
|
3
3
|
# = ObjectBase.rb -- Persistent Ruby Object Store
|
4
4
|
#
|
5
|
-
# Copyright (c) 2015 by Chris Schlaeger <chris@taskjuggler.org>
|
5
|
+
# Copyright (c) 2015, 2016 by Chris Schlaeger <chris@taskjuggler.org>
|
6
6
|
#
|
7
7
|
# MIT License
|
8
8
|
#
|
@@ -34,7 +34,7 @@ module PEROBS
|
|
34
34
|
# since it's no longer referenced once it has been evicted from the
|
35
35
|
# PEROBS::Store cache. The POXReference objects function as a transparent
|
36
36
|
# proxy for the objects they are referencing.
|
37
|
-
class POXReference
|
37
|
+
class POXReference < BasicObject
|
38
38
|
|
39
39
|
attr_reader :store, :id
|
40
40
|
|
@@ -47,12 +47,13 @@ module PEROBS
|
|
47
47
|
# Proxy all calls to unknown methods to the referenced object.
|
48
48
|
def method_missing(method_sym, *args, &block)
|
49
49
|
unless (obj = _referenced_object)
|
50
|
-
raise ::RuntimeError,
|
51
|
-
"ID #{@id} found in
|
50
|
+
::Kernel.raise ::RuntimeError,
|
51
|
+
"Internal consistency error. No object with ID #{@id} found in " +
|
52
|
+
'the store.'
|
52
53
|
end
|
53
54
|
if obj.respond_to?(:is_poxreference?)
|
54
|
-
raise ::RuntimeError,
|
55
|
-
"POXReference that references a POXReference found"
|
55
|
+
::Kernel.raise ::RuntimeError,
|
56
|
+
"POXReference that references a POXReference found."
|
56
57
|
end
|
57
58
|
obj.send(method_sym, *args, &block)
|
58
59
|
end
|
@@ -86,6 +87,18 @@ module PEROBS
|
|
86
87
|
_referenced_object == obj
|
87
88
|
end
|
88
89
|
|
90
|
+
# BasicObject provides a equal?() method that prevents method_missing from
|
91
|
+
# being called. So we have to pass the call manually to the referenced
|
92
|
+
# object.
|
93
|
+
# @param obj object to compare this object with.
|
94
|
+
def equal?(obj)
|
95
|
+
if obj.respond_to?(:is_poxreference?)
|
96
|
+
_referenced_object.equal?(obj._referenced_object)
|
97
|
+
else
|
98
|
+
_referenced_object.equal?(obj)
|
99
|
+
end
|
100
|
+
end
|
101
|
+
|
89
102
|
# Shortcut to access the _id() method of the referenced object.
|
90
103
|
def _id
|
91
104
|
@id
|
@@ -102,7 +115,7 @@ module PEROBS
|
|
102
115
|
# common to all classes of persistent objects.
|
103
116
|
class ObjectBase
|
104
117
|
|
105
|
-
attr_reader :_id, :store
|
118
|
+
attr_reader :_id, :store, :myself
|
106
119
|
|
107
120
|
# New PEROBS objects must always be created by calling # Store.new().
|
108
121
|
# PEROBS users should never call this method or equivalents of derived
|
@@ -110,17 +123,29 @@ module PEROBS
|
|
110
123
|
def initialize(store)
|
111
124
|
@store = store
|
112
125
|
unless @store.object_creation_in_progress
|
113
|
-
raise ::RuntimeError,
|
126
|
+
::Kernel.raise ::RuntimeError,
|
114
127
|
"All PEROBS objects must exclusively be created by calling " +
|
115
128
|
"Store.new(). Never call the object constructor directly."
|
116
129
|
end
|
117
130
|
@_id = @store.db.new_id
|
131
|
+
ObjectSpace.define_finalizer(self, ObjectBase._finalize(@store, @_id))
|
118
132
|
@_stash_map = nil
|
133
|
+
# Allocate a proxy object for this object. User code should only operate
|
134
|
+
# on this proxy, never on self.
|
135
|
+
@myself = POXReference.new(@store, @_id)
|
119
136
|
|
120
137
|
# Let the store know that we have a modified object.
|
121
138
|
@store.cache.cache_write(self)
|
122
139
|
end
|
123
140
|
|
141
|
+
# This method generates the destructor for the objects of this class. It
|
142
|
+
# is done this way to prevent the Proc object hanging on to a reference to
|
143
|
+
# self which would prevent the object from being collected. This internal
|
144
|
+
# method is not intended for users to call.
|
145
|
+
def ObjectBase._finalize(store, id)
|
146
|
+
proc { store._collect(id) }
|
147
|
+
end
|
148
|
+
|
124
149
|
public
|
125
150
|
|
126
151
|
# This method can be overloaded by derived classes to do some massaging on
|
@@ -157,10 +182,7 @@ module PEROBS
|
|
157
182
|
|
158
183
|
klass = store.class_map.id_to_class(db_obj['class_id'])
|
159
184
|
# Call the constructor of the specified class.
|
160
|
-
obj = store.
|
161
|
-
# The object gets created with a new ID by default. We need to restore
|
162
|
-
# the old one.
|
163
|
-
obj._change_id(id)
|
185
|
+
obj = store._construct_po(Object.const_get(klass), id)
|
164
186
|
obj._deserialize(db_obj['data'])
|
165
187
|
obj.post_restore
|
166
188
|
|
data/lib/perobs/Store.rb
CHANGED
@@ -2,7 +2,7 @@
|
|
2
2
|
#
|
3
3
|
# = Store.rb -- Persistent Ruby Object Store
|
4
4
|
#
|
5
|
-
# Copyright (c) 2015 by Chris Schlaeger <chris@taskjuggler.org>
|
5
|
+
# Copyright (c) 2015, 2016 by Chris Schlaeger <chris@taskjuggler.org>
|
6
6
|
#
|
7
7
|
# MIT License
|
8
8
|
#
|
@@ -26,6 +26,7 @@
|
|
26
26
|
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
27
27
|
|
28
28
|
require 'set'
|
29
|
+
require 'weakref'
|
29
30
|
|
30
31
|
require 'perobs/Cache'
|
31
32
|
require 'perobs/ClassMap'
|
@@ -128,16 +129,18 @@ module PEROBS
|
|
128
129
|
# the Store.new() call by PEROBS users.
|
129
130
|
@object_creation_in_progress = false
|
130
131
|
|
132
|
+
# List of PEROBS objects that are currently available as Ruby objects
|
133
|
+
# hashed by their ID.
|
134
|
+
@in_memory_objects = {}
|
135
|
+
|
131
136
|
# The Cache reduces read and write latencies by keeping a subset of the
|
132
137
|
# objects in memory.
|
133
138
|
@cache = Cache.new(options[:cache_bits] || 16)
|
134
139
|
|
135
140
|
# The named (global) objects IDs hashed by their name
|
136
141
|
unless (@root_objects = object_by_id(0))
|
137
|
-
@root_objects = construct_po(Hash)
|
138
|
-
|
139
142
|
# The root object hash always has the object ID 0.
|
140
|
-
@root_objects
|
143
|
+
@root_objects = _construct_po(Hash, 0)
|
141
144
|
# The ID change removes it from the write cache. We need to add it
|
142
145
|
# again.
|
143
146
|
@cache.cache_write(@root_objects)
|
@@ -152,26 +155,39 @@ module PEROBS
|
|
152
155
|
# constructor of the specified class.
|
153
156
|
# @return [POXReference] A reference to the newly created object.
|
154
157
|
def new(klass, *args)
|
155
|
-
|
158
|
+
_construct_po(klass, nil, *args).myself
|
156
159
|
end
|
157
160
|
|
158
161
|
# For library internal use only!
|
159
|
-
|
162
|
+
# This method will create a new PEROBS object.
|
163
|
+
# @param klass [BasicObject] Class of the object to create
|
164
|
+
# @param id [Fixnum, Bignum or nil] Requested object ID or nil
|
165
|
+
# @param *args [Array] Arguments to pass to the object constructor.
|
166
|
+
# @return [BasicObject] Newly constructed PEROBS object
|
167
|
+
def _construct_po(klass, id, *args)
|
168
|
+
unless klass.is_a?(BasicObject)
|
169
|
+
raise ArgumentError, "#{klass} is not a BasicObject derivative"
|
170
|
+
end
|
160
171
|
@object_creation_in_progress = true
|
161
172
|
obj = klass.new(self, *args)
|
162
173
|
@object_creation_in_progress = false
|
174
|
+
# If a specific object ID was requested we need to set it now.
|
175
|
+
obj._change_id(id) if id
|
176
|
+
# Add the new object to the in-memory list. We only store a weak
|
177
|
+
# reference to the object so it can be garbage collected. When this
|
178
|
+
# happens the object finalizer is triggered and calls _forget() to
|
179
|
+
# remove the object from this hash again.
|
180
|
+
@in_memory_objects[obj._id] = WeakRef.new(obj)
|
163
181
|
obj
|
164
182
|
end
|
165
183
|
|
166
|
-
|
167
184
|
# Delete the entire store. The store is no longer usable after this
|
168
185
|
# method was called.
|
169
186
|
def delete_store
|
170
187
|
@db.delete_database
|
171
|
-
@class_map = @cache = @root_objects = nil
|
188
|
+
@db = @class_map = @cache = @root_objects = nil
|
172
189
|
end
|
173
190
|
|
174
|
-
|
175
191
|
# Store the provided object under the given name. Use this to make the
|
176
192
|
# object a root or top-level object (think global variable). Each store
|
177
193
|
# should have at least one root object. Objects that are not directly or
|
@@ -187,13 +203,10 @@ module PEROBS
|
|
187
203
|
return nil
|
188
204
|
end
|
189
205
|
|
190
|
-
if obj.respond_to?(:is_poxreference?)
|
191
|
-
obj = obj._referenced_object
|
192
|
-
end
|
193
206
|
# We only allow derivatives of PEROBS::Object to be stored in the
|
194
207
|
# store.
|
195
208
|
unless obj.is_a?(ObjectBase)
|
196
|
-
raise ArgumentError,
|
209
|
+
raise ArgumentError, 'Object must be of class PEROBS::Object but ' +
|
197
210
|
"is of class #{obj.class}"
|
198
211
|
end
|
199
212
|
|
@@ -203,8 +216,6 @@ module PEROBS
|
|
203
216
|
|
204
217
|
# Store the name and mark the name list as modified.
|
205
218
|
@root_objects[name] = obj._id
|
206
|
-
# Add the object to the in-memory storage list.
|
207
|
-
@cache.cache_write(obj)
|
208
219
|
|
209
220
|
obj
|
210
221
|
end
|
@@ -217,7 +228,7 @@ module PEROBS
|
|
217
228
|
# Return nil if there is no object with that name.
|
218
229
|
return nil unless (id = @root_objects[name])
|
219
230
|
|
220
|
-
|
231
|
+
POXReference.new(self, id)
|
221
232
|
end
|
222
233
|
|
223
234
|
# Flush out all modified objects to disk and shrink the in-memory list if
|
@@ -233,6 +244,7 @@ module PEROBS
|
|
233
244
|
# from the back-end storage. The garbage collector is not invoked
|
234
245
|
# automatically. Depending on your usage pattern, you need to call this
|
235
246
|
# method periodically.
|
247
|
+
# @return [Fixnum] The number of collected objects
|
236
248
|
def gc
|
237
249
|
sync
|
238
250
|
mark
|
@@ -243,21 +255,27 @@ module PEROBS
|
|
243
255
|
# public API and should never be called by outside users. It's purely
|
244
256
|
# intended for internal use.
|
245
257
|
def object_by_id(id)
|
246
|
-
if (obj = @
|
258
|
+
if (obj = @in_memory_objects[id])
|
247
259
|
# We have the object in memory so we can just return it.
|
248
|
-
|
249
|
-
|
250
|
-
|
251
|
-
|
252
|
-
#
|
253
|
-
|
254
|
-
# Add the object to the in-memory storage list.
|
255
|
-
@cache.cache_read(obj)
|
256
|
-
|
257
|
-
return obj
|
260
|
+
begin
|
261
|
+
return obj.__getobj__
|
262
|
+
rescue WeakRef::RefError
|
263
|
+
# Due to a race condition the object can still be in the
|
264
|
+
# @in_memory_objects list but has been collected already by the Ruby
|
265
|
+
# GC. In that case we need to load it again.
|
258
266
|
end
|
259
267
|
end
|
260
268
|
|
269
|
+
# We don't have the object in memory. Let's find it in the storage.
|
270
|
+
if @db.include?(id)
|
271
|
+
# Great, object found. Read it into memory and return it.
|
272
|
+
obj = ObjectBase::read(self, id)
|
273
|
+
# Add the object to the in-memory storage list.
|
274
|
+
@cache.cache_read(obj)
|
275
|
+
|
276
|
+
return obj
|
277
|
+
end
|
278
|
+
|
261
279
|
# The requested object does not exist. Return nil.
|
262
280
|
nil
|
263
281
|
end
|
@@ -268,47 +286,24 @@ module PEROBS
|
|
268
286
|
# unreadable object is found, the reference will simply be deleted.
|
269
287
|
# @param repair [TrueClass/FalseClass] true if a repair attempt should be
|
270
288
|
# made.
|
289
|
+
# @return [Fixnum] The number of references to bad objects found.
|
271
290
|
def check(repair = true)
|
291
|
+
# All objects must have in-db version.
|
292
|
+
sync
|
272
293
|
# Run basic consistency checks first.
|
273
294
|
@db.check_db(repair)
|
274
295
|
|
296
|
+
# We will use the mark to mark all objects that we have checked already.
|
297
|
+
# Before we start, we need to clear all marks.
|
275
298
|
@db.clear_marks
|
276
|
-
|
277
|
-
|
278
|
-
# First we check the top-level objects. They are only added to the
|
279
|
-
# working set if they are OK.
|
299
|
+
|
300
|
+
errors = 0
|
280
301
|
@root_objects.each do |name, id|
|
281
|
-
|
282
|
-
stack << id
|
283
|
-
end
|
302
|
+
errors += check_object(id, repair)
|
284
303
|
end
|
285
|
-
|
286
|
-
# Delete any top-level object that is defective.
|
287
|
-
stack.each { |id| @root_objects.delete(id) }
|
288
|
-
# The remaining top-level objects are the initial working set.
|
289
|
-
stack = @root_objects.values
|
290
|
-
else
|
291
|
-
# The initial working set must only be OK objects.
|
292
|
-
stack = @root_objects.values - stack
|
293
|
-
end
|
294
|
-
stack.each { |id| @db.mark(id) }
|
304
|
+
@root_objects.delete_if { |name, id| !@db.check(id, false) }
|
295
305
|
|
296
|
-
|
297
|
-
id = stack.pop
|
298
|
-
(obj = object_by_id(id))._referenced_object_ids.each do |id|
|
299
|
-
# Add all found references that have passed the check to the working
|
300
|
-
# list for the next iterations.
|
301
|
-
if @db.check(id, repair)
|
302
|
-
unless @db.is_marked?(id)
|
303
|
-
stack << id
|
304
|
-
@db.mark(id)
|
305
|
-
end
|
306
|
-
elsif repair
|
307
|
-
# Remove references to bad objects.
|
308
|
-
obj._delete_reference_to_id(id)
|
309
|
-
end
|
310
|
-
end
|
311
|
-
end
|
306
|
+
errors
|
312
307
|
end
|
313
308
|
|
314
309
|
# This method will execute the provided block as an atomic transaction
|
@@ -339,7 +334,7 @@ module PEROBS
|
|
339
334
|
while !stack.empty?
|
340
335
|
# Get an object index from the stack.
|
341
336
|
obj = object_by_id(id = stack.pop)
|
342
|
-
yield(
|
337
|
+
yield(POXReference.new(self, id)) if block_given?
|
343
338
|
obj._referenced_object_ids.each do |id|
|
344
339
|
unless @db.is_marked?(id)
|
345
340
|
@db.mark(id)
|
@@ -355,6 +350,24 @@ module PEROBS
|
|
355
350
|
@class_map.rename(rename_map)
|
356
351
|
end
|
357
352
|
|
353
|
+
# Remove the object from the in-memory list. This is an internal method
|
354
|
+
# and should never be called from user code.
|
355
|
+
# @param id [Fixnum or Bignum] Object ID of object to remove from the list
|
356
|
+
def _collect(id, ignore_errors = false)
|
357
|
+
unless ignore_errors || @in_memory_objects.include?(id)
|
358
|
+
raise RuntimeError, "Object with id #{id} is currently not in memory"
|
359
|
+
end
|
360
|
+
@in_memory_objects.delete(id)
|
361
|
+
end
|
362
|
+
|
363
|
+
# This method returns a Hash with some statistics about this store.
|
364
|
+
def statistics
|
365
|
+
{
|
366
|
+
:in_memory_objects => @in_memory_objects.length,
|
367
|
+
:root_objects => 0 #@root_objects.length
|
368
|
+
}
|
369
|
+
end
|
370
|
+
|
358
371
|
private
|
359
372
|
|
360
373
|
# Mark phase of a mark-and-sweep garbage collector. It will mark all
|
@@ -368,8 +381,46 @@ module PEROBS
|
|
368
381
|
# Sweep phase of a mark-and-sweep garbage collector. It will remove all
|
369
382
|
# unmarked objects from the store.
|
370
383
|
def sweep
|
371
|
-
@db.delete_unmarked_objects
|
384
|
+
cntr = @db.delete_unmarked_objects.length
|
372
385
|
@cache.reset
|
386
|
+
cntr
|
387
|
+
end
|
388
|
+
|
389
|
+
# Check the object with the given start_id and all other objects that are
|
390
|
+
# somehow reachable from the start object.
|
391
|
+
# @param start_id [Fixnum or Bignum] ID of the top-level object to start
|
392
|
+
# with
|
393
|
+
# @param repair [Boolean] Delete refernces to broken objects if true
|
394
|
+
# @return [Fixnum] The number of references to bad objects.
|
395
|
+
def check_object(start_id, repair)
|
396
|
+
errors = 0
|
397
|
+
@db.mark(start_id)
|
398
|
+
# The todo list holds a touple for each object that still needs to be
|
399
|
+
# checked. The first item is the referring object and the second is the
|
400
|
+
# ID of the object to check.
|
401
|
+
todo_list = [ [ nil, start_id ] ]
|
402
|
+
|
403
|
+
while !todo_list.empty?
|
404
|
+
# Get the next PEROBS object to check
|
405
|
+
ref_obj, id = todo_list.pop
|
406
|
+
|
407
|
+
if (obj = object_by_id(id)) && @db.check(id, repair)
|
408
|
+
# The object exists and is OK. Mark is as checked.
|
409
|
+
@db.mark(id)
|
410
|
+
# Now look at all other objects referenced by this object.
|
411
|
+
obj._referenced_object_ids.each do |refd_id|
|
412
|
+
# Push them onto the todo list unless they have been marked
|
413
|
+
# already.
|
414
|
+
todo_list << [ obj, refd_id ] unless @db.is_marked?(refd_id)
|
415
|
+
end
|
416
|
+
else
|
417
|
+
# Remove references to bad objects.
|
418
|
+
ref_obj._delete_reference_to_id(id) if ref_obj && repair
|
419
|
+
errors += 1
|
420
|
+
end
|
421
|
+
end
|
422
|
+
|
423
|
+
errors
|
373
424
|
end
|
374
425
|
|
375
426
|
end
|
data/lib/perobs/version.rb
CHANGED
data/tasks/changelog.rake
CHANGED
@@ -155,7 +155,7 @@ task :changelog do
|
|
155
155
|
def getReleaseVersions
|
156
156
|
# Get list of release tags from Git repository
|
157
157
|
releaseVersions = `git tag`.split("\n").map { |r| r.chomp }.
|
158
|
-
delete_if { |r| ! (/
|
158
|
+
delete_if { |r| ! (/v\d+\.\d+\.\d+/ =~ r) }.
|
159
159
|
sort{ |a, b| compareTags(a, b) }
|
160
160
|
releaseVersions << 'HEAD'
|
161
161
|
end
|
data/test/Array_spec.rb
CHANGED
@@ -40,6 +40,10 @@ class PO < PEROBS::Object
|
|
40
40
|
_set(:name, name)
|
41
41
|
end
|
42
42
|
|
43
|
+
def get_self
|
44
|
+
self # Never do this in real user code!
|
45
|
+
end
|
46
|
+
|
43
47
|
end
|
44
48
|
|
45
49
|
describe PEROBS::Array do
|
@@ -205,4 +209,19 @@ describe PEROBS::Array do
|
|
205
209
|
pcheck { expect(a).to eq([ 1, 2, 3, 4, 5, nil ]) }
|
206
210
|
end
|
207
211
|
|
212
|
+
it 'should only provide POXReference objects' do
|
213
|
+
a = cpa([ @store.new(PO), @store.new(PO) ])
|
214
|
+
expect(a[0].respond_to?(:is_poxreference?)).to be true
|
215
|
+
a.each do |a|
|
216
|
+
expect(a.respond_to?(:is_poxreference?)).to be true
|
217
|
+
end
|
218
|
+
end
|
219
|
+
|
220
|
+
it 'should catch a leaked PEROBS::ObjectBase object' do
|
221
|
+
@store['a'] = a = @store.new(PEROBS::Array)
|
222
|
+
o = @store.new(PO)
|
223
|
+
a[0] = o.get_self
|
224
|
+
expect { @store.sync }.to raise_error(RuntimeError)
|
225
|
+
end
|
226
|
+
|
208
227
|
end
|
data/test/Hash_spec.rb
CHANGED
@@ -41,6 +41,10 @@ class PO < PEROBS::Object
|
|
41
41
|
@name = name
|
42
42
|
end
|
43
43
|
|
44
|
+
def get_self
|
45
|
+
self # Never do this in real user code!
|
46
|
+
end
|
47
|
+
|
44
48
|
end
|
45
49
|
|
46
50
|
describe PEROBS::Hash do
|
@@ -154,4 +158,11 @@ describe PEROBS::Hash do
|
|
154
158
|
pcheck { expect(h2).to eq(hb) }
|
155
159
|
end
|
156
160
|
|
161
|
+
it 'should catch a leaked PEROBS::ObjectBase object' do
|
162
|
+
@store['a'] = a = @store.new(PEROBS::Hash)
|
163
|
+
o = @store.new(PO)
|
164
|
+
a['a'] = o.get_self
|
165
|
+
expect { @store.sync }.to raise_error(RuntimeError)
|
166
|
+
end
|
167
|
+
|
157
168
|
end
|
data/test/Object_spec.rb
CHANGED
@@ -51,6 +51,10 @@ class O2 < PEROBS::Object
|
|
51
51
|
@a3.a1
|
52
52
|
end
|
53
53
|
|
54
|
+
def get_self
|
55
|
+
self # Never do this in real user code!
|
56
|
+
end
|
57
|
+
|
54
58
|
end
|
55
59
|
|
56
60
|
class O3 < PEROBS::Object
|
@@ -125,6 +129,19 @@ describe PEROBS::Store do
|
|
125
129
|
expect(o2.a3_deref).to eq('a1')
|
126
130
|
end
|
127
131
|
|
132
|
+
it 'should always return a POXReference for a PEROBS object' do
|
133
|
+
@store['o1'] = o1 = @store.new(O1)
|
134
|
+
o1.a1 = @store.new(O2)
|
135
|
+
expect(@store['o1'].respond_to?(:is_poxreference?)).to be true
|
136
|
+
expect(o1.a1.respond_to?(:is_poxreference?)).to be true
|
137
|
+
end
|
138
|
+
|
139
|
+
it 'should catch a leaked PEROBS::ObjectBase object' do
|
140
|
+
@store['a'] = a = @store.new(O1)
|
141
|
+
o = @store.new(O2)
|
142
|
+
expect { a.a1 = o.get_self }.to raise_error(ArgumentError)
|
143
|
+
end
|
144
|
+
|
128
145
|
it 'should raise an error when no attributes are defined' do
|
129
146
|
@store['o3'] = @store.new(O3)
|
130
147
|
expect { @store.sync }.to raise_error(StandardError)
|
data/test/Store_spec.rb
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
# encoding: UTF-8
|
2
2
|
#
|
3
|
-
# Copyright (c) 2015 by Chris Schlaeger <chris@taskjuggler.org>
|
3
|
+
# Copyright (c) 2015, 2016 by Chris Schlaeger <chris@taskjuggler.org>
|
4
4
|
#
|
5
5
|
# MIT License
|
6
6
|
#
|
@@ -218,19 +218,23 @@ describe PEROBS::Store do
|
|
218
218
|
p1.related = p2
|
219
219
|
p2.related = p1
|
220
220
|
p0.related = p1
|
221
|
-
@store.
|
222
|
-
@store.gc
|
221
|
+
expect(@store.check).to eq(0)
|
222
|
+
expect(@store.gc).to eq(0)
|
223
|
+
p0 = p1 = p2 = nil
|
224
|
+
GC.start
|
223
225
|
@store = PEROBS::Store.new(@db_file)
|
224
226
|
expect(@store['person0']._id).to eq(id0)
|
225
227
|
expect(@store['person0'].related._id).to eq(id1)
|
226
228
|
expect(@store['person0'].related.related._id).to eq(id2)
|
227
229
|
|
228
230
|
@store['person0'].related = nil
|
229
|
-
@store.gc
|
231
|
+
expect(@store.gc).to eq(2)
|
232
|
+
GC.start
|
230
233
|
expect(@store.object_by_id(id1)).to be_nil
|
231
234
|
expect(@store.object_by_id(id2)).to be_nil
|
232
235
|
|
233
236
|
@store = PEROBS::Store.new(@db_file)
|
237
|
+
expect(@store.check).to eq(0)
|
234
238
|
expect(@store.object_by_id(id1)).to be_nil
|
235
239
|
expect(@store.object_by_id(id2)).to be_nil
|
236
240
|
end
|
@@ -392,6 +396,17 @@ describe PEROBS::Store do
|
|
392
396
|
expect(@store['person2']).to be_nil
|
393
397
|
end
|
394
398
|
|
399
|
+
it 'should track in-memory objects properly' do
|
400
|
+
@store = PEROBS::Store.new(@db_file)
|
401
|
+
@store['person'] = @store.new(Person)
|
402
|
+
# We have the root hash and the Person object.
|
403
|
+
expect(@store.statistics[:in_memory_objects]).to eq(2)
|
404
|
+
@store.sync
|
405
|
+
GC.start
|
406
|
+
# Now the Person should be gone from memory.
|
407
|
+
expect(@store.statistics[:in_memory_objects]).to eq(1)
|
408
|
+
end
|
409
|
+
|
395
410
|
it 'should survive a real world usage test' do
|
396
411
|
options = { :engine => PEROBS::BTreeDB, :dir_bits => 4 }
|
397
412
|
@store = PEROBS::Store.new(@db_file, options)
|
@@ -437,7 +452,7 @@ describe PEROBS::Store do
|
|
437
452
|
when 6
|
438
453
|
if rand(50) == 0
|
439
454
|
@store.sync
|
440
|
-
@store.check(false)
|
455
|
+
expect(@store.check(false)).to eq(0)
|
441
456
|
end
|
442
457
|
when 7
|
443
458
|
index = rand(i)
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: perobs
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.0
|
4
|
+
version: 2.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Chris Schlaeger
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-01-
|
11
|
+
date: 2016-01-09 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|