odba 1.1.8 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/devenv.yml +30 -0
- data/.github/workflows/ruby.yml +8 -7
- data/.gitignore +11 -0
- data/{History.txt → History.md} +43 -19
- data/README.md +79 -2
- data/Rakefile +3 -15
- data/devenv.lock +171 -0
- data/devenv.nix +53 -0
- data/devenv.yaml +8 -0
- data/lib/odba/cache.rb +401 -344
- data/lib/odba/cache_entry.rb +42 -31
- data/lib/odba/connection_pool.rb +99 -72
- data/lib/odba/drbwrapper.rb +26 -18
- data/lib/odba/id_server.rb +20 -20
- data/lib/odba/index.rb +249 -215
- data/lib/odba/index_definition.rb +17 -17
- data/lib/odba/marshal.rb +15 -14
- data/lib/odba/odba.rb +40 -32
- data/lib/odba/odba_error.rb +4 -2
- data/lib/odba/persistable.rb +460 -414
- data/lib/odba/storage.rb +328 -277
- data/lib/odba/stub.rb +166 -153
- data/lib/odba/version.rb +1 -1
- data/lib/odba.rb +9 -9
- data/odba.gemspec +8 -3
- data/test/example.rb +47 -0
- data/test/helper.rb +6 -0
- data/test/suite.rb +7 -0
- data/test/test_array.rb +38 -33
- data/test/test_cache.rb +657 -622
- data/test/test_cache_entry.rb +51 -71
- data/test/test_connection_pool.rb +70 -25
- data/test/test_drbwrapper.rb +24 -19
- data/test/test_hash.rb +98 -89
- data/test/test_id_server.rb +30 -32
- data/test/test_index.rb +191 -158
- data/test/test_marshal.rb +15 -25
- data/test/test_persistable.rb +378 -369
- data/test/test_storage.rb +510 -471
- data/test/test_stub.rb +147 -135
- metadata +63 -20
- data/Guide.txt +0 -36
- data/Manifest.txt +0 -38
- data/install.rb +0 -1098
- data/lib/odba/18_19_loading_compatibility.rb +0 -77
data/lib/odba/cache.rb
CHANGED
|
@@ -1,36 +1,30 @@
|
|
|
1
1
|
#!/usr/bin/env ruby
|
|
2
|
-
|
|
2
|
+
|
|
3
3
|
# ODBA::Cache -- odba -- 27.12.2011 -- mhatakeyama@ywesee.com
|
|
4
4
|
# ODBA::Cache -- odba -- 29.04.2004 -- hwyss@ywesee.com rwaltert@ywesee.com mwalder@ywesee.com
|
|
5
5
|
|
|
6
|
-
require
|
|
7
|
-
require
|
|
8
|
-
|
|
9
|
-
require 'rubygems'
|
|
10
|
-
gem 'fastthread', '>=0.6.3'
|
|
11
|
-
rescue LoadError
|
|
12
|
-
end
|
|
13
|
-
require 'thread'
|
|
14
|
-
require 'drb'
|
|
6
|
+
require "singleton"
|
|
7
|
+
require "date"
|
|
8
|
+
require "drb"
|
|
15
9
|
|
|
16
10
|
module ODBA
|
|
17
|
-
|
|
18
|
-
|
|
11
|
+
class Cache
|
|
12
|
+
include Singleton
|
|
19
13
|
include DRb::DRbUndumped
|
|
20
|
-
|
|
21
|
-
|
|
14
|
+
CLEANER_PRIORITY = 0 # :nodoc:
|
|
15
|
+
CLEANING_INTERVAL = 5 # :nodoc:
|
|
22
16
|
attr_accessor :cleaner_step, :destroy_age, :retire_age, :debug, :file_lock
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
17
|
+
def initialize # :nodoc:
|
|
18
|
+
if self.class::CLEANING_INTERVAL > 0
|
|
19
|
+
start_cleaner
|
|
20
|
+
end
|
|
27
21
|
@retire_age = 300
|
|
28
22
|
@receiver = nil
|
|
29
|
-
|
|
23
|
+
@cache_mutex = Mutex.new
|
|
30
24
|
@deferred_indices = []
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
25
|
+
@fetched = {}
|
|
26
|
+
@prefetched = {}
|
|
27
|
+
@clean_prefetched = false
|
|
34
28
|
@cleaner_offset = 0
|
|
35
29
|
@prefetched_offset = 0
|
|
36
30
|
@cleaner_step = 500
|
|
@@ -38,64 +32,74 @@ module ODBA
|
|
|
38
32
|
@peers = []
|
|
39
33
|
@file_lock = false
|
|
40
34
|
@debug ||= false # Setting @debug to true makes two unit test fail!
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
# Returns all objects designated by _bulk_fetch_ids_ and registers
|
|
38
|
+
# _odba_caller_ for each of them. Objects which are not yet loaded are loaded
|
|
39
|
+
# from ODBA#storage.
|
|
40
|
+
def bulk_fetch(bulk_fetch_ids, odba_caller)
|
|
41
|
+
instances = []
|
|
42
|
+
loaded_ids = []
|
|
43
|
+
bulk_fetch_ids.each { |id|
|
|
44
|
+
if (entry = fetch_cache_entry(id))
|
|
45
|
+
entry.odba_add_reference(odba_caller)
|
|
46
|
+
instances.push(entry.odba_object)
|
|
47
|
+
loaded_ids.push(id)
|
|
48
|
+
end
|
|
49
|
+
}
|
|
50
|
+
bulk_fetch_ids -= loaded_ids
|
|
51
|
+
unless bulk_fetch_ids.empty?
|
|
52
|
+
rows = ODBA.storage.bulk_restore(bulk_fetch_ids)
|
|
53
|
+
instances += bulk_restore(rows, odba_caller)
|
|
54
|
+
end
|
|
55
|
+
instances
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
def bulk_restore(rows, odba_caller = nil) # :nodoc:
|
|
59
|
+
if ::Kernel::caller.size > 1000
|
|
60
|
+
# with size > 30 it crashed in parse_aips_download
|
|
61
|
+
# with size > 50 parse_aips_download was okay
|
|
62
|
+
# if nothing it crashed with a stack of > 8000
|
|
63
|
+
raise OdbaError
|
|
64
|
+
end
|
|
65
|
+
retrieved_objects = []
|
|
66
|
+
rows.each { |row|
|
|
67
|
+
obj_id = row.at(0)
|
|
68
|
+
dump = row.at(1)
|
|
67
69
|
odba_obj = fetch_or_restore(obj_id, dump, odba_caller)
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
70
|
+
retrieved_objects.push(odba_obj)
|
|
71
|
+
}
|
|
72
|
+
retrieved_objects
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
def clean # :nodoc:
|
|
73
76
|
now = Time.now
|
|
74
|
-
start = Time.now if
|
|
75
|
-
|
|
76
|
-
if
|
|
77
|
+
start = Time.now if @debug
|
|
78
|
+
@cleaned = 0
|
|
79
|
+
if @debug
|
|
77
80
|
puts "starting cleaning cycle"
|
|
78
81
|
$stdout.flush
|
|
79
82
|
end
|
|
80
83
|
retire_horizon = now - @retire_age
|
|
81
84
|
@cleaner_offset = _clean(retire_horizon, @fetched, @cleaner_offset)
|
|
82
|
-
|
|
85
|
+
if @clean_prefetched
|
|
83
86
|
@prefetched_offset = _clean(retire_horizon, @prefetched,
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
if
|
|
87
|
+
@prefetched_offset)
|
|
88
|
+
end
|
|
89
|
+
if @debug
|
|
87
90
|
puts "cleaned: #{@cleaned} objects in #{Time.now - start} seconds"
|
|
88
91
|
puts "remaining objects in @fetched: #{@fetched.size}"
|
|
89
92
|
puts "remaining objects in @prefetched: #{@prefetched.size}"
|
|
90
|
-
mbytes = File.read("/proc/#{$$}/stat").split(
|
|
93
|
+
mbytes = File.read("/proc/#{$$}/stat").split(" ").at(22).to_i / (2**20)
|
|
91
94
|
GC.start
|
|
92
95
|
puts "remaining objects in ObjectSpace: #{ObjectSpace.each_object {}}"
|
|
93
96
|
puts "memory-usage: #{mbytes}MB"
|
|
94
97
|
$stdout.flush
|
|
95
98
|
end
|
|
96
|
-
|
|
99
|
+
end
|
|
100
|
+
|
|
97
101
|
def _clean(retire_time, holder, offset) # :nodoc:
|
|
98
|
-
if
|
|
102
|
+
if offset > holder.size
|
|
99
103
|
offset = 0
|
|
100
104
|
end
|
|
101
105
|
counter = 0
|
|
@@ -103,147 +107,161 @@ module ODBA
|
|
|
103
107
|
@cache_mutex.synchronize {
|
|
104
108
|
holder.each_value { |value|
|
|
105
109
|
counter += 1
|
|
106
|
-
if
|
|
110
|
+
if counter > offset && value.odba_old?(retire_time)
|
|
107
111
|
value.odba_retire && @cleaned += 1
|
|
108
112
|
end
|
|
109
|
-
return cutoff if
|
|
113
|
+
return cutoff if counter > cutoff
|
|
110
114
|
}
|
|
111
115
|
}
|
|
112
116
|
cutoff
|
|
113
117
|
# every once in a while we'll get a 'hash modified during iteration'-Error.
|
|
114
118
|
# not to worry, we'll just try again later.
|
|
115
|
-
rescue
|
|
119
|
+
rescue
|
|
116
120
|
offset
|
|
117
121
|
end
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
122
|
+
|
|
123
|
+
# overrides the ODBA_PREFETCH constant and @odba_prefetch instance variable
|
|
124
|
+
# in Persistable. Use this if a secondary client is more memory-bound than
|
|
125
|
+
# performance-bound.
|
|
126
|
+
def clean_prefetched(flag = true)
|
|
127
|
+
if (@clean_prefetched = flag)
|
|
128
|
+
clean
|
|
129
|
+
end
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
def clear # :nodoc:
|
|
133
|
+
@fetched.clear
|
|
134
|
+
@prefetched.clear
|
|
135
|
+
end
|
|
136
|
+
|
|
130
137
|
# Creates or recreates automatically defined indices
|
|
131
138
|
def create_deferred_indices(drop_existing = false)
|
|
132
139
|
@deferred_indices.each { |definition|
|
|
133
140
|
name = definition.index_name
|
|
134
|
-
if
|
|
141
|
+
if drop_existing && indices.include?(name)
|
|
135
142
|
drop_index(name)
|
|
136
143
|
end
|
|
137
|
-
unless
|
|
144
|
+
unless indices.include?(name)
|
|
138
145
|
index = create_index(definition)
|
|
139
|
-
if
|
|
146
|
+
if index.target_klass.respond_to?(:odba_extent)
|
|
140
147
|
index.fill(index.target_klass.odba_extent)
|
|
141
148
|
end
|
|
142
149
|
end
|
|
143
150
|
}
|
|
144
151
|
end
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
152
|
+
|
|
153
|
+
# Creates a new index according to IndexDefinition
|
|
154
|
+
def create_index(index_definition, origin_module = Object)
|
|
155
|
+
transaction {
|
|
156
|
+
klass = if index_definition.fulltext
|
|
157
|
+
FulltextIndex
|
|
158
|
+
elsif index_definition.resolve_search_term.is_a?(Hash)
|
|
159
|
+
ConditionIndex
|
|
160
|
+
else
|
|
161
|
+
Index
|
|
162
|
+
end
|
|
163
|
+
index = klass.new(index_definition, origin_module)
|
|
164
|
+
indices.store(index_definition.index_name, index)
|
|
165
|
+
indices.odba_store_unsaved
|
|
166
|
+
index
|
|
167
|
+
}
|
|
168
|
+
end
|
|
169
|
+
|
|
170
|
+
# Permanently deletes _object_ from the database and deconnects all connected
|
|
171
|
+
# Persistables
|
|
172
|
+
def delete(odba_object)
|
|
173
|
+
odba_id = odba_object.odba_id
|
|
165
174
|
name = odba_object.odba_name
|
|
166
175
|
odba_object.odba_notify_observers(:delete, odba_id, odba_object.object_id)
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
176
|
+
rows = ODBA.storage.retrieve_connected_objects(odba_id)
|
|
177
|
+
rows.each { |row|
|
|
178
|
+
id = row.first
|
|
179
|
+
# Self-Referencing objects don't have to be resaved
|
|
180
|
+
begin
|
|
181
|
+
if (connected_object = fetch(id, nil))
|
|
182
|
+
connected_object.odba_cut_connection(odba_object)
|
|
183
|
+
connected_object.odba_isolated_store
|
|
184
|
+
end
|
|
185
|
+
rescue OdbaError
|
|
186
|
+
warn "OdbaError ### deleting #{odba_object.class}:#{odba_id}"
|
|
187
|
+
warn " ### while looking for connected object #{id}"
|
|
188
|
+
end
|
|
189
|
+
}
|
|
181
190
|
delete_cache_entry(odba_id)
|
|
182
191
|
delete_cache_entry(name)
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
192
|
+
ODBA.storage.delete_persistable(odba_id)
|
|
193
|
+
delete_index_element(odba_object)
|
|
194
|
+
odba_object
|
|
195
|
+
end
|
|
196
|
+
|
|
187
197
|
def delete_cache_entry(key)
|
|
188
198
|
@cache_mutex.synchronize {
|
|
189
199
|
@fetched.delete(key)
|
|
190
200
|
@prefetched.delete(key)
|
|
191
201
|
}
|
|
192
202
|
end
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
203
|
+
|
|
204
|
+
def delete_index_element(odba_object) # :nodoc:
|
|
205
|
+
odba_object.class
|
|
206
|
+
indices.each_value { |index|
|
|
196
207
|
index.delete(odba_object)
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
208
|
+
}
|
|
209
|
+
end
|
|
210
|
+
|
|
211
|
+
# Permanently deletes the index named _index_name_
|
|
212
|
+
def drop_index(index_name)
|
|
213
|
+
transaction {
|
|
214
|
+
ODBA.storage.drop_index(index_name)
|
|
215
|
+
delete(indices[index_name])
|
|
216
|
+
}
|
|
217
|
+
end
|
|
218
|
+
|
|
206
219
|
def drop_indices # :nodoc:
|
|
207
|
-
keys =
|
|
208
|
-
keys.each{ |key|
|
|
220
|
+
keys = indices.keys
|
|
221
|
+
keys.each { |key|
|
|
209
222
|
drop_index(key)
|
|
210
223
|
}
|
|
211
224
|
end
|
|
225
|
+
|
|
212
226
|
# Queue an index for creation by #setup
|
|
213
227
|
def ensure_index_deferred(index_definition)
|
|
214
228
|
@deferred_indices.push(index_definition)
|
|
215
|
-
|
|
229
|
+
end
|
|
230
|
+
|
|
216
231
|
# Get all instances of a class (- a limited extent)
|
|
217
|
-
def extent(klass, odba_caller=nil)
|
|
218
|
-
|
|
232
|
+
def extent(klass, odba_caller = nil)
|
|
233
|
+
bulk_fetch(ODBA.storage.extent_ids(klass), odba_caller)
|
|
219
234
|
end
|
|
235
|
+
|
|
220
236
|
# Get number of instances of a class
|
|
221
237
|
def count(klass)
|
|
222
238
|
ODBA.storage.extent_count(klass)
|
|
223
239
|
end
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
240
|
+
|
|
241
|
+
# Fetch a Persistable identified by _odba_id_. Registers _odba_caller_ with
|
|
242
|
+
# the CacheEntry. Loads the Persistable if it is not already loaded.
|
|
243
|
+
def fetch(odba_id, odba_caller = nil)
|
|
244
|
+
fetch_or_do(odba_id, odba_caller) {
|
|
228
245
|
load_object(odba_id, odba_caller)
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
246
|
+
}
|
|
247
|
+
end
|
|
248
|
+
|
|
249
|
+
def fetch_cache_entry(odba_id_or_name) # :nodoc:
|
|
250
|
+
@prefetched[odba_id_or_name] || @fetched[odba_id_or_name]
|
|
251
|
+
end
|
|
252
|
+
@@receiver_name = :@receiver
|
|
253
|
+
def fetch_collection(odba_obj) # :nodoc:
|
|
254
|
+
collection = []
|
|
255
|
+
bulk_fetch_ids = []
|
|
256
|
+
rows = ODBA.storage.restore_collection(odba_obj.odba_id)
|
|
239
257
|
return collection if rows.empty?
|
|
240
258
|
idx = 0
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
259
|
+
rows.each { |row|
|
|
260
|
+
key = row[0].is_a?(Integer) ? row[0] : ODBA.marshaller.load(row[0])
|
|
261
|
+
value = row[1].is_a?(Integer) ? row[1] : ODBA.marshaller.load(row[1])
|
|
262
|
+
idx += 1
|
|
245
263
|
item = nil
|
|
246
|
-
if
|
|
264
|
+
if [key, value].any? { |item| item.instance_variable_get(@@receiver_name) }
|
|
247
265
|
odba_id = odba_obj.odba_id
|
|
248
266
|
warn "stub for #{item.class}:#{item.odba_id} was saved with receiver in collection of #{odba_obj.class}:#{odba_id}"
|
|
249
267
|
warn "repair: remove [#{odba_id}, #{row[0]}, #{row[1].length}]"
|
|
@@ -255,71 +273,75 @@ module ODBA
|
|
|
255
273
|
warn "repair: insert [#{odba_id}, #{key_dump}, #{value_dump.length}]"
|
|
256
274
|
ODBA.storage.collection_store(odba_id, key_dump, value_dump)
|
|
257
275
|
end
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
276
|
+
bulk_fetch_ids.push(key.odba_id)
|
|
277
|
+
bulk_fetch_ids.push(value.odba_id)
|
|
278
|
+
collection.push([key, value])
|
|
279
|
+
}
|
|
280
|
+
bulk_fetch_ids.compact!
|
|
281
|
+
bulk_fetch_ids.uniq!
|
|
282
|
+
bulk_fetch(bulk_fetch_ids, odba_obj)
|
|
283
|
+
collection.each { |pair|
|
|
284
|
+
pair.collect! { |item|
|
|
285
|
+
if item.is_a?(ODBA::Stub)
|
|
268
286
|
## don't fetch: that may result in a conflict when storing.
|
|
269
|
-
#fetch(item.odba_id, odba_obj)
|
|
287
|
+
# fetch(item.odba_id, odba_obj)
|
|
270
288
|
item.odba_container = odba_obj
|
|
271
289
|
item
|
|
272
|
-
elsif(ce = fetch_cache_entry(item.odba_id))
|
|
290
|
+
elsif (ce = fetch_cache_entry(item.odba_id))
|
|
273
291
|
warn "collection loaded unstubbed object: #{item.odba_id}"
|
|
274
292
|
ce.odba_add_reference(odba_obj)
|
|
275
293
|
ce.odba_object
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
294
|
+
else
|
|
295
|
+
item
|
|
296
|
+
end
|
|
297
|
+
}
|
|
298
|
+
}
|
|
299
|
+
collection
|
|
300
|
+
end
|
|
301
|
+
|
|
302
|
+
def fetch_collection_element(odba_id, key) # :nodoc:
|
|
303
|
+
key_dump = ODBA.marshaller.dump(key.odba_isolated_stub)
|
|
304
|
+
## for backward-compatibility and robustness we only attempt
|
|
305
|
+
## to load if there was a dump stored in the collection table
|
|
306
|
+
if (dump = ODBA.storage.collection_fetch(odba_id, key_dump))
|
|
307
|
+
item = ODBA.marshaller.load(dump)
|
|
308
|
+
if item.is_a?(ODBA::Stub)
|
|
290
309
|
fetch(item.odba_id)
|
|
291
|
-
elsif
|
|
310
|
+
elsif item.is_a?(ODBA::Persistable)
|
|
292
311
|
warn "collection_element was unstubbed object: #{item.odba_id}"
|
|
293
312
|
fetch_or_restore(item.odba_id, dump, nil)
|
|
294
313
|
else
|
|
295
314
|
item
|
|
296
315
|
end
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
316
|
+
end
|
|
317
|
+
end
|
|
318
|
+
|
|
319
|
+
def fetch_named(name, odba_caller, &block) # :nodoc:
|
|
320
|
+
fetch_or_do(name, odba_caller) {
|
|
321
|
+
dump = ODBA.storage.restore_named(name)
|
|
322
|
+
if dump.nil?
|
|
323
|
+
odba_obj = block.call
|
|
324
|
+
odba_obj.odba_name = name
|
|
325
|
+
odba_obj.odba_store(name)
|
|
326
|
+
odba_obj
|
|
327
|
+
else
|
|
328
|
+
fetch_or_restore(name, dump, odba_caller)
|
|
329
|
+
end
|
|
330
|
+
}
|
|
331
|
+
end
|
|
332
|
+
|
|
333
|
+
def fetch_or_do(obj_id, odba_caller, &block) # :nodoc:
|
|
334
|
+
if (cache_entry = fetch_cache_entry(obj_id)) && cache_entry._odba_object
|
|
335
|
+
cache_entry.odba_add_reference(odba_caller)
|
|
336
|
+
cache_entry.odba_object
|
|
337
|
+
else
|
|
338
|
+
block.call
|
|
339
|
+
end
|
|
340
|
+
end
|
|
341
|
+
|
|
320
342
|
def fetch_or_restore(odba_id, dump, odba_caller) # :nodoc:
|
|
321
343
|
fetch_or_do(odba_id, odba_caller) {
|
|
322
|
-
odba_obj,
|
|
344
|
+
odba_obj, _ = restore(dump)
|
|
323
345
|
@cache_mutex.synchronize {
|
|
324
346
|
fetch_or_do(odba_id, odba_caller) {
|
|
325
347
|
cache_entry = CacheEntry.new(odba_obj)
|
|
@@ -335,53 +357,61 @@ module ODBA
|
|
|
335
357
|
}
|
|
336
358
|
}
|
|
337
359
|
end
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
360
|
+
|
|
361
|
+
def fill_index(index_name, targets)
|
|
362
|
+
indices[index_name].fill(targets)
|
|
363
|
+
end
|
|
364
|
+
|
|
365
|
+
# Checks wether the object identified by _odba_id_ has been loaded.
|
|
366
|
+
def include?(odba_id)
|
|
367
|
+
@fetched.include?(odba_id) || @prefetched.include?(odba_id)
|
|
368
|
+
end
|
|
369
|
+
|
|
370
|
+
def index_keys(index_name, length = nil)
|
|
371
|
+
index = indices.fetch(index_name)
|
|
372
|
+
index.keys(length)
|
|
373
|
+
end
|
|
374
|
+
|
|
375
|
+
def index_matches(index_name, substring, limit = nil, offset = 0)
|
|
376
|
+
index = indices.fetch(index_name)
|
|
377
|
+
index.matches substring, limit, offset
|
|
378
|
+
end
|
|
379
|
+
|
|
380
|
+
# Returns a Hash-table containing all stored indices.
|
|
381
|
+
def indices
|
|
382
|
+
@indices ||= fetch_named("__cache_server_indices__", self) {
|
|
383
|
+
{}
|
|
384
|
+
}
|
|
385
|
+
end
|
|
386
|
+
|
|
359
387
|
def invalidate(odba_id)
|
|
360
388
|
## when finalizers are run, no other threads will be scheduled,
|
|
361
389
|
# therefore we don't need to @cache_mutex.synchronize
|
|
362
390
|
@fetched.delete odba_id
|
|
363
391
|
@prefetched.delete odba_id
|
|
364
392
|
end
|
|
393
|
+
|
|
365
394
|
def invalidate!(*odba_ids)
|
|
366
395
|
odba_ids.each do |odba_id|
|
|
367
396
|
if entry = fetch_cache_entry(odba_id)
|
|
368
|
-
entry.odba_retire :
|
|
397
|
+
entry.odba_retire force: true
|
|
369
398
|
end
|
|
370
399
|
invalidate odba_id
|
|
371
400
|
end
|
|
372
401
|
end
|
|
373
402
|
# File lock exclusive control between processes, not threads, to create safely a new odba_id
|
|
374
403
|
# Sometimes several update jobs (processes) to the same database at the same time
|
|
375
|
-
LOCK_FILE =
|
|
376
|
-
COUNT_FILE =
|
|
404
|
+
LOCK_FILE = "/tmp/lockfile"
|
|
405
|
+
COUNT_FILE = "/tmp/count"
|
|
377
406
|
def lock(dbname)
|
|
378
407
|
lock_file = LOCK_FILE + "." + dbname
|
|
379
|
-
open(lock_file,
|
|
408
|
+
open(lock_file, "a") do |st|
|
|
380
409
|
st.flock(File::LOCK_EX)
|
|
381
410
|
yield
|
|
382
411
|
st.flock(File::LOCK_UN)
|
|
383
412
|
end
|
|
384
413
|
end
|
|
414
|
+
|
|
385
415
|
def new_id(dbname, odba_storage)
|
|
386
416
|
count_file = COUNT_FILE + "." + dbname
|
|
387
417
|
count = nil
|
|
@@ -400,66 +430,75 @@ module ODBA
|
|
|
400
430
|
end
|
|
401
431
|
count
|
|
402
432
|
end
|
|
433
|
+
|
|
403
434
|
# Returns the next valid odba_id
|
|
404
435
|
def next_id
|
|
405
436
|
if @file_lock
|
|
406
|
-
dbname = ODBA.storage.instance_variable_get(
|
|
437
|
+
dbname = ODBA.storage.instance_variable_get(:@dbi).dbi_args.first.split(":").last
|
|
407
438
|
id = new_id(dbname, ODBA.storage)
|
|
408
439
|
else
|
|
409
440
|
id = ODBA.storage.next_id
|
|
410
441
|
end
|
|
411
442
|
@peers.each do |peer|
|
|
412
|
-
peer.reserve_next_id id
|
|
443
|
+
peer.reserve_next_id id
|
|
444
|
+
rescue
|
|
445
|
+
DRb::DRbError
|
|
413
446
|
end
|
|
414
447
|
id
|
|
415
448
|
rescue OdbaDuplicateIdError
|
|
416
449
|
retry
|
|
417
450
|
end
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
451
|
+
|
|
452
|
+
# Use this to load all prefetchable Persistables from the db at once
|
|
453
|
+
def prefetch
|
|
454
|
+
bulk_restore(ODBA.storage.restore_prefetchable)
|
|
455
|
+
end
|
|
456
|
+
|
|
422
457
|
# prints loading statistics to $stdout
|
|
423
458
|
def print_stats
|
|
424
459
|
fmh = " %-20s | %10s | %5s | %6s | %6s | %6s | %-20s\n"
|
|
425
460
|
fmt = " %-20s | %10.3f | %5i | %6.3f | %6.3f | %6.3f | %s\n"
|
|
426
461
|
head = sprintf(fmh,
|
|
427
|
-
|
|
462
|
+
"class", "total", "count", "min", "max", "avg", "callers")
|
|
428
463
|
line = "-" * head.length
|
|
429
464
|
puts line
|
|
430
465
|
print head
|
|
431
466
|
puts line
|
|
432
467
|
@loading_stats.sort_by { |key, val|
|
|
433
468
|
val[:total_time]
|
|
434
|
-
}.
|
|
469
|
+
}.reverse_each { |key, val|
|
|
435
470
|
key = key.to_s
|
|
436
|
-
if
|
|
437
|
-
key = key[-20,20]
|
|
471
|
+
if key.length > 20
|
|
472
|
+
key = key[-20, 20]
|
|
438
473
|
end
|
|
439
474
|
avg = val[:total_time] / val[:count]
|
|
440
475
|
printf(fmt, key, val[:total_time], val[:count],
|
|
441
|
-
|
|
442
|
-
|
|
476
|
+
val[:times].min, val[:times].max, avg,
|
|
477
|
+
val[:callers].join(","))
|
|
443
478
|
}
|
|
444
479
|
puts line
|
|
445
480
|
$stdout.flush
|
|
446
481
|
end
|
|
482
|
+
|
|
447
483
|
# Register a peer that has access to the same DB backend
|
|
448
484
|
def register_peer peer
|
|
449
485
|
@peers.push(peer).uniq!
|
|
450
486
|
end
|
|
487
|
+
|
|
451
488
|
# Reserve an id with all registered peers
|
|
452
489
|
def reserve_next_id id
|
|
453
490
|
ODBA.storage.reserve_next_id id
|
|
454
491
|
end
|
|
492
|
+
|
|
455
493
|
# Clears the loading statistics
|
|
456
494
|
def reset_stats
|
|
457
495
|
@loading_stats.clear
|
|
458
496
|
end
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
497
|
+
|
|
498
|
+
# Find objects in an index
|
|
499
|
+
def retrieve_from_index(index_name, search_term, meta = nil)
|
|
500
|
+
index = indices.fetch(index_name)
|
|
501
|
+
ids = index.fetch_ids(search_term, meta)
|
|
463
502
|
if meta.respond_to?(:error_limit) && (limit = meta.error_limit) \
|
|
464
503
|
&& (size = ids.size) > limit.to_i
|
|
465
504
|
error = OdbaResultLimitError.new
|
|
@@ -470,47 +509,51 @@ module ODBA
|
|
|
470
509
|
error.meta = meta
|
|
471
510
|
raise error
|
|
472
511
|
end
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
512
|
+
bulk_fetch(ids, nil)
|
|
513
|
+
end
|
|
514
|
+
|
|
515
|
+
# Create necessary DB-Structure / other storage-setup
|
|
476
516
|
def setup
|
|
477
517
|
ODBA.storage.setup
|
|
478
|
-
|
|
518
|
+
indices.each_key { |index_name|
|
|
479
519
|
ODBA.storage.ensure_target_id_index(index_name)
|
|
480
520
|
}
|
|
481
521
|
create_deferred_indices
|
|
482
522
|
nil
|
|
483
523
|
end
|
|
524
|
+
|
|
484
525
|
# Returns the total number of cached objects
|
|
485
526
|
def size
|
|
486
527
|
@prefetched.size + @fetched.size
|
|
487
528
|
end
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
529
|
+
|
|
530
|
+
def start_cleaner # :nodoc:
|
|
531
|
+
@cleaner = Thread.new {
|
|
532
|
+
Thread.current.priority = self.class::CLEANER_PRIORITY
|
|
533
|
+
loop {
|
|
534
|
+
sleep(self.class::CLEANING_INTERVAL)
|
|
535
|
+
begin
|
|
536
|
+
clean
|
|
537
|
+
rescue => e
|
|
538
|
+
puts e
|
|
539
|
+
puts e.backtrace
|
|
540
|
+
end
|
|
541
|
+
}
|
|
542
|
+
}
|
|
543
|
+
end
|
|
544
|
+
|
|
545
|
+
# Store a Persistable _object_ in the database
|
|
546
|
+
def store(object)
|
|
547
|
+
odba_id = object.odba_id
|
|
548
|
+
name = object.odba_name
|
|
506
549
|
object.odba_notify_observers(:store, odba_id, object.object_id)
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
550
|
+
if (ids = Thread.current[:txids])
|
|
551
|
+
ids.unshift([odba_id, name])
|
|
552
|
+
end
|
|
510
553
|
## get target_ids before anything else
|
|
511
554
|
target_ids = object.odba_target_ids
|
|
512
|
-
|
|
513
|
-
|
|
555
|
+
store_collection_elements(object)
|
|
556
|
+
prefetchable = object.odba_prefetch?
|
|
514
557
|
dump = object.odba_isolated_dump
|
|
515
558
|
ODBA.storage.store(odba_id, dump, name, prefetchable, object.class)
|
|
516
559
|
store_object_connections(odba_id, target_ids)
|
|
@@ -518,11 +561,14 @@ module ODBA
|
|
|
518
561
|
object = store_cache_entry(odba_id, object, name)
|
|
519
562
|
update_indices(object)
|
|
520
563
|
@peers.each do |peer|
|
|
521
|
-
peer.invalidate! odba_id
|
|
564
|
+
peer.invalidate! odba_id
|
|
565
|
+
rescue
|
|
566
|
+
DRb::DRbError
|
|
522
567
|
end
|
|
523
568
|
object
|
|
524
|
-
|
|
525
|
-
|
|
569
|
+
end
|
|
570
|
+
|
|
571
|
+
def store_cache_entry(odba_id, object, name = nil) # :nodoc:
|
|
526
572
|
@cache_mutex.synchronize {
|
|
527
573
|
if cache_entry = fetch_cache_entry(odba_id)
|
|
528
574
|
cache_entry.update object
|
|
@@ -530,18 +576,19 @@ module ODBA
|
|
|
530
576
|
hash = object.odba_prefetch? ? @prefetched : @fetched
|
|
531
577
|
cache_entry = CacheEntry.new(object)
|
|
532
578
|
hash.store(odba_id, cache_entry)
|
|
533
|
-
unless
|
|
579
|
+
unless name.nil?
|
|
534
580
|
hash.store(name, cache_entry)
|
|
535
581
|
end
|
|
536
582
|
end
|
|
537
583
|
cache_entry.odba_object
|
|
538
584
|
}
|
|
539
585
|
end
|
|
586
|
+
|
|
540
587
|
def store_collection_elements(odba_obj) # :nodoc:
|
|
541
588
|
odba_id = odba_obj.odba_id
|
|
542
589
|
collection = odba_obj.odba_collection.collect { |key, value|
|
|
543
|
-
[
|
|
544
|
-
ODBA.marshaller.dump(value.odba_isolated_stub)
|
|
590
|
+
[ODBA.marshaller.dump(key.odba_isolated_stub),
|
|
591
|
+
ODBA.marshaller.dump(value.odba_isolated_stub)]
|
|
545
592
|
}
|
|
546
593
|
old_collection = ODBA.storage.restore_collection(odba_id).collect { |row|
|
|
547
594
|
[row[0], row [1]]
|
|
@@ -553,95 +600,105 @@ module ODBA
|
|
|
553
600
|
ODBA.storage.collection_store(odba_id, key_dump, value_dump)
|
|
554
601
|
}.size
|
|
555
602
|
end
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
603
|
+
|
|
604
|
+
def store_object_connections(odba_id, target_ids) # :nodoc:
|
|
605
|
+
ODBA.storage.ensure_object_connections(odba_id, target_ids)
|
|
606
|
+
end
|
|
607
|
+
|
|
608
|
+
# Executes the block in a transaction. If the transaction fails, all
|
|
609
|
+
# affected Persistable objects are reloaded from the db (which by then has
|
|
610
|
+
# also performed a rollback). Rollback is quite inefficient at this time.
|
|
611
|
+
def transaction(&block)
|
|
612
|
+
Thread.current[:txids] = []
|
|
613
|
+
ODBA.storage.transaction(&block)
|
|
614
|
+
rescue Exception => excp
|
|
615
|
+
transaction_rollback
|
|
616
|
+
raise excp
|
|
617
|
+
ensure
|
|
618
|
+
Thread.current[:txids] = nil
|
|
619
|
+
end
|
|
620
|
+
|
|
621
|
+
def transaction_rollback # :nodoc:
|
|
622
|
+
if (ids = Thread.current[:txids])
|
|
623
|
+
ids.each { |id, name|
|
|
624
|
+
if (entry = fetch_cache_entry(id))
|
|
625
|
+
if (dump = ODBA.storage.restore(id))
|
|
626
|
+
odba_obj, _ = restore(dump)
|
|
627
|
+
entry.odba_replace!(odba_obj)
|
|
628
|
+
else
|
|
629
|
+
entry.odba_cut_connections!
|
|
580
630
|
delete_cache_entry(id)
|
|
581
631
|
delete_cache_entry(name)
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
632
|
+
end
|
|
633
|
+
end
|
|
634
|
+
}
|
|
635
|
+
end
|
|
636
|
+
end
|
|
637
|
+
|
|
587
638
|
# Unregister a peer
|
|
588
639
|
def unregister_peer peer
|
|
589
640
|
@peers.delete peer
|
|
590
641
|
end
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
642
|
+
|
|
643
|
+
def update_indices(odba_object) # :nodoc:
|
|
644
|
+
if odba_object.odba_indexable?
|
|
645
|
+
indices.each { |index_name, index|
|
|
646
|
+
index.update(odba_object)
|
|
647
|
+
}
|
|
648
|
+
end
|
|
649
|
+
end
|
|
650
|
+
|
|
651
|
+
def update_references(target_ids, object) # :nodoc:
|
|
652
|
+
target_ids.each { |odba_id|
|
|
653
|
+
if (entry = fetch_cache_entry(odba_id))
|
|
654
|
+
entry.odba_add_reference(object)
|
|
655
|
+
end
|
|
656
|
+
}
|
|
657
|
+
end
|
|
658
|
+
|
|
659
|
+
private
|
|
660
|
+
|
|
606
661
|
def load_object(odba_id, odba_caller)
|
|
607
|
-
start = Time.now if
|
|
662
|
+
start = Time.now if @debug
|
|
608
663
|
dump = ODBA.storage.restore(odba_id)
|
|
609
664
|
odba_obj = restore_object(odba_id, dump, odba_caller)
|
|
610
|
-
return odba_obj unless
|
|
665
|
+
return odba_obj unless @debug
|
|
611
666
|
stats = (@loading_stats[odba_obj.class] ||= {
|
|
612
|
-
:
|
|
667
|
+
count: 0, times: [], total_time: 0, callers: []
|
|
613
668
|
})
|
|
614
669
|
stats[:count] += 1
|
|
615
670
|
time = Time.now - start
|
|
616
671
|
stats[:times].push(time)
|
|
617
672
|
stats[:total_time] += time
|
|
618
673
|
stats[:callers].push(odba_caller.class).uniq!
|
|
619
|
-
if
|
|
674
|
+
if time > 2
|
|
620
675
|
names = []
|
|
621
676
|
odba_caller.instance_variables.each { |name|
|
|
622
|
-
if
|
|
677
|
+
if odba_caller.instance_variable_get(name).odba_id == odba_id
|
|
623
678
|
names.push(name)
|
|
624
679
|
end
|
|
625
680
|
}
|
|
626
681
|
printf("long load-time (%4.2fs) for odba_id %i: %s#%s\n",
|
|
627
|
-
|
|
682
|
+
time, odba_id, odba_caller, names.join(","))
|
|
628
683
|
end
|
|
629
684
|
odba_obj
|
|
630
685
|
end
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
686
|
+
|
|
687
|
+
def restore(dump)
|
|
688
|
+
odba_obj = ODBA.marshaller.load(dump)
|
|
689
|
+
unless odba_obj.is_a?(Persistable)
|
|
634
690
|
odba_obj.extend(Persistable)
|
|
635
691
|
end
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
692
|
+
collection = fetch_collection(odba_obj)
|
|
693
|
+
odba_obj.odba_restore(collection)
|
|
694
|
+
[odba_obj, collection]
|
|
695
|
+
end
|
|
696
|
+
|
|
697
|
+
def restore_object(odba_id, dump, odba_caller)
|
|
698
|
+
if dump.nil?
|
|
699
|
+
raise OdbaError, "Unknown odba_id #{odba_id}"
|
|
700
|
+
end
|
|
701
|
+
fetch_or_restore(odba_id, dump, odba_caller)
|
|
702
|
+
end
|
|
703
|
+
end
|
|
647
704
|
end
|