odba 1.1.9 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/devenv.yml +30 -0
- data/.github/workflows/ruby.yml +8 -7
- data/.gitignore +11 -0
- data/{History.txt → History.md} +40 -20
- data/README.md +77 -1
- data/Rakefile +3 -15
- data/devenv.lock +171 -0
- data/devenv.nix +53 -0
- data/devenv.yaml +8 -0
- data/lib/odba/cache.rb +395 -344
- data/lib/odba/cache_entry.rb +42 -31
- data/lib/odba/connection_pool.rb +99 -72
- data/lib/odba/drbwrapper.rb +26 -18
- data/lib/odba/id_server.rb +20 -20
- data/lib/odba/index.rb +249 -215
- data/lib/odba/index_definition.rb +17 -17
- data/lib/odba/marshal.rb +15 -14
- data/lib/odba/odba.rb +40 -32
- data/lib/odba/odba_error.rb +4 -2
- data/lib/odba/persistable.rb +460 -414
- data/lib/odba/storage.rb +328 -277
- data/lib/odba/stub.rb +166 -153
- data/lib/odba/version.rb +1 -1
- data/lib/odba.rb +9 -9
- data/odba.gemspec +8 -3
- data/test/example.rb +47 -0
- data/test/helper.rb +6 -0
- data/test/suite.rb +7 -0
- data/test/test_array.rb +38 -33
- data/test/test_cache.rb +657 -622
- data/test/test_cache_entry.rb +51 -71
- data/test/test_connection_pool.rb +70 -25
- data/test/test_drbwrapper.rb +24 -19
- data/test/test_hash.rb +98 -89
- data/test/test_id_server.rb +30 -32
- data/test/test_index.rb +191 -158
- data/test/test_marshal.rb +15 -25
- data/test/test_persistable.rb +378 -369
- data/test/test_storage.rb +510 -471
- data/test/test_stub.rb +147 -135
- metadata +63 -20
- data/Guide.txt +0 -36
- data/Manifest.txt +0 -38
- data/install.rb +0 -1098
- data/lib/odba/18_19_loading_compatibility.rb +0 -77
data/lib/odba/cache.rb
CHANGED
|
@@ -1,36 +1,30 @@
|
|
|
1
1
|
#!/usr/bin/env ruby
|
|
2
|
-
|
|
2
|
+
|
|
3
3
|
# ODBA::Cache -- odba -- 27.12.2011 -- mhatakeyama@ywesee.com
|
|
4
4
|
# ODBA::Cache -- odba -- 29.04.2004 -- hwyss@ywesee.com rwaltert@ywesee.com mwalder@ywesee.com
|
|
5
5
|
|
|
6
|
-
require
|
|
7
|
-
require
|
|
8
|
-
|
|
9
|
-
require 'rubygems'
|
|
10
|
-
gem 'fastthread', '>=0.6.3'
|
|
11
|
-
rescue LoadError
|
|
12
|
-
end
|
|
13
|
-
require 'thread'
|
|
14
|
-
require 'drb'
|
|
6
|
+
require "singleton"
|
|
7
|
+
require "date"
|
|
8
|
+
require "drb"
|
|
15
9
|
|
|
16
10
|
module ODBA
|
|
17
|
-
|
|
18
|
-
|
|
11
|
+
class Cache
|
|
12
|
+
include Singleton
|
|
19
13
|
include DRb::DRbUndumped
|
|
20
|
-
|
|
21
|
-
|
|
14
|
+
CLEANER_PRIORITY = 0 # :nodoc:
|
|
15
|
+
CLEANING_INTERVAL = 5 # :nodoc:
|
|
22
16
|
attr_accessor :cleaner_step, :destroy_age, :retire_age, :debug, :file_lock
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
17
|
+
def initialize # :nodoc:
|
|
18
|
+
if self.class::CLEANING_INTERVAL > 0
|
|
19
|
+
start_cleaner
|
|
20
|
+
end
|
|
27
21
|
@retire_age = 300
|
|
28
22
|
@receiver = nil
|
|
29
|
-
|
|
23
|
+
@cache_mutex = Mutex.new
|
|
30
24
|
@deferred_indices = []
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
25
|
+
@fetched = {}
|
|
26
|
+
@prefetched = {}
|
|
27
|
+
@clean_prefetched = false
|
|
34
28
|
@cleaner_offset = 0
|
|
35
29
|
@prefetched_offset = 0
|
|
36
30
|
@cleaner_step = 500
|
|
@@ -38,70 +32,74 @@ module ODBA
|
|
|
38
32
|
@peers = []
|
|
39
33
|
@file_lock = false
|
|
40
34
|
@debug ||= false # Setting @debug to true makes two unit test fail!
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
# Returns all objects designated by _bulk_fetch_ids_ and registers
|
|
38
|
+
# _odba_caller_ for each of them. Objects which are not yet loaded are loaded
|
|
39
|
+
# from ODBA#storage.
|
|
40
|
+
def bulk_fetch(bulk_fetch_ids, odba_caller)
|
|
41
|
+
instances = []
|
|
42
|
+
loaded_ids = []
|
|
43
|
+
bulk_fetch_ids.each { |id|
|
|
44
|
+
if (entry = fetch_cache_entry(id))
|
|
45
|
+
entry.odba_add_reference(odba_caller)
|
|
46
|
+
instances.push(entry.odba_object)
|
|
47
|
+
loaded_ids.push(id)
|
|
48
|
+
end
|
|
49
|
+
}
|
|
50
|
+
bulk_fetch_ids -= loaded_ids
|
|
51
|
+
unless bulk_fetch_ids.empty?
|
|
52
|
+
rows = ODBA.storage.bulk_restore(bulk_fetch_ids)
|
|
53
|
+
instances += bulk_restore(rows, odba_caller)
|
|
54
|
+
end
|
|
55
|
+
instances
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
def bulk_restore(rows, odba_caller = nil) # :nodoc:
|
|
63
59
|
if ::Kernel::caller.size > 1000
|
|
64
60
|
# with size > 30 it crashed in parse_aips_download
|
|
65
61
|
# with size > 50 parse_aips_download was okay
|
|
66
62
|
# if nothing it crashed with a stack of > 8000
|
|
67
63
|
raise OdbaError
|
|
68
64
|
end
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
65
|
+
retrieved_objects = []
|
|
66
|
+
rows.each { |row|
|
|
67
|
+
obj_id = row.at(0)
|
|
68
|
+
dump = row.at(1)
|
|
73
69
|
odba_obj = fetch_or_restore(obj_id, dump, odba_caller)
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
70
|
+
retrieved_objects.push(odba_obj)
|
|
71
|
+
}
|
|
72
|
+
retrieved_objects
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
def clean # :nodoc:
|
|
79
76
|
now = Time.now
|
|
80
|
-
start = Time.now if
|
|
81
|
-
|
|
82
|
-
if
|
|
77
|
+
start = Time.now if @debug
|
|
78
|
+
@cleaned = 0
|
|
79
|
+
if @debug
|
|
83
80
|
puts "starting cleaning cycle"
|
|
84
81
|
$stdout.flush
|
|
85
82
|
end
|
|
86
83
|
retire_horizon = now - @retire_age
|
|
87
84
|
@cleaner_offset = _clean(retire_horizon, @fetched, @cleaner_offset)
|
|
88
|
-
|
|
85
|
+
if @clean_prefetched
|
|
89
86
|
@prefetched_offset = _clean(retire_horizon, @prefetched,
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
if
|
|
87
|
+
@prefetched_offset)
|
|
88
|
+
end
|
|
89
|
+
if @debug
|
|
93
90
|
puts "cleaned: #{@cleaned} objects in #{Time.now - start} seconds"
|
|
94
91
|
puts "remaining objects in @fetched: #{@fetched.size}"
|
|
95
92
|
puts "remaining objects in @prefetched: #{@prefetched.size}"
|
|
96
|
-
mbytes = File.read("/proc/#{$$}/stat").split(
|
|
93
|
+
mbytes = File.read("/proc/#{$$}/stat").split(" ").at(22).to_i / (2**20)
|
|
97
94
|
GC.start
|
|
98
95
|
puts "remaining objects in ObjectSpace: #{ObjectSpace.each_object {}}"
|
|
99
96
|
puts "memory-usage: #{mbytes}MB"
|
|
100
97
|
$stdout.flush
|
|
101
98
|
end
|
|
102
|
-
|
|
99
|
+
end
|
|
100
|
+
|
|
103
101
|
def _clean(retire_time, holder, offset) # :nodoc:
|
|
104
|
-
if
|
|
102
|
+
if offset > holder.size
|
|
105
103
|
offset = 0
|
|
106
104
|
end
|
|
107
105
|
counter = 0
|
|
@@ -109,147 +107,161 @@ module ODBA
|
|
|
109
107
|
@cache_mutex.synchronize {
|
|
110
108
|
holder.each_value { |value|
|
|
111
109
|
counter += 1
|
|
112
|
-
if
|
|
110
|
+
if counter > offset && value.odba_old?(retire_time)
|
|
113
111
|
value.odba_retire && @cleaned += 1
|
|
114
112
|
end
|
|
115
|
-
return cutoff if
|
|
113
|
+
return cutoff if counter > cutoff
|
|
116
114
|
}
|
|
117
115
|
}
|
|
118
116
|
cutoff
|
|
119
117
|
# every once in a while we'll get a 'hash modified during iteration'-Error.
|
|
120
118
|
# not to worry, we'll just try again later.
|
|
121
|
-
rescue
|
|
119
|
+
rescue
|
|
122
120
|
offset
|
|
123
121
|
end
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
122
|
+
|
|
123
|
+
# overrides the ODBA_PREFETCH constant and @odba_prefetch instance variable
|
|
124
|
+
# in Persistable. Use this if a secondary client is more memory-bound than
|
|
125
|
+
# performance-bound.
|
|
126
|
+
def clean_prefetched(flag = true)
|
|
127
|
+
if (@clean_prefetched = flag)
|
|
128
|
+
clean
|
|
129
|
+
end
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
def clear # :nodoc:
|
|
133
|
+
@fetched.clear
|
|
134
|
+
@prefetched.clear
|
|
135
|
+
end
|
|
136
|
+
|
|
136
137
|
# Creates or recreates automatically defined indices
|
|
137
138
|
def create_deferred_indices(drop_existing = false)
|
|
138
139
|
@deferred_indices.each { |definition|
|
|
139
140
|
name = definition.index_name
|
|
140
|
-
if
|
|
141
|
+
if drop_existing && indices.include?(name)
|
|
141
142
|
drop_index(name)
|
|
142
143
|
end
|
|
143
|
-
unless
|
|
144
|
+
unless indices.include?(name)
|
|
144
145
|
index = create_index(definition)
|
|
145
|
-
if
|
|
146
|
+
if index.target_klass.respond_to?(:odba_extent)
|
|
146
147
|
index.fill(index.target_klass.odba_extent)
|
|
147
148
|
end
|
|
148
149
|
end
|
|
149
150
|
}
|
|
150
151
|
end
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
152
|
+
|
|
153
|
+
# Creates a new index according to IndexDefinition
|
|
154
|
+
def create_index(index_definition, origin_module = Object)
|
|
155
|
+
transaction {
|
|
156
|
+
klass = if index_definition.fulltext
|
|
157
|
+
FulltextIndex
|
|
158
|
+
elsif index_definition.resolve_search_term.is_a?(Hash)
|
|
159
|
+
ConditionIndex
|
|
160
|
+
else
|
|
161
|
+
Index
|
|
162
|
+
end
|
|
163
|
+
index = klass.new(index_definition, origin_module)
|
|
164
|
+
indices.store(index_definition.index_name, index)
|
|
165
|
+
indices.odba_store_unsaved
|
|
166
|
+
index
|
|
167
|
+
}
|
|
168
|
+
end
|
|
169
|
+
|
|
170
|
+
# Permanently deletes _object_ from the database and deconnects all connected
|
|
171
|
+
# Persistables
|
|
172
|
+
def delete(odba_object)
|
|
173
|
+
odba_id = odba_object.odba_id
|
|
171
174
|
name = odba_object.odba_name
|
|
172
175
|
odba_object.odba_notify_observers(:delete, odba_id, odba_object.object_id)
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
176
|
+
rows = ODBA.storage.retrieve_connected_objects(odba_id)
|
|
177
|
+
rows.each { |row|
|
|
178
|
+
id = row.first
|
|
179
|
+
# Self-Referencing objects don't have to be resaved
|
|
180
|
+
begin
|
|
181
|
+
if (connected_object = fetch(id, nil))
|
|
182
|
+
connected_object.odba_cut_connection(odba_object)
|
|
183
|
+
connected_object.odba_isolated_store
|
|
184
|
+
end
|
|
185
|
+
rescue OdbaError
|
|
186
|
+
warn "OdbaError ### deleting #{odba_object.class}:#{odba_id}"
|
|
187
|
+
warn " ### while looking for connected object #{id}"
|
|
188
|
+
end
|
|
189
|
+
}
|
|
187
190
|
delete_cache_entry(odba_id)
|
|
188
191
|
delete_cache_entry(name)
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
192
|
+
ODBA.storage.delete_persistable(odba_id)
|
|
193
|
+
delete_index_element(odba_object)
|
|
194
|
+
odba_object
|
|
195
|
+
end
|
|
196
|
+
|
|
193
197
|
def delete_cache_entry(key)
|
|
194
198
|
@cache_mutex.synchronize {
|
|
195
199
|
@fetched.delete(key)
|
|
196
200
|
@prefetched.delete(key)
|
|
197
201
|
}
|
|
198
202
|
end
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
203
|
+
|
|
204
|
+
def delete_index_element(odba_object) # :nodoc:
|
|
205
|
+
odba_object.class
|
|
206
|
+
indices.each_value { |index|
|
|
202
207
|
index.delete(odba_object)
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
208
|
+
}
|
|
209
|
+
end
|
|
210
|
+
|
|
211
|
+
# Permanently deletes the index named _index_name_
|
|
212
|
+
def drop_index(index_name)
|
|
213
|
+
transaction {
|
|
214
|
+
ODBA.storage.drop_index(index_name)
|
|
215
|
+
delete(indices[index_name])
|
|
216
|
+
}
|
|
217
|
+
end
|
|
218
|
+
|
|
212
219
|
def drop_indices # :nodoc:
|
|
213
|
-
keys =
|
|
214
|
-
keys.each{ |key|
|
|
220
|
+
keys = indices.keys
|
|
221
|
+
keys.each { |key|
|
|
215
222
|
drop_index(key)
|
|
216
223
|
}
|
|
217
224
|
end
|
|
225
|
+
|
|
218
226
|
# Queue an index for creation by #setup
|
|
219
227
|
def ensure_index_deferred(index_definition)
|
|
220
228
|
@deferred_indices.push(index_definition)
|
|
221
|
-
|
|
229
|
+
end
|
|
230
|
+
|
|
222
231
|
# Get all instances of a class (- a limited extent)
|
|
223
|
-
def extent(klass, odba_caller=nil)
|
|
224
|
-
|
|
232
|
+
def extent(klass, odba_caller = nil)
|
|
233
|
+
bulk_fetch(ODBA.storage.extent_ids(klass), odba_caller)
|
|
225
234
|
end
|
|
235
|
+
|
|
226
236
|
# Get number of instances of a class
|
|
227
237
|
def count(klass)
|
|
228
238
|
ODBA.storage.extent_count(klass)
|
|
229
239
|
end
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
240
|
+
|
|
241
|
+
# Fetch a Persistable identified by _odba_id_. Registers _odba_caller_ with
|
|
242
|
+
# the CacheEntry. Loads the Persistable if it is not already loaded.
|
|
243
|
+
def fetch(odba_id, odba_caller = nil)
|
|
244
|
+
fetch_or_do(odba_id, odba_caller) {
|
|
234
245
|
load_object(odba_id, odba_caller)
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
246
|
+
}
|
|
247
|
+
end
|
|
248
|
+
|
|
249
|
+
def fetch_cache_entry(odba_id_or_name) # :nodoc:
|
|
250
|
+
@prefetched[odba_id_or_name] || @fetched[odba_id_or_name]
|
|
251
|
+
end
|
|
252
|
+
@@receiver_name = :@receiver
|
|
253
|
+
def fetch_collection(odba_obj) # :nodoc:
|
|
254
|
+
collection = []
|
|
255
|
+
bulk_fetch_ids = []
|
|
256
|
+
rows = ODBA.storage.restore_collection(odba_obj.odba_id)
|
|
245
257
|
return collection if rows.empty?
|
|
246
258
|
idx = 0
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
259
|
+
rows.each { |row|
|
|
260
|
+
key = row[0].is_a?(Integer) ? row[0] : ODBA.marshaller.load(row[0])
|
|
261
|
+
value = row[1].is_a?(Integer) ? row[1] : ODBA.marshaller.load(row[1])
|
|
262
|
+
idx += 1
|
|
251
263
|
item = nil
|
|
252
|
-
if
|
|
264
|
+
if [key, value].any? { |item| item.instance_variable_get(@@receiver_name) }
|
|
253
265
|
odba_id = odba_obj.odba_id
|
|
254
266
|
warn "stub for #{item.class}:#{item.odba_id} was saved with receiver in collection of #{odba_obj.class}:#{odba_id}"
|
|
255
267
|
warn "repair: remove [#{odba_id}, #{row[0]}, #{row[1].length}]"
|
|
@@ -261,71 +273,75 @@ module ODBA
|
|
|
261
273
|
warn "repair: insert [#{odba_id}, #{key_dump}, #{value_dump.length}]"
|
|
262
274
|
ODBA.storage.collection_store(odba_id, key_dump, value_dump)
|
|
263
275
|
end
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
276
|
+
bulk_fetch_ids.push(key.odba_id)
|
|
277
|
+
bulk_fetch_ids.push(value.odba_id)
|
|
278
|
+
collection.push([key, value])
|
|
279
|
+
}
|
|
280
|
+
bulk_fetch_ids.compact!
|
|
281
|
+
bulk_fetch_ids.uniq!
|
|
282
|
+
bulk_fetch(bulk_fetch_ids, odba_obj)
|
|
283
|
+
collection.each { |pair|
|
|
284
|
+
pair.collect! { |item|
|
|
285
|
+
if item.is_a?(ODBA::Stub)
|
|
274
286
|
## don't fetch: that may result in a conflict when storing.
|
|
275
|
-
#fetch(item.odba_id, odba_obj)
|
|
287
|
+
# fetch(item.odba_id, odba_obj)
|
|
276
288
|
item.odba_container = odba_obj
|
|
277
289
|
item
|
|
278
|
-
elsif(ce = fetch_cache_entry(item.odba_id))
|
|
290
|
+
elsif (ce = fetch_cache_entry(item.odba_id))
|
|
279
291
|
warn "collection loaded unstubbed object: #{item.odba_id}"
|
|
280
292
|
ce.odba_add_reference(odba_obj)
|
|
281
293
|
ce.odba_object
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
294
|
+
else
|
|
295
|
+
item
|
|
296
|
+
end
|
|
297
|
+
}
|
|
298
|
+
}
|
|
299
|
+
collection
|
|
300
|
+
end
|
|
301
|
+
|
|
302
|
+
def fetch_collection_element(odba_id, key) # :nodoc:
|
|
303
|
+
key_dump = ODBA.marshaller.dump(key.odba_isolated_stub)
|
|
304
|
+
## for backward-compatibility and robustness we only attempt
|
|
305
|
+
## to load if there was a dump stored in the collection table
|
|
306
|
+
if (dump = ODBA.storage.collection_fetch(odba_id, key_dump))
|
|
307
|
+
item = ODBA.marshaller.load(dump)
|
|
308
|
+
if item.is_a?(ODBA::Stub)
|
|
296
309
|
fetch(item.odba_id)
|
|
297
|
-
elsif
|
|
310
|
+
elsif item.is_a?(ODBA::Persistable)
|
|
298
311
|
warn "collection_element was unstubbed object: #{item.odba_id}"
|
|
299
312
|
fetch_or_restore(item.odba_id, dump, nil)
|
|
300
313
|
else
|
|
301
314
|
item
|
|
302
315
|
end
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
316
|
+
end
|
|
317
|
+
end
|
|
318
|
+
|
|
319
|
+
def fetch_named(name, odba_caller, &block) # :nodoc:
|
|
320
|
+
fetch_or_do(name, odba_caller) {
|
|
321
|
+
dump = ODBA.storage.restore_named(name)
|
|
322
|
+
if dump.nil?
|
|
323
|
+
odba_obj = block.call
|
|
324
|
+
odba_obj.odba_name = name
|
|
325
|
+
odba_obj.odba_store(name)
|
|
326
|
+
odba_obj
|
|
327
|
+
else
|
|
328
|
+
fetch_or_restore(name, dump, odba_caller)
|
|
329
|
+
end
|
|
330
|
+
}
|
|
331
|
+
end
|
|
332
|
+
|
|
333
|
+
def fetch_or_do(obj_id, odba_caller, &block) # :nodoc:
|
|
334
|
+
if (cache_entry = fetch_cache_entry(obj_id)) && cache_entry._odba_object
|
|
335
|
+
cache_entry.odba_add_reference(odba_caller)
|
|
336
|
+
cache_entry.odba_object
|
|
337
|
+
else
|
|
338
|
+
block.call
|
|
339
|
+
end
|
|
340
|
+
end
|
|
341
|
+
|
|
326
342
|
def fetch_or_restore(odba_id, dump, odba_caller) # :nodoc:
|
|
327
343
|
fetch_or_do(odba_id, odba_caller) {
|
|
328
|
-
odba_obj,
|
|
344
|
+
odba_obj, _ = restore(dump)
|
|
329
345
|
@cache_mutex.synchronize {
|
|
330
346
|
fetch_or_do(odba_id, odba_caller) {
|
|
331
347
|
cache_entry = CacheEntry.new(odba_obj)
|
|
@@ -341,53 +357,61 @@ module ODBA
|
|
|
341
357
|
}
|
|
342
358
|
}
|
|
343
359
|
end
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
360
|
+
|
|
361
|
+
def fill_index(index_name, targets)
|
|
362
|
+
indices[index_name].fill(targets)
|
|
363
|
+
end
|
|
364
|
+
|
|
365
|
+
# Checks wether the object identified by _odba_id_ has been loaded.
|
|
366
|
+
def include?(odba_id)
|
|
367
|
+
@fetched.include?(odba_id) || @prefetched.include?(odba_id)
|
|
368
|
+
end
|
|
369
|
+
|
|
370
|
+
def index_keys(index_name, length = nil)
|
|
371
|
+
index = indices.fetch(index_name)
|
|
372
|
+
index.keys(length)
|
|
373
|
+
end
|
|
374
|
+
|
|
375
|
+
def index_matches(index_name, substring, limit = nil, offset = 0)
|
|
376
|
+
index = indices.fetch(index_name)
|
|
377
|
+
index.matches substring, limit, offset
|
|
378
|
+
end
|
|
379
|
+
|
|
380
|
+
# Returns a Hash-table containing all stored indices.
|
|
381
|
+
def indices
|
|
382
|
+
@indices ||= fetch_named("__cache_server_indices__", self) {
|
|
383
|
+
{}
|
|
384
|
+
}
|
|
385
|
+
end
|
|
386
|
+
|
|
365
387
|
def invalidate(odba_id)
|
|
366
388
|
## when finalizers are run, no other threads will be scheduled,
|
|
367
389
|
# therefore we don't need to @cache_mutex.synchronize
|
|
368
390
|
@fetched.delete odba_id
|
|
369
391
|
@prefetched.delete odba_id
|
|
370
392
|
end
|
|
393
|
+
|
|
371
394
|
def invalidate!(*odba_ids)
|
|
372
395
|
odba_ids.each do |odba_id|
|
|
373
396
|
if entry = fetch_cache_entry(odba_id)
|
|
374
|
-
entry.odba_retire :
|
|
397
|
+
entry.odba_retire force: true
|
|
375
398
|
end
|
|
376
399
|
invalidate odba_id
|
|
377
400
|
end
|
|
378
401
|
end
|
|
379
402
|
# File lock exclusive control between processes, not threads, to create safely a new odba_id
|
|
380
403
|
# Sometimes several update jobs (processes) to the same database at the same time
|
|
381
|
-
LOCK_FILE =
|
|
382
|
-
COUNT_FILE =
|
|
404
|
+
LOCK_FILE = "/tmp/lockfile"
|
|
405
|
+
COUNT_FILE = "/tmp/count"
|
|
383
406
|
def lock(dbname)
|
|
384
407
|
lock_file = LOCK_FILE + "." + dbname
|
|
385
|
-
open(lock_file,
|
|
408
|
+
open(lock_file, "a") do |st|
|
|
386
409
|
st.flock(File::LOCK_EX)
|
|
387
410
|
yield
|
|
388
411
|
st.flock(File::LOCK_UN)
|
|
389
412
|
end
|
|
390
413
|
end
|
|
414
|
+
|
|
391
415
|
def new_id(dbname, odba_storage)
|
|
392
416
|
count_file = COUNT_FILE + "." + dbname
|
|
393
417
|
count = nil
|
|
@@ -406,66 +430,75 @@ module ODBA
|
|
|
406
430
|
end
|
|
407
431
|
count
|
|
408
432
|
end
|
|
433
|
+
|
|
409
434
|
# Returns the next valid odba_id
|
|
410
435
|
def next_id
|
|
411
436
|
if @file_lock
|
|
412
|
-
dbname = ODBA.storage.instance_variable_get(
|
|
437
|
+
dbname = ODBA.storage.instance_variable_get(:@dbi).dbi_args.first.split(":").last
|
|
413
438
|
id = new_id(dbname, ODBA.storage)
|
|
414
439
|
else
|
|
415
440
|
id = ODBA.storage.next_id
|
|
416
441
|
end
|
|
417
442
|
@peers.each do |peer|
|
|
418
|
-
peer.reserve_next_id id
|
|
443
|
+
peer.reserve_next_id id
|
|
444
|
+
rescue
|
|
445
|
+
DRb::DRbError
|
|
419
446
|
end
|
|
420
447
|
id
|
|
421
448
|
rescue OdbaDuplicateIdError
|
|
422
449
|
retry
|
|
423
450
|
end
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
451
|
+
|
|
452
|
+
# Use this to load all prefetchable Persistables from the db at once
|
|
453
|
+
def prefetch
|
|
454
|
+
bulk_restore(ODBA.storage.restore_prefetchable)
|
|
455
|
+
end
|
|
456
|
+
|
|
428
457
|
# prints loading statistics to $stdout
|
|
429
458
|
def print_stats
|
|
430
459
|
fmh = " %-20s | %10s | %5s | %6s | %6s | %6s | %-20s\n"
|
|
431
460
|
fmt = " %-20s | %10.3f | %5i | %6.3f | %6.3f | %6.3f | %s\n"
|
|
432
461
|
head = sprintf(fmh,
|
|
433
|
-
|
|
462
|
+
"class", "total", "count", "min", "max", "avg", "callers")
|
|
434
463
|
line = "-" * head.length
|
|
435
464
|
puts line
|
|
436
465
|
print head
|
|
437
466
|
puts line
|
|
438
467
|
@loading_stats.sort_by { |key, val|
|
|
439
468
|
val[:total_time]
|
|
440
|
-
}.
|
|
469
|
+
}.reverse_each { |key, val|
|
|
441
470
|
key = key.to_s
|
|
442
|
-
if
|
|
443
|
-
key = key[-20,20]
|
|
471
|
+
if key.length > 20
|
|
472
|
+
key = key[-20, 20]
|
|
444
473
|
end
|
|
445
474
|
avg = val[:total_time] / val[:count]
|
|
446
475
|
printf(fmt, key, val[:total_time], val[:count],
|
|
447
|
-
|
|
448
|
-
|
|
476
|
+
val[:times].min, val[:times].max, avg,
|
|
477
|
+
val[:callers].join(","))
|
|
449
478
|
}
|
|
450
479
|
puts line
|
|
451
480
|
$stdout.flush
|
|
452
481
|
end
|
|
482
|
+
|
|
453
483
|
# Register a peer that has access to the same DB backend
|
|
454
484
|
def register_peer peer
|
|
455
485
|
@peers.push(peer).uniq!
|
|
456
486
|
end
|
|
487
|
+
|
|
457
488
|
# Reserve an id with all registered peers
|
|
458
489
|
def reserve_next_id id
|
|
459
490
|
ODBA.storage.reserve_next_id id
|
|
460
491
|
end
|
|
492
|
+
|
|
461
493
|
# Clears the loading statistics
|
|
462
494
|
def reset_stats
|
|
463
495
|
@loading_stats.clear
|
|
464
496
|
end
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
497
|
+
|
|
498
|
+
# Find objects in an index
|
|
499
|
+
def retrieve_from_index(index_name, search_term, meta = nil)
|
|
500
|
+
index = indices.fetch(index_name)
|
|
501
|
+
ids = index.fetch_ids(search_term, meta)
|
|
469
502
|
if meta.respond_to?(:error_limit) && (limit = meta.error_limit) \
|
|
470
503
|
&& (size = ids.size) > limit.to_i
|
|
471
504
|
error = OdbaResultLimitError.new
|
|
@@ -476,47 +509,51 @@ module ODBA
|
|
|
476
509
|
error.meta = meta
|
|
477
510
|
raise error
|
|
478
511
|
end
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
512
|
+
bulk_fetch(ids, nil)
|
|
513
|
+
end
|
|
514
|
+
|
|
515
|
+
# Create necessary DB-Structure / other storage-setup
|
|
482
516
|
def setup
|
|
483
517
|
ODBA.storage.setup
|
|
484
|
-
|
|
518
|
+
indices.each_key { |index_name|
|
|
485
519
|
ODBA.storage.ensure_target_id_index(index_name)
|
|
486
520
|
}
|
|
487
521
|
create_deferred_indices
|
|
488
522
|
nil
|
|
489
523
|
end
|
|
524
|
+
|
|
490
525
|
# Returns the total number of cached objects
|
|
491
526
|
def size
|
|
492
527
|
@prefetched.size + @fetched.size
|
|
493
528
|
end
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
529
|
+
|
|
530
|
+
def start_cleaner # :nodoc:
|
|
531
|
+
@cleaner = Thread.new {
|
|
532
|
+
Thread.current.priority = self.class::CLEANER_PRIORITY
|
|
533
|
+
loop {
|
|
534
|
+
sleep(self.class::CLEANING_INTERVAL)
|
|
535
|
+
begin
|
|
536
|
+
clean
|
|
537
|
+
rescue => e
|
|
538
|
+
puts e
|
|
539
|
+
puts e.backtrace
|
|
540
|
+
end
|
|
541
|
+
}
|
|
542
|
+
}
|
|
543
|
+
end
|
|
544
|
+
|
|
545
|
+
# Store a Persistable _object_ in the database
|
|
546
|
+
def store(object)
|
|
547
|
+
odba_id = object.odba_id
|
|
548
|
+
name = object.odba_name
|
|
512
549
|
object.odba_notify_observers(:store, odba_id, object.object_id)
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
550
|
+
if (ids = Thread.current[:txids])
|
|
551
|
+
ids.unshift([odba_id, name])
|
|
552
|
+
end
|
|
516
553
|
## get target_ids before anything else
|
|
517
554
|
target_ids = object.odba_target_ids
|
|
518
|
-
|
|
519
|
-
|
|
555
|
+
store_collection_elements(object)
|
|
556
|
+
prefetchable = object.odba_prefetch?
|
|
520
557
|
dump = object.odba_isolated_dump
|
|
521
558
|
ODBA.storage.store(odba_id, dump, name, prefetchable, object.class)
|
|
522
559
|
store_object_connections(odba_id, target_ids)
|
|
@@ -524,11 +561,14 @@ module ODBA
|
|
|
524
561
|
object = store_cache_entry(odba_id, object, name)
|
|
525
562
|
update_indices(object)
|
|
526
563
|
@peers.each do |peer|
|
|
527
|
-
peer.invalidate! odba_id
|
|
564
|
+
peer.invalidate! odba_id
|
|
565
|
+
rescue
|
|
566
|
+
DRb::DRbError
|
|
528
567
|
end
|
|
529
568
|
object
|
|
530
|
-
|
|
531
|
-
|
|
569
|
+
end
|
|
570
|
+
|
|
571
|
+
def store_cache_entry(odba_id, object, name = nil) # :nodoc:
|
|
532
572
|
@cache_mutex.synchronize {
|
|
533
573
|
if cache_entry = fetch_cache_entry(odba_id)
|
|
534
574
|
cache_entry.update object
|
|
@@ -536,18 +576,19 @@ module ODBA
|
|
|
536
576
|
hash = object.odba_prefetch? ? @prefetched : @fetched
|
|
537
577
|
cache_entry = CacheEntry.new(object)
|
|
538
578
|
hash.store(odba_id, cache_entry)
|
|
539
|
-
unless
|
|
579
|
+
unless name.nil?
|
|
540
580
|
hash.store(name, cache_entry)
|
|
541
581
|
end
|
|
542
582
|
end
|
|
543
583
|
cache_entry.odba_object
|
|
544
584
|
}
|
|
545
585
|
end
|
|
586
|
+
|
|
546
587
|
def store_collection_elements(odba_obj) # :nodoc:
|
|
547
588
|
odba_id = odba_obj.odba_id
|
|
548
589
|
collection = odba_obj.odba_collection.collect { |key, value|
|
|
549
|
-
[
|
|
550
|
-
ODBA.marshaller.dump(value.odba_isolated_stub)
|
|
590
|
+
[ODBA.marshaller.dump(key.odba_isolated_stub),
|
|
591
|
+
ODBA.marshaller.dump(value.odba_isolated_stub)]
|
|
551
592
|
}
|
|
552
593
|
old_collection = ODBA.storage.restore_collection(odba_id).collect { |row|
|
|
553
594
|
[row[0], row [1]]
|
|
@@ -559,95 +600,105 @@ module ODBA
|
|
|
559
600
|
ODBA.storage.collection_store(odba_id, key_dump, value_dump)
|
|
560
601
|
}.size
|
|
561
602
|
end
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
603
|
+
|
|
604
|
+
def store_object_connections(odba_id, target_ids) # :nodoc:
|
|
605
|
+
ODBA.storage.ensure_object_connections(odba_id, target_ids)
|
|
606
|
+
end
|
|
607
|
+
|
|
608
|
+
# Executes the block in a transaction. If the transaction fails, all
|
|
609
|
+
# affected Persistable objects are reloaded from the db (which by then has
|
|
610
|
+
# also performed a rollback). Rollback is quite inefficient at this time.
|
|
611
|
+
def transaction(&block)
|
|
612
|
+
Thread.current[:txids] = []
|
|
613
|
+
ODBA.storage.transaction(&block)
|
|
614
|
+
rescue Exception => excp
|
|
615
|
+
transaction_rollback
|
|
616
|
+
raise excp
|
|
617
|
+
ensure
|
|
618
|
+
Thread.current[:txids] = nil
|
|
619
|
+
end
|
|
620
|
+
|
|
621
|
+
def transaction_rollback # :nodoc:
|
|
622
|
+
if (ids = Thread.current[:txids])
|
|
623
|
+
ids.each { |id, name|
|
|
624
|
+
if (entry = fetch_cache_entry(id))
|
|
625
|
+
if (dump = ODBA.storage.restore(id))
|
|
626
|
+
odba_obj, _ = restore(dump)
|
|
627
|
+
entry.odba_replace!(odba_obj)
|
|
628
|
+
else
|
|
629
|
+
entry.odba_cut_connections!
|
|
586
630
|
delete_cache_entry(id)
|
|
587
631
|
delete_cache_entry(name)
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
632
|
+
end
|
|
633
|
+
end
|
|
634
|
+
}
|
|
635
|
+
end
|
|
636
|
+
end
|
|
637
|
+
|
|
593
638
|
# Unregister a peer
|
|
594
639
|
def unregister_peer peer
|
|
595
640
|
@peers.delete peer
|
|
596
641
|
end
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
642
|
+
|
|
643
|
+
def update_indices(odba_object) # :nodoc:
|
|
644
|
+
if odba_object.odba_indexable?
|
|
645
|
+
indices.each { |index_name, index|
|
|
646
|
+
index.update(odba_object)
|
|
647
|
+
}
|
|
648
|
+
end
|
|
649
|
+
end
|
|
650
|
+
|
|
651
|
+
def update_references(target_ids, object) # :nodoc:
|
|
652
|
+
target_ids.each { |odba_id|
|
|
653
|
+
if (entry = fetch_cache_entry(odba_id))
|
|
654
|
+
entry.odba_add_reference(object)
|
|
655
|
+
end
|
|
656
|
+
}
|
|
657
|
+
end
|
|
658
|
+
|
|
659
|
+
private
|
|
660
|
+
|
|
612
661
|
def load_object(odba_id, odba_caller)
|
|
613
|
-
start = Time.now if
|
|
662
|
+
start = Time.now if @debug
|
|
614
663
|
dump = ODBA.storage.restore(odba_id)
|
|
615
664
|
odba_obj = restore_object(odba_id, dump, odba_caller)
|
|
616
|
-
return odba_obj unless
|
|
665
|
+
return odba_obj unless @debug
|
|
617
666
|
stats = (@loading_stats[odba_obj.class] ||= {
|
|
618
|
-
:
|
|
667
|
+
count: 0, times: [], total_time: 0, callers: []
|
|
619
668
|
})
|
|
620
669
|
stats[:count] += 1
|
|
621
670
|
time = Time.now - start
|
|
622
671
|
stats[:times].push(time)
|
|
623
672
|
stats[:total_time] += time
|
|
624
673
|
stats[:callers].push(odba_caller.class).uniq!
|
|
625
|
-
if
|
|
674
|
+
if time > 2
|
|
626
675
|
names = []
|
|
627
676
|
odba_caller.instance_variables.each { |name|
|
|
628
|
-
if
|
|
677
|
+
if odba_caller.instance_variable_get(name).odba_id == odba_id
|
|
629
678
|
names.push(name)
|
|
630
679
|
end
|
|
631
680
|
}
|
|
632
681
|
printf("long load-time (%4.2fs) for odba_id %i: %s#%s\n",
|
|
633
|
-
|
|
682
|
+
time, odba_id, odba_caller, names.join(","))
|
|
634
683
|
end
|
|
635
684
|
odba_obj
|
|
636
685
|
end
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
686
|
+
|
|
687
|
+
def restore(dump)
|
|
688
|
+
odba_obj = ODBA.marshaller.load(dump)
|
|
689
|
+
unless odba_obj.is_a?(Persistable)
|
|
640
690
|
odba_obj.extend(Persistable)
|
|
641
691
|
end
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
692
|
+
collection = fetch_collection(odba_obj)
|
|
693
|
+
odba_obj.odba_restore(collection)
|
|
694
|
+
[odba_obj, collection]
|
|
695
|
+
end
|
|
696
|
+
|
|
697
|
+
def restore_object(odba_id, dump, odba_caller)
|
|
698
|
+
if dump.nil?
|
|
699
|
+
raise OdbaError, "Unknown odba_id #{odba_id}"
|
|
700
|
+
end
|
|
701
|
+
fetch_or_restore(odba_id, dump, odba_caller)
|
|
702
|
+
end
|
|
703
|
+
end
|
|
653
704
|
end
|