promiscuous 0.90.0 → 0.91.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/lib/promiscuous/amqp/bunny.rb +63 -36
- data/lib/promiscuous/amqp/fake.rb +3 -1
- data/lib/promiscuous/amqp/hot_bunnies.rb +26 -16
- data/lib/promiscuous/amqp/null.rb +1 -0
- data/lib/promiscuous/amqp.rb +12 -12
- data/lib/promiscuous/cli.rb +70 -29
- data/lib/promiscuous/config.rb +54 -29
- data/lib/promiscuous/convenience.rb +1 -1
- data/lib/promiscuous/dependency.rb +25 -6
- data/lib/promiscuous/error/connection.rb +11 -9
- data/lib/promiscuous/error/dependency.rb +8 -1
- data/lib/promiscuous/loader.rb +4 -2
- data/lib/promiscuous/publisher/bootstrap/connection.rb +25 -0
- data/lib/promiscuous/publisher/bootstrap/data.rb +127 -0
- data/lib/promiscuous/publisher/bootstrap/mode.rb +19 -0
- data/lib/promiscuous/publisher/bootstrap/status.rb +40 -0
- data/lib/promiscuous/publisher/bootstrap/version.rb +46 -0
- data/lib/promiscuous/publisher/bootstrap.rb +27 -0
- data/lib/promiscuous/publisher/context/base.rb +67 -0
- data/lib/promiscuous/{middleware.rb → publisher/context/middleware.rb} +16 -13
- data/lib/promiscuous/publisher/context/transaction.rb +36 -0
- data/lib/promiscuous/publisher/context.rb +4 -88
- data/lib/promiscuous/publisher/mock_generator.rb +9 -9
- data/lib/promiscuous/publisher/model/active_record.rb +7 -7
- data/lib/promiscuous/publisher/model/base.rb +29 -29
- data/lib/promiscuous/publisher/model/ephemeral.rb +5 -3
- data/lib/promiscuous/publisher/model/mock.rb +9 -5
- data/lib/promiscuous/publisher/model/mongoid.rb +5 -22
- data/lib/promiscuous/publisher/operation/active_record.rb +360 -0
- data/lib/promiscuous/publisher/operation/atomic.rb +167 -0
- data/lib/promiscuous/publisher/operation/base.rb +279 -474
- data/lib/promiscuous/publisher/operation/mongoid.rb +153 -145
- data/lib/promiscuous/publisher/operation/non_persistent.rb +28 -0
- data/lib/promiscuous/publisher/operation/proxy_for_query.rb +42 -0
- data/lib/promiscuous/publisher/operation/transaction.rb +85 -0
- data/lib/promiscuous/publisher/operation.rb +1 -1
- data/lib/promiscuous/publisher/worker.rb +7 -7
- data/lib/promiscuous/publisher.rb +1 -1
- data/lib/promiscuous/railtie.rb +20 -5
- data/lib/promiscuous/redis.rb +104 -56
- data/lib/promiscuous/subscriber/message_processor/base.rb +38 -0
- data/lib/promiscuous/subscriber/message_processor/bootstrap.rb +17 -0
- data/lib/promiscuous/subscriber/message_processor/regular.rb +192 -0
- data/lib/promiscuous/subscriber/message_processor.rb +4 -0
- data/lib/promiscuous/subscriber/model/base.rb +20 -15
- data/lib/promiscuous/subscriber/model/mongoid.rb +4 -4
- data/lib/promiscuous/subscriber/model/observer.rb +16 -2
- data/lib/promiscuous/subscriber/operation/base.rb +68 -0
- data/lib/promiscuous/subscriber/operation/bootstrap.rb +54 -0
- data/lib/promiscuous/subscriber/operation/regular.rb +13 -0
- data/lib/promiscuous/subscriber/operation.rb +3 -166
- data/lib/promiscuous/subscriber/worker/message.rb +61 -35
- data/lib/promiscuous/subscriber/worker/message_synchronizer.rb +90 -59
- data/lib/promiscuous/subscriber/worker/pump.rb +17 -5
- data/lib/promiscuous/subscriber/worker/recorder.rb +4 -1
- data/lib/promiscuous/subscriber/worker/runner.rb +49 -9
- data/lib/promiscuous/subscriber/worker/stats.rb +2 -2
- data/lib/promiscuous/subscriber/worker.rb +6 -0
- data/lib/promiscuous/subscriber.rb +1 -1
- data/lib/promiscuous/timer.rb +31 -18
- data/lib/promiscuous/version.rb +1 -1
- data/lib/promiscuous.rb +23 -3
- metadata +104 -89
- data/lib/promiscuous/subscriber/payload.rb +0 -34
@@ -4,7 +4,7 @@ raise "moped > 1.3.2 please" unless Gem.loaded_specs['moped'].version >= Ge
|
|
4
4
|
require 'yaml'
|
5
5
|
|
6
6
|
class Moped::PromiscuousCollectionWrapper < Moped::Collection
|
7
|
-
class PromiscuousCollectionOperation < Promiscuous::Publisher::Operation::
|
7
|
+
class PromiscuousCollectionOperation < Promiscuous::Publisher::Operation::Atomic
|
8
8
|
def initialize(options={})
|
9
9
|
super
|
10
10
|
@operation = :create
|
@@ -21,13 +21,13 @@ class Moped::PromiscuousCollectionWrapper < Moped::Collection
|
|
21
21
|
rescue NameError
|
22
22
|
end
|
23
23
|
|
24
|
-
def
|
25
|
-
#
|
26
|
-
|
27
|
-
@document.to_yaml
|
24
|
+
def recovery_payload
|
25
|
+
# We use yaml because we need the BSON types.
|
26
|
+
[@instance.class.promiscuous_collection_name, @instance.id, @document.to_yaml]
|
28
27
|
end
|
29
28
|
|
30
|
-
def self.recover_operation(
|
29
|
+
def self.recover_operation(collection, instance_id, document)
|
30
|
+
model = Promiscuous::Publisher::Model::Mongoid.collection_mapping[collection]
|
31
31
|
document = YAML.load(document)
|
32
32
|
instance = Mongoid::Factory.from_db(model, document)
|
33
33
|
new(:collection => model.collection, :document => document, :instance => instance)
|
@@ -40,18 +40,17 @@ class Moped::PromiscuousCollectionWrapper < Moped::Collection
|
|
40
40
|
end
|
41
41
|
end
|
42
42
|
|
43
|
-
def
|
44
|
-
@document[VERSION_FIELD] = @instance_version
|
45
|
-
end
|
46
|
-
|
47
|
-
def execute_persistent(&db_operation)
|
43
|
+
def execute_instrumented(query)
|
48
44
|
@instance = Mongoid::Factory.from_db(model, @document)
|
49
45
|
super
|
50
46
|
end
|
51
47
|
|
52
|
-
def
|
53
|
-
|
54
|
-
|
48
|
+
def should_instrument_query?
|
49
|
+
super && model
|
50
|
+
end
|
51
|
+
|
52
|
+
def recoverable_failure?(exception)
|
53
|
+
exception.is_a?(Moped::Errors::ConnectionFailure)
|
55
54
|
end
|
56
55
|
end
|
57
56
|
|
@@ -74,8 +73,39 @@ class Moped::PromiscuousCollectionWrapper < Moped::Collection
|
|
74
73
|
end
|
75
74
|
|
76
75
|
class Moped::PromiscuousQueryWrapper < Moped::Query
|
77
|
-
|
78
|
-
|
76
|
+
module PromiscuousHelpers
|
77
|
+
def collection_name
|
78
|
+
@collection_name ||= @query.collection.is_a?(String) ? @query.collection : @query.collection.name
|
79
|
+
end
|
80
|
+
|
81
|
+
def model
|
82
|
+
@model ||= Promiscuous::Publisher::Model::Mongoid.collection_mapping[collection_name]
|
83
|
+
end
|
84
|
+
|
85
|
+
def get_selector_instance
|
86
|
+
selector = @query.operation.selector["$query"] || @query.operation.selector
|
87
|
+
|
88
|
+
# TODO use the original instance for an update/delete, that would be
|
89
|
+
# an even better hint.
|
90
|
+
|
91
|
+
# We only support == selectors, no $in, or $gt.
|
92
|
+
@selector = selector.select { |k,v| k.to_s =~ /^[^$]/ && !v.is_a?(Hash) }
|
93
|
+
|
94
|
+
# @instance is not really a proper instance of a model, it's just a
|
95
|
+
# convenient representation of a selector as explain in base.rb,
|
96
|
+
# which explain why we don't want any constructor to be called.
|
97
|
+
# Note that this optimistic mechanism also works with writes because
|
98
|
+
# the instance gets reloaded once the lock is taken. If the
|
99
|
+
# dependencies were incorrect, the locks will be released and
|
100
|
+
# reacquired appropriately.
|
101
|
+
model.allocate.tap { |doc| doc.instance_variable_set(:@attributes, @selector) }
|
102
|
+
end
|
103
|
+
end
|
104
|
+
|
105
|
+
class PromiscuousWriteOperation < Promiscuous::Publisher::Operation::Atomic
|
106
|
+
include Moped::PromiscuousQueryWrapper::PromiscuousHelpers
|
107
|
+
|
108
|
+
attr_accessor :change
|
79
109
|
|
80
110
|
def initialize(options={})
|
81
111
|
super
|
@@ -83,87 +113,60 @@ class Moped::PromiscuousQueryWrapper < Moped::Query
|
|
83
113
|
@change = options[:change]
|
84
114
|
end
|
85
115
|
|
86
|
-
def
|
87
|
-
@
|
116
|
+
def recovery_payload
|
117
|
+
[@instance.class.promiscuous_collection_name, @instance.id]
|
88
118
|
end
|
89
119
|
|
90
|
-
def
|
91
|
-
|
120
|
+
def self.recover_operation(collection, instance_id)
|
121
|
+
# TODO We need to use the primary database. We cannot read from a secondary.
|
122
|
+
model = Promiscuous::Publisher::Model::Mongoid.collection_mapping[collection]
|
123
|
+
query = model.unscoped.where(:id => instance_id).query
|
124
|
+
|
125
|
+
# We no-op the update operation instead of making it idempotent.
|
126
|
+
# To do so, we do a dummy update on the document.
|
127
|
+
# The original caller will fail because the lock was unlocked, so we'll
|
128
|
+
# won't send a different message.
|
129
|
+
new(:query => query, :change => {}).tap { |op| op.instance_eval { reload_instance } }
|
92
130
|
end
|
93
131
|
|
94
|
-
def
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
# TODO refactor this not so pretty instance_eval
|
100
|
-
op.instance_eval do
|
101
|
-
reload_instance
|
102
|
-
@instance ||= get_selector_instance
|
132
|
+
def recover_db_operation
|
133
|
+
if operation == :update
|
134
|
+
without_promiscuous { @query.update(@change) }
|
135
|
+
else
|
136
|
+
without_promiscuous { @query.remove }
|
103
137
|
end
|
104
|
-
op
|
105
138
|
end
|
106
139
|
|
107
|
-
def
|
108
|
-
|
109
|
-
# The original caller will fail because the lock was unlocked.
|
110
|
-
without_promiscuous { @query.update(@change) }
|
111
|
-
@operation = :dummy
|
140
|
+
def recoverable_failure?(exception)
|
141
|
+
exception.is_a?(Moped::Errors::ConnectionFailure)
|
112
142
|
end
|
113
143
|
|
114
144
|
def fetch_instance
|
115
|
-
|
116
|
-
Mongoid::Factory.from_db(model,
|
145
|
+
raw_instance = without_promiscuous { @query.first }
|
146
|
+
Mongoid::Factory.from_db(model, raw_instance) if raw_instance
|
117
147
|
end
|
118
148
|
|
119
149
|
def use_id_selector(options={})
|
120
|
-
selector = {'_id' => @instance.id}
|
150
|
+
selector = {'_id' => @instance.id}.merge(@query.selector.select { |k,v| k.to_s.include?("_id") })
|
121
151
|
|
122
152
|
if options[:use_atomic_version_selector]
|
123
|
-
version = @instance[
|
124
|
-
selector.merge!(
|
153
|
+
version = @instance[Promiscuous::Config.version_field]
|
154
|
+
selector.merge!(Promiscuous::Config.version_field => version)
|
125
155
|
end
|
126
156
|
|
127
157
|
@query.selector = selector
|
128
158
|
end
|
129
159
|
|
130
|
-
def
|
160
|
+
def stash_version_in_document(version)
|
131
161
|
@change['$set'] ||= {}
|
132
|
-
@change['$set'][
|
162
|
+
@change['$set'][Promiscuous::Config.version_field] = version
|
133
163
|
end
|
134
164
|
|
135
|
-
def
|
136
|
-
selector = @query.operation.selector["$query"] || @query.operation.selector
|
137
|
-
|
138
|
-
# TODO use the original instance for an update/delete, that would be
|
139
|
-
# an even better hint.
|
140
|
-
|
141
|
-
# We only support == selectors, no $in, or $gt.
|
142
|
-
@selector = selector.select { |k,v| k.to_s =~ /^[^$]/ && !v.is_a?(Hash) }
|
143
|
-
|
144
|
-
# @instance is not really a proper instance of a model, it's just a
|
145
|
-
# convenient representation of a selector as explain in base.rb,
|
146
|
-
# which explain why we don't want any constructor to be called.
|
147
|
-
# Note that this optimistic mechanism also works with writes because
|
148
|
-
# the instance gets reloaded once the lock is taken. If the
|
149
|
-
# dependencies were incorrect, the locks will be released and
|
150
|
-
# reacquired appropriately.
|
151
|
-
model.allocate.tap { |doc| doc.instance_variable_set(:@attributes, @selector) }
|
152
|
-
end
|
153
|
-
|
154
|
-
def execute_persistent(&db_operation)
|
165
|
+
def execute_instrumented(query)
|
155
166
|
# We are trying to be optimistic for the locking. We are trying to figure
|
156
167
|
# out our dependencies with the selector upfront to avoid an extra read
|
157
168
|
# from reload_instance.
|
158
|
-
@instance
|
159
|
-
super
|
160
|
-
end
|
161
|
-
|
162
|
-
def execute_non_persistent(&db_operation)
|
163
|
-
if multi?
|
164
|
-
@instance = get_selector_instance
|
165
|
-
@selector_keys = @selector.keys
|
166
|
-
end
|
169
|
+
@instance ||= get_selector_instance
|
167
170
|
super
|
168
171
|
end
|
169
172
|
|
@@ -185,48 +188,55 @@ class Moped::PromiscuousQueryWrapper < Moped::Query
|
|
185
188
|
# TODO maybe we should cache these things
|
186
189
|
# TODO discover field dependencies automatically (hard)
|
187
190
|
aliases = Hash[model.aliased_fields.map { |k,v| [v,k] }]
|
188
|
-
attributes = fields_in_query(@change).map { |f|
|
191
|
+
attributes = fields_in_query(@change).map { |f| [aliases[f.to_s], f] }.flatten.compact.map(&:to_sym)
|
189
192
|
(attributes & model.published_db_fields).present?
|
190
193
|
end
|
191
194
|
|
192
|
-
def
|
193
|
-
|
194
|
-
|
195
|
-
|
195
|
+
def should_instrument_query?
|
196
|
+
super && model && any_published_field_changed?
|
197
|
+
end
|
198
|
+
end
|
196
199
|
|
197
|
-
|
198
|
-
|
199
|
-
|
200
|
-
|
200
|
+
class PromiscuousReadOperation < Promiscuous::Publisher::Operation::NonPersistent
|
201
|
+
include Moped::PromiscuousQueryWrapper::PromiscuousHelpers
|
202
|
+
|
203
|
+
def initialize(options={})
|
201
204
|
super
|
205
|
+
@operation = :read
|
206
|
+
@query = options[:query]
|
207
|
+
end
|
208
|
+
|
209
|
+
def query_dependencies
|
210
|
+
deps = dependencies_for(get_selector_instance)
|
211
|
+
deps.empty? ? super : deps
|
202
212
|
end
|
203
|
-
end
|
204
213
|
|
205
|
-
|
206
|
-
|
214
|
+
def should_instrument_query?
|
215
|
+
super && model
|
216
|
+
end
|
207
217
|
end
|
208
218
|
|
209
|
-
def
|
210
|
-
|
211
|
-
@operation.selector = value
|
219
|
+
def promiscuous_read_operation(options={})
|
220
|
+
PromiscuousReadOperation.new(options.merge(:query => self))
|
212
221
|
end
|
213
222
|
|
214
|
-
def
|
215
|
-
|
223
|
+
def promiscuous_write_operation(operation, options={})
|
224
|
+
PromiscuousWriteOperation.new(options.merge(:query => self, :operation => operation))
|
216
225
|
end
|
217
226
|
|
218
|
-
def
|
219
|
-
|
227
|
+
def selector=(value)
|
228
|
+
@selector = value
|
229
|
+
@operation.selector = value
|
220
230
|
end
|
221
231
|
|
222
232
|
# Moped::Query
|
223
233
|
|
224
234
|
def count(*args)
|
225
|
-
|
235
|
+
promiscuous_read_operation(:operation_ext => :count).execute { super }.to_i
|
226
236
|
end
|
227
237
|
|
228
238
|
def distinct(key)
|
229
|
-
|
239
|
+
promiscuous_read_operation(:operation_ext => :distinct).execute { super }
|
230
240
|
end
|
231
241
|
|
232
242
|
def each
|
@@ -240,68 +250,82 @@ class Moped::PromiscuousQueryWrapper < Moped::Query
|
|
240
250
|
alias :cursor :each
|
241
251
|
|
242
252
|
def first
|
243
|
-
#
|
244
|
-
# sure that we add the id, otherwise we
|
245
|
-
|
246
|
-
|
247
|
-
|
253
|
+
# FIXME If the the user is using something like .only(), we need to make
|
254
|
+
# sure that we add the id, otherwise we are screwed.
|
255
|
+
op = promiscuous_read_operation
|
256
|
+
|
257
|
+
op.execute do |query|
|
258
|
+
query.non_instrumented { super }
|
259
|
+
query.instrumented do
|
260
|
+
super.tap do |doc|
|
261
|
+
op.instances = doc ? [Mongoid::Factory.from_db(op.model, doc)] : []
|
262
|
+
end
|
263
|
+
end
|
248
264
|
end
|
249
265
|
end
|
250
266
|
alias :one :first
|
251
267
|
|
252
268
|
def update(change, flags=nil)
|
253
|
-
|
254
|
-
|
269
|
+
update_op = promiscuous_write_operation(:update, :change => change)
|
270
|
+
|
271
|
+
if flags && update_op.should_instrument_query?
|
272
|
+
raise "You cannot do a multi update. Instead, update each document separately." if flags.include?(:multi)
|
273
|
+
raise "No upsert support yet" if flags.include?(:upsert)
|
274
|
+
end
|
255
275
|
|
256
|
-
|
257
|
-
|
258
|
-
|
259
|
-
|
276
|
+
update_op.execute do |query|
|
277
|
+
query.non_instrumented { super }
|
278
|
+
query.instrumented do |op|
|
279
|
+
raw_instance = without_promiscuous { modify(change, :new => true) }
|
280
|
+
op.instance = Mongoid::Factory.from_db(op.model, raw_instance)
|
260
281
|
{'updatedExisting' => true, 'n' => 1, 'err' => nil, 'ok' => 1.0}
|
261
|
-
else
|
262
|
-
super
|
263
282
|
end
|
264
283
|
end
|
265
284
|
end
|
266
285
|
|
267
286
|
def modify(change, options={})
|
268
|
-
|
269
|
-
|
287
|
+
promiscuous_write_operation(:update, :change => change).execute do |query|
|
288
|
+
query.non_instrumented { super }
|
289
|
+
query.instrumented do |op|
|
290
|
+
raise "You can only use find_and_modify() with :new => true" if !options[:new]
|
291
|
+
super.tap { |raw_instance| op.instance = Mongoid::Factory.from_db(op.model, raw_instance) }
|
292
|
+
end
|
293
|
+
end
|
270
294
|
end
|
271
295
|
|
272
296
|
def remove
|
273
|
-
|
274
|
-
# FIXME raise when recovery raced
|
297
|
+
promiscuous_write_operation(:destroy).execute { super }
|
275
298
|
end
|
276
299
|
|
277
300
|
def remove_all
|
278
|
-
|
301
|
+
raise "Instead of doing a multi delete, delete each document separatly.\n" +
|
302
|
+
"Declare your has_many relationships with :dependent => :destroy instead of :delete"
|
279
303
|
end
|
280
304
|
end
|
281
305
|
|
282
306
|
class Moped::PromiscuousCursorWrapper < Moped::Cursor
|
283
|
-
def promiscuous_operation(op, options={})
|
284
|
-
Moped::PromiscuousQueryWrapper::PromiscuousQueryOperation.new(
|
285
|
-
options.merge(:query => @query, :operation => op))
|
286
|
-
end
|
287
|
-
|
288
307
|
# Moped::Cursor
|
289
|
-
|
290
|
-
|
291
|
-
|
292
|
-
|
308
|
+
def promiscuous_read_each(&block)
|
309
|
+
op = Moped::PromiscuousQueryWrapper::PromiscuousReadOperation.new(
|
310
|
+
:query => @query, :operation_ext => :each)
|
311
|
+
|
312
|
+
op.execute do |query|
|
313
|
+
query.non_instrumented { block.call.to_a }
|
314
|
+
query.instrumented do
|
315
|
+
block.call.to_a.tap do |docs|
|
316
|
+
op.instances = docs.map { |doc| Mongoid::Factory.from_db(op.model, doc) }
|
317
|
+
end
|
318
|
+
end
|
319
|
+
end
|
293
320
|
end
|
294
321
|
|
295
322
|
def load_docs
|
296
|
-
|
297
|
-
promiscuous_operation(:read, :multi => !should_fake_single_read).execute do |operation|
|
298
|
-
operation && should_fake_single_read ? fake_single_read(operation) : super
|
299
|
-
end.to_a
|
323
|
+
promiscuous_read_each { super }
|
300
324
|
end
|
301
325
|
|
302
326
|
def get_more
|
303
327
|
# TODO support batch_size
|
304
|
-
|
328
|
+
promiscuous_read_each { super }
|
305
329
|
end
|
306
330
|
|
307
331
|
def initialize(session, query_operation)
|
@@ -313,9 +337,8 @@ end
|
|
313
337
|
class Moped::PromiscuousDatabase < Moped::Database
|
314
338
|
# TODO it might be safer to use the alias attribute method because promiscuous
|
315
339
|
# may come late in the loading.
|
316
|
-
def
|
317
|
-
Moped::PromiscuousQueryWrapper::
|
318
|
-
options.merge(:operation => op))
|
340
|
+
def promiscuous_read_operation(options={})
|
341
|
+
Moped::PromiscuousQueryWrapper::PromiscuousReadOperation.new(options)
|
319
342
|
end
|
320
343
|
|
321
344
|
# Moped::Database
|
@@ -323,29 +346,13 @@ class Moped::PromiscuousDatabase < Moped::Database
|
|
323
346
|
def command(command)
|
324
347
|
if command[:mapreduce]
|
325
348
|
query = Moped::Query.new(self[command[:mapreduce]], command[:query])
|
326
|
-
|
327
|
-
:operation_ext => :mapreduce, :multi => true).execute { super }
|
349
|
+
promiscuous_read_operation(:query => query, :operation_ext => :mapreduce).execute { super }
|
328
350
|
else
|
329
351
|
super
|
330
352
|
end
|
331
353
|
end
|
332
354
|
end
|
333
355
|
|
334
|
-
class Mongoid::Contextual::Mongo
|
335
|
-
alias_method :each_hijacked, :each
|
336
|
-
|
337
|
-
def each(&block)
|
338
|
-
query.without_promiscuous! if criteria.options[:without_promiscuous]
|
339
|
-
each_hijacked(&block)
|
340
|
-
end
|
341
|
-
end
|
342
|
-
|
343
|
-
module Origin::Optional
|
344
|
-
def without_promiscuous
|
345
|
-
clone.tap { |criteria| criteria.options.store(:without_promiscuous, true) }
|
346
|
-
end
|
347
|
-
end
|
348
|
-
|
349
356
|
class Mongoid::Validations::UniquenessValidator
|
350
357
|
alias_method :validate_root_without_promisucous, :validate_root
|
351
358
|
def validate_root(*args)
|
@@ -354,7 +361,8 @@ class Mongoid::Validations::UniquenessValidator
|
|
354
361
|
end
|
355
362
|
|
356
363
|
class Moped::BSON::ObjectId
|
357
|
-
# No {"$oid": "123"}, it's horrible
|
364
|
+
# No {"$oid": "123"}, it's horrible.
|
365
|
+
# TODO Document this shit.
|
358
366
|
def to_json(*args)
|
359
367
|
"\"#{to_s}\""
|
360
368
|
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
class Promiscuous::Publisher::Operation::NonPersistent < Promiscuous::Publisher::Operation::Base
|
2
|
+
# XXX As opposed to atomic operations, NonPersistent operations deals with an
|
3
|
+
# array of instances
|
4
|
+
attr_accessor :instances
|
5
|
+
|
6
|
+
def initialize(options={})
|
7
|
+
super
|
8
|
+
@instances = options[:instances].to_a
|
9
|
+
end
|
10
|
+
|
11
|
+
def execute_instrumented(db_operation)
|
12
|
+
db_operation.call_and_remember_result(:instrumented)
|
13
|
+
|
14
|
+
unless db_operation.failed?
|
15
|
+
current_context.read_operations << self if read?
|
16
|
+
trace_operation
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
def operation_payloads
|
21
|
+
return [] if self.failed?
|
22
|
+
@instances.map { |instance| payloads_for(instance) }
|
23
|
+
end
|
24
|
+
|
25
|
+
def query_dependencies
|
26
|
+
@instances.map { |instance| dependencies_for(instance) }
|
27
|
+
end
|
28
|
+
end
|
@@ -0,0 +1,42 @@
|
|
1
|
+
class Promiscuous::Publisher::Operation::ProxyForQuery
|
2
|
+
attr_accessor :exception, :result
|
3
|
+
|
4
|
+
def initialize(operation, &block)
|
5
|
+
@operation = operation
|
6
|
+
@queries = {}
|
7
|
+
|
8
|
+
if block.arity == 1
|
9
|
+
block.call(self)
|
10
|
+
else
|
11
|
+
self.non_instrumented { block.call }
|
12
|
+
self.instrumented { block.call }
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
def prepare(&block)
|
17
|
+
@queries[:prepare] = block
|
18
|
+
end
|
19
|
+
|
20
|
+
def non_instrumented(&block)
|
21
|
+
@queries[:non_instrumented] = block
|
22
|
+
end
|
23
|
+
|
24
|
+
def instrumented(&block)
|
25
|
+
@queries[:instrumented] = block
|
26
|
+
end
|
27
|
+
|
28
|
+
def call_and_remember_result(which)
|
29
|
+
raise "Fatal: #{which} query unspecified" unless @queries[which]
|
30
|
+
@result = @queries[which].call(@operation)
|
31
|
+
rescue Exception => e
|
32
|
+
@exception = e
|
33
|
+
end
|
34
|
+
|
35
|
+
def failed?
|
36
|
+
!!@exception
|
37
|
+
end
|
38
|
+
|
39
|
+
def result
|
40
|
+
failed? ? (raise @exception) : @result
|
41
|
+
end
|
42
|
+
end
|
@@ -0,0 +1,85 @@
|
|
1
|
+
class Promiscuous::Publisher::Operation::Transaction < Promiscuous::Publisher::Operation::Base
|
2
|
+
attr_accessor :transaction_id, :transaction_operations, :operation_payloads
|
3
|
+
|
4
|
+
def initialize(options={})
|
5
|
+
super
|
6
|
+
@operation = :commit
|
7
|
+
@transaction_operations = options[:transaction_operations].to_a
|
8
|
+
@transaction_id = options[:transaction_id]
|
9
|
+
@operation_payloads = options[:operation_payloads]
|
10
|
+
end
|
11
|
+
|
12
|
+
def dependency_for_op_lock
|
13
|
+
# We don't take locks on rows as the database already have the locks on them
|
14
|
+
# until the transaction is committed.
|
15
|
+
# A lock on the transaction ID is taken so we know when we conflict with
|
16
|
+
# the recovery mechanism.
|
17
|
+
Promiscuous::Dependency.new("__transactions__", self.transaction_id, :dont_hash => true)
|
18
|
+
end
|
19
|
+
|
20
|
+
def pending_writes
|
21
|
+
# TODO (performance) Return a list of writes that:
|
22
|
+
# - Never touch the same id (the latest write is sufficient)
|
23
|
+
# - create/update and then delete should be invisible
|
24
|
+
@transaction_operations
|
25
|
+
end
|
26
|
+
|
27
|
+
def query_dependencies
|
28
|
+
@query_dependencies ||= pending_writes.map(&:query_dependencies).flatten
|
29
|
+
end
|
30
|
+
|
31
|
+
def operation_payloads
|
32
|
+
@operation_payloads ||= pending_writes.map(&:operation_payloads).flatten
|
33
|
+
end
|
34
|
+
|
35
|
+
alias cache_operation_payloads operation_payloads
|
36
|
+
|
37
|
+
def should_instrument_query?
|
38
|
+
super && !pending_writes.empty?
|
39
|
+
end
|
40
|
+
|
41
|
+
def execute_instrumented(query)
|
42
|
+
unless self.recovering?
|
43
|
+
generate_read_dependencies
|
44
|
+
acquire_op_lock
|
45
|
+
|
46
|
+
# As opposed to atomic operations, we know the values of the instances
|
47
|
+
# before the database operation, and not after, so only one stage
|
48
|
+
# of recovery is used.
|
49
|
+
cache_operation_payloads
|
50
|
+
|
51
|
+
query.call_and_remember_result(:prepare)
|
52
|
+
end
|
53
|
+
|
54
|
+
self.increment_read_and_write_dependencies
|
55
|
+
|
56
|
+
query.call_and_remember_result(:instrumented)
|
57
|
+
|
58
|
+
# We can't do anything if the prepared commit doesn't go through.
|
59
|
+
# Either it's a network failure, or the database is having some real
|
60
|
+
# difficulties. The recovery mechanism will have to retry the transaction.
|
61
|
+
return if query.failed?
|
62
|
+
|
63
|
+
# We take a timestamp right after the write is performed because latency
|
64
|
+
# measurements are performed on the subscriber.
|
65
|
+
record_timestamp
|
66
|
+
|
67
|
+
ensure_op_still_locked
|
68
|
+
|
69
|
+
generate_payload
|
70
|
+
clear_previous_dependencies
|
71
|
+
|
72
|
+
publish_payload_in_redis
|
73
|
+
release_op_lock
|
74
|
+
publish_payload_in_rabbitmq_async
|
75
|
+
end
|
76
|
+
|
77
|
+
def recovery_payload
|
78
|
+
# TODO just save the table/ids, or publish the real payload directly.
|
79
|
+
[@transaction_id, @operation_payloads]
|
80
|
+
end
|
81
|
+
|
82
|
+
def self.recover_operation(transaction_id, operation_payloads)
|
83
|
+
new(:transaction_id => transaction_id, :operation_payloads => operation_payloads)
|
84
|
+
end
|
85
|
+
end
|
@@ -1,11 +1,10 @@
|
|
1
1
|
class Promiscuous::Publisher::Worker
|
2
2
|
def initialize
|
3
|
-
@recovery_timer = Promiscuous::Timer.new
|
4
|
-
@timeout = Promiscuous::Config.recovery_timeout
|
3
|
+
@recovery_timer = Promiscuous::Timer.new("recovery", Promiscuous::Config.recovery_timeout) { try_recover }
|
5
4
|
end
|
6
5
|
|
7
6
|
def start
|
8
|
-
@recovery_timer.
|
7
|
+
@recovery_timer.start(:run_immediately => true)
|
9
8
|
end
|
10
9
|
|
11
10
|
def stop
|
@@ -13,10 +12,11 @@ class Promiscuous::Publisher::Worker
|
|
13
12
|
end
|
14
13
|
|
15
14
|
def try_recover
|
16
|
-
Promiscuous::Publisher::Operation::Base.
|
17
|
-
Promiscuous::Publisher::Operation::Base.recover_payloads_for_rabbitmq
|
15
|
+
Promiscuous::Publisher::Operation::Base.run_recovery_mechanisms
|
18
16
|
rescue Exception => e
|
19
|
-
Promiscuous.warn "[recovery] #{e}
|
20
|
-
Promiscuous::Config.error_notifier.
|
17
|
+
Promiscuous.warn "[recovery] #{e}\n#{e.backtrace.join("\n")}"
|
18
|
+
Promiscuous::Config.error_notifier.call(e)
|
19
|
+
ensure
|
20
|
+
ActiveRecord::Base.clear_active_connections! if defined?(ActiveRecord::Base)
|
21
21
|
end
|
22
22
|
end
|