promiscuous 0.90.0 → 0.91.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (65) hide show
  1. checksums.yaml +7 -0
  2. data/lib/promiscuous/amqp/bunny.rb +63 -36
  3. data/lib/promiscuous/amqp/fake.rb +3 -1
  4. data/lib/promiscuous/amqp/hot_bunnies.rb +26 -16
  5. data/lib/promiscuous/amqp/null.rb +1 -0
  6. data/lib/promiscuous/amqp.rb +12 -12
  7. data/lib/promiscuous/cli.rb +70 -29
  8. data/lib/promiscuous/config.rb +54 -29
  9. data/lib/promiscuous/convenience.rb +1 -1
  10. data/lib/promiscuous/dependency.rb +25 -6
  11. data/lib/promiscuous/error/connection.rb +11 -9
  12. data/lib/promiscuous/error/dependency.rb +8 -1
  13. data/lib/promiscuous/loader.rb +4 -2
  14. data/lib/promiscuous/publisher/bootstrap/connection.rb +25 -0
  15. data/lib/promiscuous/publisher/bootstrap/data.rb +127 -0
  16. data/lib/promiscuous/publisher/bootstrap/mode.rb +19 -0
  17. data/lib/promiscuous/publisher/bootstrap/status.rb +40 -0
  18. data/lib/promiscuous/publisher/bootstrap/version.rb +46 -0
  19. data/lib/promiscuous/publisher/bootstrap.rb +27 -0
  20. data/lib/promiscuous/publisher/context/base.rb +67 -0
  21. data/lib/promiscuous/{middleware.rb → publisher/context/middleware.rb} +16 -13
  22. data/lib/promiscuous/publisher/context/transaction.rb +36 -0
  23. data/lib/promiscuous/publisher/context.rb +4 -88
  24. data/lib/promiscuous/publisher/mock_generator.rb +9 -9
  25. data/lib/promiscuous/publisher/model/active_record.rb +7 -7
  26. data/lib/promiscuous/publisher/model/base.rb +29 -29
  27. data/lib/promiscuous/publisher/model/ephemeral.rb +5 -3
  28. data/lib/promiscuous/publisher/model/mock.rb +9 -5
  29. data/lib/promiscuous/publisher/model/mongoid.rb +5 -22
  30. data/lib/promiscuous/publisher/operation/active_record.rb +360 -0
  31. data/lib/promiscuous/publisher/operation/atomic.rb +167 -0
  32. data/lib/promiscuous/publisher/operation/base.rb +279 -474
  33. data/lib/promiscuous/publisher/operation/mongoid.rb +153 -145
  34. data/lib/promiscuous/publisher/operation/non_persistent.rb +28 -0
  35. data/lib/promiscuous/publisher/operation/proxy_for_query.rb +42 -0
  36. data/lib/promiscuous/publisher/operation/transaction.rb +85 -0
  37. data/lib/promiscuous/publisher/operation.rb +1 -1
  38. data/lib/promiscuous/publisher/worker.rb +7 -7
  39. data/lib/promiscuous/publisher.rb +1 -1
  40. data/lib/promiscuous/railtie.rb +20 -5
  41. data/lib/promiscuous/redis.rb +104 -56
  42. data/lib/promiscuous/subscriber/message_processor/base.rb +38 -0
  43. data/lib/promiscuous/subscriber/message_processor/bootstrap.rb +17 -0
  44. data/lib/promiscuous/subscriber/message_processor/regular.rb +192 -0
  45. data/lib/promiscuous/subscriber/message_processor.rb +4 -0
  46. data/lib/promiscuous/subscriber/model/base.rb +20 -15
  47. data/lib/promiscuous/subscriber/model/mongoid.rb +4 -4
  48. data/lib/promiscuous/subscriber/model/observer.rb +16 -2
  49. data/lib/promiscuous/subscriber/operation/base.rb +68 -0
  50. data/lib/promiscuous/subscriber/operation/bootstrap.rb +54 -0
  51. data/lib/promiscuous/subscriber/operation/regular.rb +13 -0
  52. data/lib/promiscuous/subscriber/operation.rb +3 -166
  53. data/lib/promiscuous/subscriber/worker/message.rb +61 -35
  54. data/lib/promiscuous/subscriber/worker/message_synchronizer.rb +90 -59
  55. data/lib/promiscuous/subscriber/worker/pump.rb +17 -5
  56. data/lib/promiscuous/subscriber/worker/recorder.rb +4 -1
  57. data/lib/promiscuous/subscriber/worker/runner.rb +49 -9
  58. data/lib/promiscuous/subscriber/worker/stats.rb +2 -2
  59. data/lib/promiscuous/subscriber/worker.rb +6 -0
  60. data/lib/promiscuous/subscriber.rb +1 -1
  61. data/lib/promiscuous/timer.rb +31 -18
  62. data/lib/promiscuous/version.rb +1 -1
  63. data/lib/promiscuous.rb +23 -3
  64. metadata +104 -89
  65. data/lib/promiscuous/subscriber/payload.rb +0 -34
@@ -4,7 +4,7 @@ raise "moped > 1.3.2 please" unless Gem.loaded_specs['moped'].version >= Ge
4
4
  require 'yaml'
5
5
 
6
6
  class Moped::PromiscuousCollectionWrapper < Moped::Collection
7
- class PromiscuousCollectionOperation < Promiscuous::Publisher::Operation::Base
7
+ class PromiscuousCollectionOperation < Promiscuous::Publisher::Operation::Atomic
8
8
  def initialize(options={})
9
9
  super
10
10
  @operation = :create
@@ -21,13 +21,13 @@ class Moped::PromiscuousCollectionWrapper < Moped::Collection
21
21
  rescue NameError
22
22
  end
23
23
 
24
- def serialize_document_for_create_recovery
25
- # TODO the serialization/deserialization is not very nice, but we need
26
- # the bson types.
27
- @document.to_yaml
24
+ def recovery_payload
25
+ # We use yaml because we need the BSON types.
26
+ [@instance.class.promiscuous_collection_name, @instance.id, @document.to_yaml]
28
27
  end
29
28
 
30
- def self.recover_operation(model, instance_id, document)
29
+ def self.recover_operation(collection, instance_id, document)
30
+ model = Promiscuous::Publisher::Model::Mongoid.collection_mapping[collection]
31
31
  document = YAML.load(document)
32
32
  instance = Mongoid::Factory.from_db(model, document)
33
33
  new(:collection => model.collection, :document => document, :instance => instance)
@@ -40,18 +40,17 @@ class Moped::PromiscuousCollectionWrapper < Moped::Collection
40
40
  end
41
41
  end
42
42
 
43
- def stash_version_in_write_query
44
- @document[VERSION_FIELD] = @instance_version
45
- end
46
-
47
- def execute_persistent(&db_operation)
43
+ def execute_instrumented(query)
48
44
  @instance = Mongoid::Factory.from_db(model, @document)
49
45
  super
50
46
  end
51
47
 
52
- def execute(&db_operation)
53
- return db_operation.call unless model
54
- super
48
+ def should_instrument_query?
49
+ super && model
50
+ end
51
+
52
+ def recoverable_failure?(exception)
53
+ exception.is_a?(Moped::Errors::ConnectionFailure)
55
54
  end
56
55
  end
57
56
 
@@ -74,8 +73,39 @@ class Moped::PromiscuousCollectionWrapper < Moped::Collection
74
73
  end
75
74
 
76
75
  class Moped::PromiscuousQueryWrapper < Moped::Query
77
- class PromiscuousQueryOperation < Promiscuous::Publisher::Operation::Base
78
- attr_accessor :raw_instance, :new_raw_instance, :change
76
+ module PromiscuousHelpers
77
+ def collection_name
78
+ @collection_name ||= @query.collection.is_a?(String) ? @query.collection : @query.collection.name
79
+ end
80
+
81
+ def model
82
+ @model ||= Promiscuous::Publisher::Model::Mongoid.collection_mapping[collection_name]
83
+ end
84
+
85
+ def get_selector_instance
86
+ selector = @query.operation.selector["$query"] || @query.operation.selector
87
+
88
+ # TODO use the original instance for an update/delete, that would be
89
+ # an even better hint.
90
+
91
+ # We only support == selectors, no $in, or $gt.
92
+ @selector = selector.select { |k,v| k.to_s =~ /^[^$]/ && !v.is_a?(Hash) }
93
+
94
+ # @instance is not really a proper instance of a model, it's just a
95
+ # convenient representation of a selector as explain in base.rb,
96
+ # which explain why we don't want any constructor to be called.
97
+ # Note that this optimistic mechanism also works with writes because
98
+ # the instance gets reloaded once the lock is taken. If the
99
+ # dependencies were incorrect, the locks will be released and
100
+ # reacquired appropriately.
101
+ model.allocate.tap { |doc| doc.instance_variable_set(:@attributes, @selector) }
102
+ end
103
+ end
104
+
105
+ class PromiscuousWriteOperation < Promiscuous::Publisher::Operation::Atomic
106
+ include Moped::PromiscuousQueryWrapper::PromiscuousHelpers
107
+
108
+ attr_accessor :change
79
109
 
80
110
  def initialize(options={})
81
111
  super
@@ -83,87 +113,60 @@ class Moped::PromiscuousQueryWrapper < Moped::Query
83
113
  @change = options[:change]
84
114
  end
85
115
 
86
- def collection_name
87
- @collection_name ||= @query.collection.is_a?(String) ? @query.collection : @query.collection.name
116
+ def recovery_payload
117
+ [@instance.class.promiscuous_collection_name, @instance.id]
88
118
  end
89
119
 
90
- def model
91
- @model ||= Promiscuous::Publisher::Model::Mongoid.collection_mapping[collection_name]
120
+ def self.recover_operation(collection, instance_id)
121
+ # TODO We need to use the primary database. We cannot read from a secondary.
122
+ model = Promiscuous::Publisher::Model::Mongoid.collection_mapping[collection]
123
+ query = model.unscoped.where(:id => instance_id).query
124
+
125
+ # We no-op the update operation instead of making it idempotent.
126
+ # To do so, we do a dummy update on the document.
127
+ # The original caller will fail because the lock was unlocked, so we'll
128
+ # won't send a different message.
129
+ new(:query => query, :change => {}).tap { |op| op.instance_eval { reload_instance } }
92
130
  end
93
131
 
94
- def self.recover_operation(model, instance_id, document)
95
- # TODO We need to use the primary database. We cannot read from a
96
- # secondary.
97
- query = model.unscoped.where(:id => instance_id).query
98
- op = new(:query => query, :change => {})
99
- # TODO refactor this not so pretty instance_eval
100
- op.instance_eval do
101
- reload_instance
102
- @instance ||= get_selector_instance
132
+ def recover_db_operation
133
+ if operation == :update
134
+ without_promiscuous { @query.update(@change) }
135
+ else
136
+ without_promiscuous { @query.remove }
103
137
  end
104
- op
105
138
  end
106
139
 
107
- def recover_db_operation
108
- # We no-op the update/destroy operation instead of making it idempotent.
109
- # The original caller will fail because the lock was unlocked.
110
- without_promiscuous { @query.update(@change) }
111
- @operation = :dummy
140
+ def recoverable_failure?(exception)
141
+ exception.is_a?(Moped::Errors::ConnectionFailure)
112
142
  end
113
143
 
114
144
  def fetch_instance
115
- @raw_instance = @new_raw_instance || without_promiscuous { @query.first }
116
- Mongoid::Factory.from_db(model, @raw_instance) if @raw_instance
145
+ raw_instance = without_promiscuous { @query.first }
146
+ Mongoid::Factory.from_db(model, raw_instance) if raw_instance
117
147
  end
118
148
 
119
149
  def use_id_selector(options={})
120
- selector = {'_id' => @instance.id}
150
+ selector = {'_id' => @instance.id}.merge(@query.selector.select { |k,v| k.to_s.include?("_id") })
121
151
 
122
152
  if options[:use_atomic_version_selector]
123
- version = @instance[VERSION_FIELD]
124
- selector.merge!(VERSION_FIELD => version) if version
153
+ version = @instance[Promiscuous::Config.version_field]
154
+ selector.merge!(Promiscuous::Config.version_field => version)
125
155
  end
126
156
 
127
157
  @query.selector = selector
128
158
  end
129
159
 
130
- def stash_version_in_write_query
160
+ def stash_version_in_document(version)
131
161
  @change['$set'] ||= {}
132
- @change['$set'][VERSION_FIELD] = @instance_version
162
+ @change['$set'][Promiscuous::Config.version_field] = version
133
163
  end
134
164
 
135
- def get_selector_instance
136
- selector = @query.operation.selector["$query"] || @query.operation.selector
137
-
138
- # TODO use the original instance for an update/delete, that would be
139
- # an even better hint.
140
-
141
- # We only support == selectors, no $in, or $gt.
142
- @selector = selector.select { |k,v| k.to_s =~ /^[^$]/ && !v.is_a?(Hash) }
143
-
144
- # @instance is not really a proper instance of a model, it's just a
145
- # convenient representation of a selector as explain in base.rb,
146
- # which explain why we don't want any constructor to be called.
147
- # Note that this optimistic mechanism also works with writes because
148
- # the instance gets reloaded once the lock is taken. If the
149
- # dependencies were incorrect, the locks will be released and
150
- # reacquired appropriately.
151
- model.allocate.tap { |doc| doc.instance_variable_set(:@attributes, @selector) }
152
- end
153
-
154
- def execute_persistent(&db_operation)
165
+ def execute_instrumented(query)
155
166
  # We are trying to be optimistic for the locking. We are trying to figure
156
167
  # out our dependencies with the selector upfront to avoid an extra read
157
168
  # from reload_instance.
158
- @instance = get_selector_instance
159
- super
160
- end
161
-
162
- def execute_non_persistent(&db_operation)
163
- if multi?
164
- @instance = get_selector_instance
165
- @selector_keys = @selector.keys
166
- end
169
+ @instance ||= get_selector_instance
167
170
  super
168
171
  end
169
172
 
@@ -185,48 +188,55 @@ class Moped::PromiscuousQueryWrapper < Moped::Query
185
188
  # TODO maybe we should cache these things
186
189
  # TODO discover field dependencies automatically (hard)
187
190
  aliases = Hash[model.aliased_fields.map { |k,v| [v,k] }]
188
- attributes = fields_in_query(@change).map { |f| (aliases[f.to_s] || f).to_sym }
191
+ attributes = fields_in_query(@change).map { |f| [aliases[f.to_s], f] }.flatten.compact.map(&:to_sym)
189
192
  (attributes & model.published_db_fields).present?
190
193
  end
191
194
 
192
- def execute(&db_operation)
193
- return db_operation.call if @query.without_promiscuous?
194
- return db_operation.call unless model
195
- return db_operation.call unless any_published_field_changed?
195
+ def should_instrument_query?
196
+ super && model && any_published_field_changed?
197
+ end
198
+ end
196
199
 
197
- # We cannot do multi update/destroy
198
- if (operation == :update || operation == :destroy) && multi?
199
- raise Promiscuous::Error::Dependency.new(:operation => self)
200
- end
200
+ class PromiscuousReadOperation < Promiscuous::Publisher::Operation::NonPersistent
201
+ include Moped::PromiscuousQueryWrapper::PromiscuousHelpers
202
+
203
+ def initialize(options={})
201
204
  super
205
+ @operation = :read
206
+ @query = options[:query]
207
+ end
208
+
209
+ def query_dependencies
210
+ deps = dependencies_for(get_selector_instance)
211
+ deps.empty? ? super : deps
202
212
  end
203
- end
204
213
 
205
- def promiscuous_operation(operation, options={})
206
- PromiscuousQueryOperation.new(options.merge(:query => self, :operation => operation))
214
+ def should_instrument_query?
215
+ super && model
216
+ end
207
217
  end
208
218
 
209
- def selector=(value)
210
- @selector = value
211
- @operation.selector = value
219
+ def promiscuous_read_operation(options={})
220
+ PromiscuousReadOperation.new(options.merge(:query => self))
212
221
  end
213
222
 
214
- def without_promiscuous!
215
- @without_promiscuous = true
223
+ def promiscuous_write_operation(operation, options={})
224
+ PromiscuousWriteOperation.new(options.merge(:query => self, :operation => operation))
216
225
  end
217
226
 
218
- def without_promiscuous?
219
- !!@without_promiscuous
227
+ def selector=(value)
228
+ @selector = value
229
+ @operation.selector = value
220
230
  end
221
231
 
222
232
  # Moped::Query
223
233
 
224
234
  def count(*args)
225
- promiscuous_operation(:read, :multi => true, :operation_ext => :count).execute { super }.to_i
235
+ promiscuous_read_operation(:operation_ext => :count).execute { super }.to_i
226
236
  end
227
237
 
228
238
  def distinct(key)
229
- promiscuous_operation(:read, :multi => true).execute { super }
239
+ promiscuous_read_operation(:operation_ext => :distinct).execute { super }
230
240
  end
231
241
 
232
242
  def each
@@ -240,68 +250,82 @@ class Moped::PromiscuousQueryWrapper < Moped::Query
240
250
  alias :cursor :each
241
251
 
242
252
  def first
243
- # TODO If the the user is using something like .only(), we need to make
244
- # sure that we add the id, otherwise we may not be able to perform the
245
- # dependency optimization by resolving the selector to an id.
246
- promiscuous_operation(:read).execute do |operation|
247
- operation ? operation.raw_instance : super
253
+ # FIXME If the the user is using something like .only(), we need to make
254
+ # sure that we add the id, otherwise we are screwed.
255
+ op = promiscuous_read_operation
256
+
257
+ op.execute do |query|
258
+ query.non_instrumented { super }
259
+ query.instrumented do
260
+ super.tap do |doc|
261
+ op.instances = doc ? [Mongoid::Factory.from_db(op.model, doc)] : []
262
+ end
263
+ end
248
264
  end
249
265
  end
250
266
  alias :one :first
251
267
 
252
268
  def update(change, flags=nil)
253
- multi = flags && flags.include?(:multi)
254
- raise "No upsert support yet" if flags && flags.include?(:upsert)
269
+ update_op = promiscuous_write_operation(:update, :change => change)
270
+
271
+ if flags && update_op.should_instrument_query?
272
+ raise "You cannot do a multi update. Instead, update each document separately." if flags.include?(:multi)
273
+ raise "No upsert support yet" if flags.include?(:upsert)
274
+ end
255
275
 
256
- promiscuous_operation(:update, :change => change, :multi => multi).execute do |operation|
257
- if operation
258
- operation.new_raw_instance = without_promiscuous { modify(change, :new => true) }
259
- # FIXME raise when recovery raced
276
+ update_op.execute do |query|
277
+ query.non_instrumented { super }
278
+ query.instrumented do |op|
279
+ raw_instance = without_promiscuous { modify(change, :new => true) }
280
+ op.instance = Mongoid::Factory.from_db(op.model, raw_instance)
260
281
  {'updatedExisting' => true, 'n' => 1, 'err' => nil, 'ok' => 1.0}
261
- else
262
- super
263
282
  end
264
283
  end
265
284
  end
266
285
 
267
286
  def modify(change, options={})
268
- promiscuous_operation(:update, :change => change).execute { super }
269
- # FIXME raise when recovery raced
287
+ promiscuous_write_operation(:update, :change => change).execute do |query|
288
+ query.non_instrumented { super }
289
+ query.instrumented do |op|
290
+ raise "You can only use find_and_modify() with :new => true" if !options[:new]
291
+ super.tap { |raw_instance| op.instance = Mongoid::Factory.from_db(op.model, raw_instance) }
292
+ end
293
+ end
270
294
  end
271
295
 
272
296
  def remove
273
- promiscuous_operation(:destroy).execute { super }
274
- # FIXME raise when recovery raced
297
+ promiscuous_write_operation(:destroy).execute { super }
275
298
  end
276
299
 
277
300
  def remove_all
278
- promiscuous_operation(:destroy, :multi => true).execute { super }
301
+ raise "Instead of doing a multi delete, delete each document separatly.\n" +
302
+ "Declare your has_many relationships with :dependent => :destroy instead of :delete"
279
303
  end
280
304
  end
281
305
 
282
306
  class Moped::PromiscuousCursorWrapper < Moped::Cursor
283
- def promiscuous_operation(op, options={})
284
- Moped::PromiscuousQueryWrapper::PromiscuousQueryOperation.new(
285
- options.merge(:query => @query, :operation => op))
286
- end
287
-
288
307
  # Moped::Cursor
289
-
290
- def fake_single_read(operation)
291
- @cursor_id = 0
292
- [operation.raw_instance].compact
308
+ def promiscuous_read_each(&block)
309
+ op = Moped::PromiscuousQueryWrapper::PromiscuousReadOperation.new(
310
+ :query => @query, :operation_ext => :each)
311
+
312
+ op.execute do |query|
313
+ query.non_instrumented { block.call.to_a }
314
+ query.instrumented do
315
+ block.call.to_a.tap do |docs|
316
+ op.instances = docs.map { |doc| Mongoid::Factory.from_db(op.model, doc) }
317
+ end
318
+ end
319
+ end
293
320
  end
294
321
 
295
322
  def load_docs
296
- should_fake_single_read = @limit == 1
297
- promiscuous_operation(:read, :multi => !should_fake_single_read).execute do |operation|
298
- operation && should_fake_single_read ? fake_single_read(operation) : super
299
- end.to_a
323
+ promiscuous_read_each { super }
300
324
  end
301
325
 
302
326
  def get_more
303
327
  # TODO support batch_size
304
- promiscuous_operation(:read, :multi => true).execute { super }
328
+ promiscuous_read_each { super }
305
329
  end
306
330
 
307
331
  def initialize(session, query_operation)
@@ -313,9 +337,8 @@ end
313
337
  class Moped::PromiscuousDatabase < Moped::Database
314
338
  # TODO it might be safer to use the alias attribute method because promiscuous
315
339
  # may come late in the loading.
316
- def promiscuous_operation(op, options={})
317
- Moped::PromiscuousQueryWrapper::PromiscuousQueryOperation.new(
318
- options.merge(:operation => op))
340
+ def promiscuous_read_operation(options={})
341
+ Moped::PromiscuousQueryWrapper::PromiscuousReadOperation.new(options)
319
342
  end
320
343
 
321
344
  # Moped::Database
@@ -323,29 +346,13 @@ class Moped::PromiscuousDatabase < Moped::Database
323
346
  def command(command)
324
347
  if command[:mapreduce]
325
348
  query = Moped::Query.new(self[command[:mapreduce]], command[:query])
326
- promiscuous_operation(:read, :query => query,
327
- :operation_ext => :mapreduce, :multi => true).execute { super }
349
+ promiscuous_read_operation(:query => query, :operation_ext => :mapreduce).execute { super }
328
350
  else
329
351
  super
330
352
  end
331
353
  end
332
354
  end
333
355
 
334
- class Mongoid::Contextual::Mongo
335
- alias_method :each_hijacked, :each
336
-
337
- def each(&block)
338
- query.without_promiscuous! if criteria.options[:without_promiscuous]
339
- each_hijacked(&block)
340
- end
341
- end
342
-
343
- module Origin::Optional
344
- def without_promiscuous
345
- clone.tap { |criteria| criteria.options.store(:without_promiscuous, true) }
346
- end
347
- end
348
-
349
356
  class Mongoid::Validations::UniquenessValidator
350
357
  alias_method :validate_root_without_promisucous, :validate_root
351
358
  def validate_root(*args)
@@ -354,7 +361,8 @@ class Mongoid::Validations::UniquenessValidator
354
361
  end
355
362
 
356
363
  class Moped::BSON::ObjectId
357
- # No {"$oid": "123"}, it's horrible
364
+ # No {"$oid": "123"}, it's horrible.
365
+ # TODO Document this shit.
358
366
  def to_json(*args)
359
367
  "\"#{to_s}\""
360
368
  end
@@ -0,0 +1,28 @@
1
+ class Promiscuous::Publisher::Operation::NonPersistent < Promiscuous::Publisher::Operation::Base
2
+ # XXX As opposed to atomic operations, NonPersistent operations deals with an
3
+ # array of instances
4
+ attr_accessor :instances
5
+
6
+ def initialize(options={})
7
+ super
8
+ @instances = options[:instances].to_a
9
+ end
10
+
11
+ def execute_instrumented(db_operation)
12
+ db_operation.call_and_remember_result(:instrumented)
13
+
14
+ unless db_operation.failed?
15
+ current_context.read_operations << self if read?
16
+ trace_operation
17
+ end
18
+ end
19
+
20
+ def operation_payloads
21
+ return [] if self.failed?
22
+ @instances.map { |instance| payloads_for(instance) }
23
+ end
24
+
25
+ def query_dependencies
26
+ @instances.map { |instance| dependencies_for(instance) }
27
+ end
28
+ end
@@ -0,0 +1,42 @@
1
+ class Promiscuous::Publisher::Operation::ProxyForQuery
2
+ attr_accessor :exception, :result
3
+
4
+ def initialize(operation, &block)
5
+ @operation = operation
6
+ @queries = {}
7
+
8
+ if block.arity == 1
9
+ block.call(self)
10
+ else
11
+ self.non_instrumented { block.call }
12
+ self.instrumented { block.call }
13
+ end
14
+ end
15
+
16
+ def prepare(&block)
17
+ @queries[:prepare] = block
18
+ end
19
+
20
+ def non_instrumented(&block)
21
+ @queries[:non_instrumented] = block
22
+ end
23
+
24
+ def instrumented(&block)
25
+ @queries[:instrumented] = block
26
+ end
27
+
28
+ def call_and_remember_result(which)
29
+ raise "Fatal: #{which} query unspecified" unless @queries[which]
30
+ @result = @queries[which].call(@operation)
31
+ rescue Exception => e
32
+ @exception = e
33
+ end
34
+
35
+ def failed?
36
+ !!@exception
37
+ end
38
+
39
+ def result
40
+ failed? ? (raise @exception) : @result
41
+ end
42
+ end
@@ -0,0 +1,85 @@
1
+ class Promiscuous::Publisher::Operation::Transaction < Promiscuous::Publisher::Operation::Base
2
+ attr_accessor :transaction_id, :transaction_operations, :operation_payloads
3
+
4
+ def initialize(options={})
5
+ super
6
+ @operation = :commit
7
+ @transaction_operations = options[:transaction_operations].to_a
8
+ @transaction_id = options[:transaction_id]
9
+ @operation_payloads = options[:operation_payloads]
10
+ end
11
+
12
+ def dependency_for_op_lock
13
+ # We don't take locks on rows as the database already have the locks on them
14
+ # until the transaction is committed.
15
+ # A lock on the transaction ID is taken so we know when we conflict with
16
+ # the recovery mechanism.
17
+ Promiscuous::Dependency.new("__transactions__", self.transaction_id, :dont_hash => true)
18
+ end
19
+
20
+ def pending_writes
21
+ # TODO (performance) Return a list of writes that:
22
+ # - Never touch the same id (the latest write is sufficient)
23
+ # - create/update and then delete should be invisible
24
+ @transaction_operations
25
+ end
26
+
27
+ def query_dependencies
28
+ @query_dependencies ||= pending_writes.map(&:query_dependencies).flatten
29
+ end
30
+
31
+ def operation_payloads
32
+ @operation_payloads ||= pending_writes.map(&:operation_payloads).flatten
33
+ end
34
+
35
+ alias cache_operation_payloads operation_payloads
36
+
37
+ def should_instrument_query?
38
+ super && !pending_writes.empty?
39
+ end
40
+
41
+ def execute_instrumented(query)
42
+ unless self.recovering?
43
+ generate_read_dependencies
44
+ acquire_op_lock
45
+
46
+ # As opposed to atomic operations, we know the values of the instances
47
+ # before the database operation, and not after, so only one stage
48
+ # of recovery is used.
49
+ cache_operation_payloads
50
+
51
+ query.call_and_remember_result(:prepare)
52
+ end
53
+
54
+ self.increment_read_and_write_dependencies
55
+
56
+ query.call_and_remember_result(:instrumented)
57
+
58
+ # We can't do anything if the prepared commit doesn't go through.
59
+ # Either it's a network failure, or the database is having some real
60
+ # difficulties. The recovery mechanism will have to retry the transaction.
61
+ return if query.failed?
62
+
63
+ # We take a timestamp right after the write is performed because latency
64
+ # measurements are performed on the subscriber.
65
+ record_timestamp
66
+
67
+ ensure_op_still_locked
68
+
69
+ generate_payload
70
+ clear_previous_dependencies
71
+
72
+ publish_payload_in_redis
73
+ release_op_lock
74
+ publish_payload_in_rabbitmq_async
75
+ end
76
+
77
+ def recovery_payload
78
+ # TODO just save the table/ids, or publish the real payload directly.
79
+ [@transaction_id, @operation_payloads]
80
+ end
81
+
82
+ def self.recover_operation(transaction_id, operation_payloads)
83
+ new(:transaction_id => transaction_id, :operation_payloads => operation_payloads)
84
+ end
85
+ end
@@ -1,4 +1,4 @@
1
1
  module Promiscuous::Publisher::Operation
2
2
  extend Promiscuous::Autoload
3
- autoload :Base
3
+ autoload :Base, :Transaction, :Atomic, :NonPersistent, :ProxyForQuery
4
4
  end
@@ -1,11 +1,10 @@
1
1
  class Promiscuous::Publisher::Worker
2
2
  def initialize
3
- @recovery_timer = Promiscuous::Timer.new
4
- @timeout = Promiscuous::Config.recovery_timeout
3
+ @recovery_timer = Promiscuous::Timer.new("recovery", Promiscuous::Config.recovery_timeout) { try_recover }
5
4
  end
6
5
 
7
6
  def start
8
- @recovery_timer.run_every(@timeout, :run_immediately => true) { try_recover }
7
+ @recovery_timer.start(:run_immediately => true)
9
8
  end
10
9
 
11
10
  def stop
@@ -13,10 +12,11 @@ class Promiscuous::Publisher::Worker
13
12
  end
14
13
 
15
14
  def try_recover
16
- Promiscuous::Publisher::Operation::Base.recover_locks
17
- Promiscuous::Publisher::Operation::Base.recover_payloads_for_rabbitmq
15
+ Promiscuous::Publisher::Operation::Base.run_recovery_mechanisms
18
16
  rescue Exception => e
19
- Promiscuous.warn "[recovery] #{e} #{e.backtrace.join("\n")}"
20
- Promiscuous::Config.error_notifier.try(:call, e)
17
+ Promiscuous.warn "[recovery] #{e}\n#{e.backtrace.join("\n")}"
18
+ Promiscuous::Config.error_notifier.call(e)
19
+ ensure
20
+ ActiveRecord::Base.clear_active_connections! if defined?(ActiveRecord::Base)
21
21
  end
22
22
  end
@@ -1,6 +1,6 @@
1
1
  module Promiscuous::Publisher
2
2
  extend Promiscuous::Autoload
3
- autoload :Model, :Operation, :MockGenerator, :Context, :Worker
3
+ autoload :Model, :Operation, :MockGenerator, :Context, :Worker, :Bootstrap
4
4
 
5
5
  extend ActiveSupport::Concern
6
6