promiscuous 0.53.1 → 0.90.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (106) hide show
  1. data/lib/promiscuous.rb +25 -28
  2. data/lib/promiscuous/amqp.rb +27 -8
  3. data/lib/promiscuous/amqp/bunny.rb +131 -16
  4. data/lib/promiscuous/amqp/fake.rb +52 -0
  5. data/lib/promiscuous/amqp/hot_bunnies.rb +56 -0
  6. data/lib/promiscuous/amqp/null.rb +6 -6
  7. data/lib/promiscuous/cli.rb +108 -24
  8. data/lib/promiscuous/config.rb +73 -12
  9. data/lib/promiscuous/convenience.rb +18 -0
  10. data/lib/promiscuous/dependency.rb +59 -0
  11. data/lib/promiscuous/dsl.rb +36 -0
  12. data/lib/promiscuous/error.rb +3 -1
  13. data/lib/promiscuous/error/already_processed.rb +5 -0
  14. data/lib/promiscuous/error/base.rb +1 -0
  15. data/lib/promiscuous/error/connection.rb +7 -5
  16. data/lib/promiscuous/error/dependency.rb +111 -0
  17. data/lib/promiscuous/error/lock_unavailable.rb +12 -0
  18. data/lib/promiscuous/error/lost_lock.rb +12 -0
  19. data/lib/promiscuous/error/missing_context.rb +29 -0
  20. data/lib/promiscuous/error/publisher.rb +5 -15
  21. data/lib/promiscuous/error/recovery.rb +7 -0
  22. data/lib/promiscuous/error/subscriber.rb +2 -4
  23. data/lib/promiscuous/key.rb +36 -0
  24. data/lib/promiscuous/loader.rb +12 -16
  25. data/lib/promiscuous/middleware.rb +112 -0
  26. data/lib/promiscuous/publisher.rb +7 -4
  27. data/lib/promiscuous/publisher/context.rb +92 -0
  28. data/lib/promiscuous/publisher/mock_generator.rb +72 -0
  29. data/lib/promiscuous/publisher/model.rb +3 -86
  30. data/lib/promiscuous/publisher/model/active_record.rb +8 -15
  31. data/lib/promiscuous/publisher/model/base.rb +136 -0
  32. data/lib/promiscuous/publisher/model/ephemeral.rb +69 -0
  33. data/lib/promiscuous/publisher/model/mock.rb +61 -0
  34. data/lib/promiscuous/publisher/model/mongoid.rb +57 -100
  35. data/lib/promiscuous/{common/lint.rb → publisher/operation.rb} +1 -1
  36. data/lib/promiscuous/publisher/operation/base.rb +707 -0
  37. data/lib/promiscuous/publisher/operation/mongoid.rb +370 -0
  38. data/lib/promiscuous/publisher/worker.rb +22 -0
  39. data/lib/promiscuous/railtie.rb +21 -3
  40. data/lib/promiscuous/redis.rb +132 -40
  41. data/lib/promiscuous/resque.rb +12 -0
  42. data/lib/promiscuous/sidekiq.rb +15 -0
  43. data/lib/promiscuous/subscriber.rb +9 -20
  44. data/lib/promiscuous/subscriber/model.rb +4 -104
  45. data/lib/promiscuous/subscriber/model/active_record.rb +10 -0
  46. data/lib/promiscuous/subscriber/model/base.rb +96 -0
  47. data/lib/promiscuous/subscriber/model/mongoid.rb +86 -0
  48. data/lib/promiscuous/subscriber/model/observer.rb +37 -0
  49. data/lib/promiscuous/subscriber/operation.rb +167 -0
  50. data/lib/promiscuous/subscriber/payload.rb +34 -0
  51. data/lib/promiscuous/subscriber/worker.rb +22 -18
  52. data/lib/promiscuous/subscriber/worker/message.rb +48 -25
  53. data/lib/promiscuous/subscriber/worker/message_synchronizer.rb +273 -181
  54. data/lib/promiscuous/subscriber/worker/pump.rb +17 -43
  55. data/lib/promiscuous/subscriber/worker/recorder.rb +24 -0
  56. data/lib/promiscuous/subscriber/worker/runner.rb +24 -3
  57. data/lib/promiscuous/subscriber/worker/stats.rb +62 -0
  58. data/lib/promiscuous/timer.rb +38 -0
  59. data/lib/promiscuous/version.rb +1 -1
  60. metadata +98 -143
  61. data/README.md +0 -33
  62. data/lib/promiscuous/amqp/ruby_amqp.rb +0 -140
  63. data/lib/promiscuous/common.rb +0 -4
  64. data/lib/promiscuous/common/class_helpers.rb +0 -12
  65. data/lib/promiscuous/common/lint/base.rb +0 -24
  66. data/lib/promiscuous/common/options.rb +0 -51
  67. data/lib/promiscuous/ephemeral.rb +0 -14
  68. data/lib/promiscuous/error/recover.rb +0 -1
  69. data/lib/promiscuous/observer.rb +0 -5
  70. data/lib/promiscuous/publisher/active_record.rb +0 -7
  71. data/lib/promiscuous/publisher/amqp.rb +0 -18
  72. data/lib/promiscuous/publisher/attributes.rb +0 -32
  73. data/lib/promiscuous/publisher/base.rb +0 -23
  74. data/lib/promiscuous/publisher/class.rb +0 -36
  75. data/lib/promiscuous/publisher/envelope.rb +0 -7
  76. data/lib/promiscuous/publisher/ephemeral.rb +0 -9
  77. data/lib/promiscuous/publisher/lint.rb +0 -35
  78. data/lib/promiscuous/publisher/lint/amqp.rb +0 -14
  79. data/lib/promiscuous/publisher/lint/attributes.rb +0 -12
  80. data/lib/promiscuous/publisher/lint/base.rb +0 -5
  81. data/lib/promiscuous/publisher/lint/class.rb +0 -15
  82. data/lib/promiscuous/publisher/lint/polymorphic.rb +0 -22
  83. data/lib/promiscuous/publisher/mock.rb +0 -79
  84. data/lib/promiscuous/publisher/mongoid.rb +0 -33
  85. data/lib/promiscuous/publisher/mongoid/embedded.rb +0 -27
  86. data/lib/promiscuous/publisher/mongoid/embedded_many.rb +0 -12
  87. data/lib/promiscuous/publisher/polymorphic.rb +0 -8
  88. data/lib/promiscuous/subscriber/active_record.rb +0 -11
  89. data/lib/promiscuous/subscriber/amqp.rb +0 -25
  90. data/lib/promiscuous/subscriber/attributes.rb +0 -35
  91. data/lib/promiscuous/subscriber/base.rb +0 -29
  92. data/lib/promiscuous/subscriber/class.rb +0 -29
  93. data/lib/promiscuous/subscriber/dummy.rb +0 -19
  94. data/lib/promiscuous/subscriber/envelope.rb +0 -18
  95. data/lib/promiscuous/subscriber/lint.rb +0 -30
  96. data/lib/promiscuous/subscriber/lint/amqp.rb +0 -21
  97. data/lib/promiscuous/subscriber/lint/attributes.rb +0 -21
  98. data/lib/promiscuous/subscriber/lint/base.rb +0 -14
  99. data/lib/promiscuous/subscriber/lint/class.rb +0 -13
  100. data/lib/promiscuous/subscriber/lint/polymorphic.rb +0 -39
  101. data/lib/promiscuous/subscriber/mongoid.rb +0 -27
  102. data/lib/promiscuous/subscriber/mongoid/embedded.rb +0 -17
  103. data/lib/promiscuous/subscriber/mongoid/embedded_many.rb +0 -44
  104. data/lib/promiscuous/subscriber/observer.rb +0 -26
  105. data/lib/promiscuous/subscriber/polymorphic.rb +0 -36
  106. data/lib/promiscuous/subscriber/upsert.rb +0 -12
@@ -1,4 +1,4 @@
1
- module Promiscuous::Common::Lint
1
+ module Promiscuous::Publisher::Operation
2
2
  extend Promiscuous::Autoload
3
3
  autoload :Base
4
4
  end
@@ -0,0 +1,707 @@
1
+ class Promiscuous::Publisher::Operation::Base
2
+ class TryAgain < RuntimeError; end
3
+ VERSION_FIELD = '_pv'
4
+
5
+ attr_accessor :operation, :operation_ext, :instance, :selector_keys
6
+
7
+ def initialize(options={})
8
+ # XXX instance is not always an instance, it can be a selector
9
+ # representation.
10
+ @instance = options[:instance]
11
+ @operation = options[:operation]
12
+ @operation_ext = options[:operation_ext]
13
+ @multi = options[:multi]
14
+ end
15
+
16
+ def read?
17
+ operation == :read
18
+ end
19
+
20
+ def write?
21
+ !read?
22
+ end
23
+
24
+ def multi?
25
+ !!@multi
26
+ end
27
+
28
+ def single?
29
+ !@multi
30
+ end
31
+
32
+ def persists?
33
+ # TODO For writes in transactions, it should be false
34
+ write?
35
+ end
36
+
37
+ def failed?
38
+ !!@exception
39
+ end
40
+
41
+ def current_context
42
+ @current_context ||= Promiscuous::Publisher::Context.current
43
+ end
44
+
45
+ def record_timestamp
46
+ # Records the number of milliseconds since epoch, which we use send sending
47
+ # the payload over. It's good for latency measurements.
48
+ time = Time.now
49
+ @timestamp = time.to_i * 1000 + time.usec / 1000
50
+ end
51
+
52
+ def self.rabbitmq_staging_set_key
53
+ Promiscuous::Key.new(:pub).join('rabbitmq_staging').to_s
54
+ end
55
+
56
+ delegate :rabbitmq_staging_set_key, :to => self
57
+
58
+ def on_rabbitmq_confirm
59
+ # These requests could be parallelized, rabbitmq persisted the operation.
60
+ # XXX TODO
61
+ # Promiscuous::Redis.slave.del(@payload_recovery_key) if Promiscuous::Redis.slave
62
+
63
+ @payload_recovery_node.multi do
64
+ @payload_recovery_node.del(@payload_recovery_key)
65
+ @payload_recovery_node.zrem(rabbitmq_staging_set_key, @payload_recovery_key)
66
+ end
67
+ end
68
+
69
+ def publish_payload_in_rabbitmq_async
70
+ Promiscuous::AMQP.publish(:key => @amqp_key, :payload => @payload,
71
+ :on_confirm => method(:on_rabbitmq_confirm))
72
+ end
73
+
74
+ def self.recover_payloads_for_rabbitmq
75
+ return unless Promiscuous::Redis.master
76
+ # This method is regularly called from a worker to resend payloads that
77
+ # never got their confirm. We get the oldest queued message, and test if
78
+ # it's old enough to for a republish (default 10 seconds).
79
+ # Any sort of race is okay since we would just republish, and that's okay.
80
+
81
+ Promiscuous::Redis.master.nodes.each do |node|
82
+ loop do
83
+ key, time = node.zrange(rabbitmq_staging_set_key, 0, 1, :with_scores => true).flatten
84
+ break unless key && Time.now.to_i >= time.to_i + Promiscuous::Config.recovery_timeout
85
+
86
+ # Refresh the score so we skip it next time we look for something to recover.
87
+ node.zadd(rabbitmq_staging_set_key, Time.now.to_i, key)
88
+ payload = node.get(key)
89
+
90
+ Promiscuous.info "[payload recovery] #{payload}"
91
+ new.instance_eval do
92
+ @payload_recovery_node = node
93
+ @payload_recovery_key = key
94
+ @amqp_key = MultiJson.load(payload)['__amqp__']
95
+ @payload = payload
96
+ publish_payload_in_rabbitmq_async
97
+ end
98
+ end
99
+ end
100
+ end
101
+
102
+ def publish_payload_in_redis
103
+ # TODO Optimize and DRY this up
104
+ r = @committed_read_deps
105
+ w = @committed_write_deps
106
+
107
+ master_node = w.first.redis_node
108
+ operation_recovery_key = w.first.key(:pub).join('operation_recovery').to_s
109
+ # We identify a payload with a unique key (id:id_value:current_version) to
110
+ # avoid collisions with other updates on the same document.
111
+ @payload_recovery_node = master_node
112
+ @payload_recovery_key = w.first.key(:pub).join(w.first.version).to_s
113
+
114
+ # We need to be able to recover from a redis failure. By sending the
115
+ # payload to the slave first, we ensure that we can replay the lost
116
+ # payloads if the primary came to fail.
117
+ # We still need to recover the lost operations. This can be done by doing a
118
+ # version diff from what is stored in the database and the recovered redis slave.
119
+ # XXX TODO
120
+ # Promiscuous::Redis.slave.set(@payload_recovery_key, @payload) if Promiscuous::Redis.slave
121
+
122
+ # We don't care if we get raced by someone recovering our operation. It can
123
+ # happen if we lost the lock without knowing about it.
124
+ # The payload can be sent twice, which is okay since the subscribers
125
+ # tolerate it.
126
+
127
+ nodes = (w+r).map(&:redis_node).uniq
128
+ if nodes.size == 1
129
+ # We just have the master node. Since we are atomic, we don't need to do
130
+ # the 2pc dance.
131
+ master_node.multi do
132
+ master_node.del(operation_recovery_key)
133
+ master_node.set(@payload_recovery_key, @payload)
134
+ master_node.zadd(rabbitmq_staging_set_key, Time.now.to_i, @payload_recovery_key)
135
+ end
136
+ else
137
+ master_node.multi do
138
+ master_node.set(@payload_recovery_key, @payload)
139
+ master_node.zadd(rabbitmq_staging_set_key, Time.now.to_i, @payload_recovery_key)
140
+ end
141
+
142
+ # The payload is safe now. We can cleanup all the versions on the
143
+ # secondary. Note that we need to clear the master node at the end,
144
+ # as it acts as a lock on the other keys. This is important to avoid a
145
+ # race where we would delete data that doesn't belong to the current
146
+ # operation due to a lock loss.
147
+ nodes.reject { |node| node == master_node }
148
+ .each { |node| node.del(operation_recovery_key) }
149
+ master_node.del(operation_recovery_key)
150
+ end
151
+ end
152
+
153
+ def generate_payload_and_clear_operations
154
+ # TODO Transactions with multi writes
155
+ raise "We don't support multi writes yet" if previous_successful_operations.select(&:write?).size > 1
156
+ raise "The instance is gone, or there is a version mismatch" unless @instance
157
+
158
+ payload = @instance.promiscuous.payload(:with_attributes => operation.in?([:create, :update]))
159
+ payload[:context] = current_context.name
160
+ payload[:timestamp] = @timestamp
161
+
162
+ # If the db operation has failed, so we publish a dummy operation on the
163
+ # failed instance. It's better than using the Dummy polisher class
164
+ # because a subscriber can choose not to receive any of these messages.
165
+ payload[:operation] = self.failed? ? :dummy : operation
166
+
167
+ # We need to consider the last write operation as an implicit read
168
+ # dependency. This is why we don't need to consider the read dependencies
169
+ # happening before a first write when publishing the second write in a
170
+ # context.
171
+ payload[:dependencies] = {}
172
+ payload[:dependencies][:read] = @committed_read_deps if @committed_read_deps.present?
173
+ payload[:dependencies][:write] = @committed_write_deps
174
+
175
+ current_context.last_write_dependency = @committed_write_deps.first
176
+ current_context.operations.clear
177
+
178
+ @amqp_key = payload[:__amqp__]
179
+ @payload = MultiJson.dump(payload)
180
+ end
181
+
182
+ def self.recover_operation_from_lock(lock)
183
+ # We happen to have acquired a never released lock.
184
+ # The database instance is thus still prestine.
185
+ # Three cases to consider:
186
+ # 1) the key is not an id dependency or the payload queue stage was passed
187
+ # 2) The write query was never executed, we must send a dummy operation
188
+ # 3) The write query was executed, but never passed the payload queue stage
189
+
190
+ master_node = lock.node
191
+ recovery_data = master_node.hgetall("#{lock.key}:operation_recovery")
192
+ return nil unless recovery_data.present? # case 1)
193
+
194
+ Promiscuous.info "[operation recovery] #{lock.key} -> #{recovery_data}"
195
+
196
+ collection, instance_id, operation,
197
+ document, read_dependencies, write_dependencies = *MultiJson.load(recovery_data['payload'])
198
+
199
+ operation = operation.to_sym
200
+ read_dependencies.map! { |k| Promiscuous::Dependency.parse(k.to_s) }
201
+ write_dependencies.map! { |k| Promiscuous::Dependency.parse(k.to_s) }
202
+
203
+ model = Promiscuous::Publisher::Model.publishers[collection]
204
+
205
+ if model.is_a? Promiscuous::Publisher::Model::Ephemeral
206
+ operation = :dummy
207
+ else
208
+ # TODO Abstract db operations.
209
+ # We need to query on the root model
210
+ model = model.collection.name.singularize.camelize.constantize
211
+ end
212
+
213
+ op_klass = model.get_operation_class_for(operation)
214
+ op = op_klass.recover_operation(model, instance_id, document)
215
+ op.operation = operation
216
+
217
+ Promiscuous.context :operation_recovery, :detached_from_parent => true do
218
+ op.instance_eval do
219
+ @read_dependencies = read_dependencies
220
+ @write_dependencies = write_dependencies
221
+ @locks = [lock]
222
+ execute_persistent_locked { recover_db_operation }
223
+ end
224
+ end
225
+
226
+ lock.unlock
227
+ rescue Exception => e
228
+ message = "cannot recover #{lock.key} -> #{recovery_data}"
229
+ raise Promiscuous::Error::Recovery.new(message, e)
230
+ end
231
+
232
+ def increment_read_and_write_dependencies(read_dependencies, write_dependencies)
233
+ # We collapse all operations, ignoring the read/write interleaving.
234
+ # It doesn't matter since all write operations are serialized, so the first
235
+ # write in the transaction can have all the read dependencies.
236
+ r = read_dependencies
237
+ w = write_dependencies
238
+
239
+ # We don't need to do a read dependency if we are writing to it, so we
240
+ # prune them. The subscriber assumes the pruning (i.e. the intersection of
241
+ # r and w is empty) when it calculates the happens before relationships.
242
+ r -= w
243
+
244
+ master_node = w.first.redis_node
245
+ operation_recovery_key = w.first
246
+
247
+ # We group all the dependencies by their respective shards
248
+ # The master node will have the responsability to hold the recovery data.
249
+ # We do the master node first. The seconaries can be done in parallel.
250
+ (w+r).group_by(&:redis_node).each do |node, deps|
251
+ r_deps = deps.select { |dep| dep.in? r }
252
+ w_deps = deps.select { |dep| dep.in? w }
253
+
254
+ argv = []
255
+ argv << Promiscuous::Key.new(:pub) # key prefixes
256
+ argv << MultiJson.dump([r_deps, w_deps])
257
+
258
+ # Each shard have their own recovery payload. The master recovery node
259
+ # has the full operation recovery, and the others just have their versions.
260
+ argv << operation_recovery_key.as_json
261
+ if node == master_node
262
+ # We are on the master node, which holds the recovery payload
263
+ document = serialize_document_for_create_recovery if operation == :create
264
+ argv << MultiJson.dump([@instance.class.promiscuous_collection_name,
265
+ @instance.id, operation, document, r, w])
266
+ end
267
+
268
+ # We are going to store all the versions in redis, to be able to recover.
269
+ # We store all our increments in a transaction_id key in JSON format.
270
+ # Note that the transaction_id is the id of the current instance.
271
+ @@increment_script ||= Promiscuous::Redis::Script.new <<-SCRIPT
272
+ local prefix = ARGV[1] .. ':'
273
+ local deps = cjson.decode(ARGV[2])
274
+ local read_deps = deps[1]
275
+ local write_deps = deps[2]
276
+ local operation_recovery_key = prefix .. ARGV[3] .. ':operation_recovery'
277
+ local operation_recovery_payload = ARGV[4]
278
+
279
+ local read_versions = {}
280
+ local write_versions = {}
281
+
282
+ if redis.call('exists', operation_recovery_key) == 1 then
283
+ for i, dep in ipairs(read_deps) do
284
+ local key = prefix .. dep
285
+ read_versions[i] = redis.call('get', key .. ':w')
286
+ end
287
+ for i, dep in ipairs(write_deps) do
288
+ local key = prefix .. dep
289
+ write_versions[i] = redis.call('get', key .. ':w')
290
+ end
291
+ else
292
+ for i, dep in ipairs(read_deps) do
293
+ local key = prefix .. dep
294
+ redis.call('incr', key .. ':rw')
295
+ read_versions[i] = redis.call('get', key .. ':w')
296
+ redis.call('hset', operation_recovery_key, dep, read_versions[i])
297
+ end
298
+
299
+ for i, dep in ipairs(write_deps) do
300
+ local key = prefix .. dep
301
+ write_versions[i] = redis.call('incr', key .. ':rw')
302
+ redis.call('set', key .. ':w', write_versions[i])
303
+ redis.call('hset', operation_recovery_key, dep, write_versions[i])
304
+ end
305
+
306
+ if operation_recovery_payload then
307
+ redis.call('hset', operation_recovery_key, 'payload', operation_recovery_payload)
308
+ end
309
+ end
310
+
311
+ return { read_versions, write_versions }
312
+ SCRIPT
313
+ read_versions, write_versions = @@increment_script.eval(node, :argv => argv)
314
+
315
+ r_deps.zip(read_versions).each { |dep, version| dep.version = version.to_i }
316
+ w_deps.zip(write_versions).each { |dep, version| dep.version = version.to_i }
317
+ end
318
+
319
+ @committed_read_deps = r
320
+ @committed_write_deps = w
321
+ @instance_version = w.first.version
322
+ end
323
+
324
+ LOCK_OPTIONS = { :timeout => 10.seconds, # after 10 seconds, we give up
325
+ :sleep => 0.01, # polling every 10ms.
326
+ :expire => 1.minute } # after one minute, we are considered dead
327
+
328
+ def self.lock_options
329
+ LOCK_OPTIONS.merge({ :lock_set => Promiscuous::Key.new(:pub).join('lock_set').to_s })
330
+ end
331
+
332
+ def self.recover_locks
333
+ return unless Promiscuous::Redis.master
334
+ # This method is regularly called from a worker to recover locks by doing a
335
+ # locking/unlocking cycle.
336
+
337
+ Promiscuous::Redis.master.nodes.each do |node|
338
+ loop do
339
+ key, time = node.zrange(lock_options[:lock_set], 0, 1, :with_scores => true).flatten
340
+ break unless key && Time.now.to_i >= time.to_i + lock_options[:expire]
341
+
342
+ mutex = Promiscuous::Redis::Mutex.new(key, lock_options.merge(:node => node))
343
+ case mutex.lock
344
+ when :recovered then recover_operation_from_lock(mutex)
345
+ when true then mutex.unlock
346
+ when false then ;
347
+ end
348
+ end
349
+ end
350
+ end
351
+
352
+ def locks_from_write_dependencies
353
+ # XXX TODO Support multi row writes
354
+ instance_dep = write_dependencies.first
355
+ return [] unless instance_dep
356
+ options = self.class.lock_options.merge(:node => instance_dep.redis_node)
357
+ [Promiscuous::Redis::Mutex.new(instance_dep.key(:pub).to_s, options)]
358
+ end
359
+
360
+ def lock_write_dependencies
361
+ # returns true if we could get all the locks, false otherwise
362
+
363
+ start_at = Time.now
364
+ @recovered_locks = []
365
+
366
+ # We acquire all the locks in order, and unlock everything if one come
367
+ # to fail. lock/unlock return true/false when they succeed/fail
368
+ locks = locks_from_write_dependencies
369
+ locks.reduce(->{ @locks = locks; true }) do |chain, l|
370
+ lambda do
371
+ return false if Time.now - start_at > LOCK_OPTIONS[:timeout]
372
+ case l.lock
373
+ # Note that we do not unlock the recovered lock if the chain fails
374
+ when :recovered then @recovered_locks << l; chain.call
375
+ when true then chain.call or (l.unlock; false)
376
+ when false then @unavailable_lock = l; false
377
+ end
378
+ end
379
+ end.call
380
+ end
381
+
382
+ def unlock_write_dependencies
383
+ # returns true if we could unlock all the locks, false otherwise
384
+ return true if @locks.blank?
385
+ @locks.reduce(true) { |result, l| l.unlock && result }.tap { @locks = nil }
386
+ end
387
+
388
+ def _reload_instance_dependencies
389
+ if read?
390
+ # We want to use the smallest subset that we can depend on when doing
391
+ # reads. tracked_dependencies comes sorted from the smallest subset to
392
+ # the largest. For maximum performance on the subscriber side, we thus
393
+ # pick the first one. In most cases, it should resolve to the id
394
+ # dependency.
395
+ best_dependency = @instance.promiscuous.tracked_dependencies.first
396
+ unless best_dependency
397
+ raise Promiscuous::Error::Dependency.new(:operation => self)
398
+ end
399
+ [best_dependency]
400
+ else
401
+ # Note that tracked_dependencies will not return the id dependency if it
402
+ # doesn't exist which can only happen for create operations and auto
403
+ # generated ids. Be aware that with auto generated id, create operation
404
+ # might not provide the id dependency.
405
+ @instance.promiscuous.tracked_dependencies
406
+ end
407
+ end
408
+
409
+ def reload_instance_dependencies
410
+ # Returns true when the dependencies changed, false otherwise
411
+ @write_dependencies = nil
412
+ old = @instance_dependencies
413
+ @instance_dependencies = _reload_instance_dependencies
414
+ old != @instance_dependencies
415
+ end
416
+
417
+ def instance_dependencies
418
+ reload_instance_dependencies unless @instance_dependencies
419
+ @instance_dependencies
420
+ end
421
+
422
+ def previous_successful_operations
423
+ current_context.operations.reject(&:failed?)
424
+ end
425
+
426
+ def read_dependencies
427
+ # We memoize the read dependencies not just for performance, but also
428
+ # because we store the versions once incremented in these.
429
+ return @read_dependencies if @read_dependencies
430
+ read_dependencies = previous_successful_operations.select(&:read?)
431
+ .map(&:instance_dependencies).flatten
432
+
433
+ # We implicitly have a read dependency on the latest write.
434
+ if current_context.last_write_dependency
435
+ current_context.last_write_dependency.version = nil
436
+ read_dependencies << current_context.last_write_dependency
437
+ end
438
+
439
+ @read_dependencies = read_dependencies.uniq
440
+ end
441
+ alias verify_read_dependencies read_dependencies
442
+
443
+ def write_dependencies
444
+ # The cache is cleared when we call reload_instance_dependencies
445
+ @write_dependencies ||= previous_successful_operations.select(&:write?)
446
+ .map(&:instance_dependencies).flatten.uniq
447
+ end
448
+
449
+ def reload_instance
450
+ @instance = without_promiscuous { fetch_instance }
451
+ end
452
+
453
+ def perform_db_operation_with_no_exceptions(&db_operation)
454
+ going_to_execute_db_operation
455
+ @result = db_operation.call(self)
456
+ rescue Exception => e
457
+ @exception = e
458
+ end
459
+
460
+ def lock_instance_for_execute_persistent
461
+ current_context.add_operation(self)
462
+
463
+ # Note: At first, @instance can be a representation of a selector, to
464
+ # become a real model instance once we get to fetch it from the db with
465
+ # reload_instance to lock an instance that matches the selector.
466
+ # This is a good thing because we allow the underlying driver to hook from
467
+ # the model interface to the driver interface easily.
468
+ auto_unlock = true
469
+
470
+ begin
471
+ unless lock_write_dependencies
472
+ raise Promiscuous::Error::LockUnavailable.new(@unavailable_lock.key)
473
+ end
474
+
475
+ if @recovered_locks.present?
476
+ # When recovering locks, if we fail, we must not release the lock again
477
+ # to allow another one to do the recovery.
478
+ auto_unlock = false
479
+ @recovered_locks.each { |lock| self.class.recover_operation_from_lock(lock) }
480
+ auto_unlock = true
481
+ raise TryAgain
482
+ end
483
+
484
+ if operation != :create
485
+ # We need to lock and update all the dependencies before any other
486
+ # readers can see our write through any one of our tracked attributes.
487
+
488
+ # We want to reload the instance to make sure we have all the locked
489
+ # dependencies that we need. It's a query we cannot avoid when we have
490
+ # tracked dependencies. There is a bit of room for optimization.
491
+ # If the selector doesn't fetch any instance, the query has no effect
492
+ # so we can bypass it as if nothing happened. If reload_instance
493
+ # raises an exception, it's okay to let it bubble up since we haven't
494
+ # touch anything yet except for the locks (which will be unlocked on
495
+ # the way out)
496
+ return false unless reload_instance
497
+
498
+ # If reload_instance changed the current instance because the selector,
499
+ # we need to unlock the old instance, lock this new instance, and
500
+ # retry. XXX What should we do if we are going in a live lock?
501
+ # Sleep with some jitter?
502
+ if reload_instance_dependencies
503
+ raise TryAgain
504
+ end
505
+ end
506
+ rescue TryAgain
507
+ unlock_write_dependencies if auto_unlock
508
+ retry
509
+ end
510
+
511
+ verify_read_dependencies
512
+ if write_dependencies.blank?
513
+ # TODO We don't like auto generated ids. A good solution is to do all
514
+ # writes in a transaction, so we can know the ids at commit time.
515
+ raise "We don't support auto generated id yet"
516
+ end
517
+
518
+ # We are now in the possession of an instance that matches the original
519
+ # selector, we can proceed.
520
+ auto_unlock = false
521
+ true
522
+ ensure
523
+ # In case of an exception was raised before we updated the version in
524
+ # redis, we can unlock because we don't need recovery.
525
+ unlock_write_dependencies if auto_unlock
526
+ end
527
+
528
+ def execute_persistent_locked(&db_operation)
529
+ # We are going to commit all the pending writes in the context if we are
530
+ # doing a transaction commit. We also commit the current write operation for
531
+ # atomic writes without transactions. We enable the recovery mechanism by
532
+ # having someone expiring our lock if we die in the middle.
533
+
534
+ # All the versions are updated and a marked as pending for publish in Redis
535
+ # atomically in case we die before we could write the versions in the
536
+ # database. Once incremented, concurrent queries that are reading our
537
+ # instance will be serialized after our write, even through it may read our
538
+ # old instance. This is a race that we tolerate.
539
+ # XXX We also stash the document for create operations, so the recovery can
540
+ # redo the create to avoid races when instances are getting partitioned.
541
+ increment_read_and_write_dependencies(read_dependencies, write_dependencies)
542
+
543
+ # From this point, if we die, the one expiring our write locks must finish
544
+ # the publish, either by sending a dummy, or by sending the real instance.
545
+ # We could have die before or after the database query.
546
+
547
+ # We save the versions in the database, as it is our source of truth.
548
+ # This allow a reconstruction of redis in the face of failures.
549
+ # We would also need to send a special message to the subscribers to reset
550
+ # their read counters to the last write version since we would not be able
551
+ # to restore the read counters (and we don't want to store them because
552
+ # this would dramatically augment our footprint on the db).
553
+ #
554
+ # If we are doing a destroy operation, and redis dies right after, and
555
+ # we happen to lost contact with rabbitmq, recovery is going to be complex:
556
+ # we would need to do a diff from the dummy subscriber to see what
557
+ # documents are missing on our side to be able to resend the destroy
558
+ # message.
559
+
560
+ case operation
561
+ when :create
562
+ stash_version_in_write_query
563
+ when :update
564
+ stash_version_in_write_query
565
+ # We are now in the possession of an instance that matches the original
566
+ # selector. We need to make sure the db_operation will operate on it,
567
+ # instead of the original selector.
568
+ use_id_selector(:use_atomic_version_selector => true)
569
+ # We need to use an atomic versioned selector to make sure that
570
+ # if we lose the lock for a long period of time, we don't mess up
571
+ # with other people's updates. Also we make sure that the recovery
572
+ # mechanism is not racing with us.
573
+ when :destroy
574
+ use_id_selector(:use_atomic_version_selector => true)
575
+ end
576
+
577
+ # Perform the actual database query (single write or transaction commit).
578
+ # If successful, the result goes in @result, otherwise, @exception contains
579
+ # the thrown exception.
580
+ perform_db_operation_with_no_exceptions(&db_operation)
581
+
582
+ # We take a timestamp right after the write is performed because latency
583
+ # measurements are performed on the subscriber.
584
+ record_timestamp
585
+
586
+ if operation == :update && !failed?
587
+ # The underlying driver should implement some sort of find and modify
588
+ # operation in the previous write query to avoid this extra read query.
589
+ # If reload_instance raise an exception, we let it bubble up,
590
+ # and we'll trigger the recovery mechanism.
591
+ use_id_selector
592
+ reload_instance
593
+ end
594
+
595
+ unless @locks.first.still_locked?
596
+ # We lost the lock, let the recovery mechanism do its thing.
597
+ # This is a code optimization to avoid checking if the db operation
598
+ # succeeded or not because of the db operation race during recovery.
599
+ raise Promiscuous::Error::LostLock.new(@locks.first.key)
600
+ end
601
+
602
+ generate_payload_and_clear_operations
603
+
604
+ # As soon as we unlock the locks, the rescuer will not be able to assume
605
+ # that the database instance is still pristine, and so we need to stash the
606
+ # payload in redis. If redis dies, we don't care because it can be
607
+ # reconstructed. Subscribers can see "compressed" updates.
608
+ publish_payload_in_redis
609
+
610
+ # TODO Performance: merge these 3 redis operations to speed things up.
611
+ unlock_write_dependencies
612
+
613
+ # If we die from this point on, a recovery worker can republish our payload
614
+ # since we queued it in Redis.
615
+
616
+ # We don't care if we lost the lock and got recovered, subscribers are
617
+ # immune to duplicate messages.
618
+ publish_payload_in_rabbitmq_async
619
+ end
620
+
621
+ # --- the following methods can be overridden by the driver --- #
622
+
623
+ def execute_persistent(&db_operation)
624
+ return nil unless lock_instance_for_execute_persistent
625
+ execute_persistent_locked(&db_operation)
626
+ end
627
+
628
+ def execute_non_persistent(&db_operation)
629
+ # We are getting here in the following cases:
630
+ # * read: we fetch the instance. It's the driver's job to cache the
631
+ # raw instance and return it during db_operation.
632
+ # * multi read: nothing to do, we'll keep our current selector, sadly
633
+ # * write in a transaction: TODO
634
+
635
+ if single?
636
+ # If the query misses, we don't bother
637
+ return nil unless reload_instance
638
+ use_id_selector
639
+ end
640
+
641
+ # We don't do any reload_instance_dependencies at this point (and thus we
642
+ # won't raise an exception on a multi read that we cannot track).
643
+ # We'll wait until the commit, and hopefully with tainting, we'll be able to
644
+ # tell if we should depend the multi read operation in question.
645
+ perform_db_operation_with_no_exceptions(&db_operation)
646
+ # If the db_operation raises, we don't consider this failed operation when
647
+ # committing the next persistent write by omitting the operation in the
648
+ # context.
649
+ current_context.add_operation(self) unless failed?
650
+ end
651
+
652
+ def execute(&db_operation)
653
+ # execute returns the result of the db_operation to perform
654
+ db_operation ||= proc {}
655
+ return db_operation.call if Promiscuous.disabled
656
+
657
+ unless current_context
658
+ raise Promiscuous::Error::MissingContext if write?
659
+ return db_operation.call # Don't care for a read
660
+ end
661
+
662
+ self.persists? ? execute_persistent(&db_operation) :
663
+ execute_non_persistent(&db_operation)
664
+
665
+ @exception ? (raise @exception) : @result
666
+ end
667
+
668
+ def fetch_instance
669
+ # This method is overridden to use the original query selector.
670
+ # Should return nil if the instance is not found.
671
+ @instance
672
+ end
673
+
674
+ def serialize_document_for_create_recovery
675
+ # Overridden to be able to redo the create during recovery.
676
+ nil
677
+ end
678
+
679
+ def self.recover_operation(model, instance_id, document)
680
+ # Overriden to reconstruct the operation. If the database is read, only the
681
+ # primary must be used.
682
+ new(:instance => model.new { |instance| instance.id = instance_id })
683
+ end
684
+
685
+ def recover_db_operation
686
+ # Overriden to reexecute the db operation during recovery (or make sure that
687
+ # it will never succeed).
688
+ end
689
+
690
+ def use_id_selector(options={})
691
+ # Overridden to use the {:id => @instance.id} selector.
692
+ # if use_atomic_version_selector is passed, the driver must
693
+ # add the VERSION_FIELD selector if present in original instance.
694
+ end
695
+
696
+ def use_versioned_selector
697
+ # Overridden to use the {VERSION_FIELD => @instance[VERSION_FIELD]} selector.
698
+ end
699
+
700
+ def stash_version_in_write_query
701
+ # Overridden to update the query to set 'instance.VERSION_FIELD = @instance_version'
702
+ end
703
+
704
+ def going_to_execute_db_operation
705
+ # Test hook
706
+ end
707
+ end