promiscuous 0.100.5 → 1.0.0.beta1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. checksums.yaml +4 -4
  2. data/lib/promiscuous.rb +5 -1
  3. data/lib/promiscuous/config.rb +6 -5
  4. data/lib/promiscuous/dsl.rb +0 -4
  5. data/lib/promiscuous/loader.rb +0 -5
  6. data/lib/promiscuous/mongoid.rb +15 -5
  7. data/lib/promiscuous/publisher.rb +1 -1
  8. data/lib/promiscuous/publisher/model/active_record.rb +6 -1
  9. data/lib/promiscuous/publisher/model/base.rb +8 -11
  10. data/lib/promiscuous/publisher/model/mock.rb +2 -2
  11. data/lib/promiscuous/publisher/model/mongoid.rb +3 -4
  12. data/lib/promiscuous/publisher/operation/active_record.rb +13 -69
  13. data/lib/promiscuous/publisher/operation/atomic.rb +15 -158
  14. data/lib/promiscuous/publisher/operation/base.rb +13 -381
  15. data/lib/promiscuous/publisher/operation/ephemeral.rb +12 -8
  16. data/lib/promiscuous/publisher/operation/mongoid.rb +22 -92
  17. data/lib/promiscuous/publisher/operation/non_persistent.rb +0 -9
  18. data/lib/promiscuous/publisher/operation/proxy_for_query.rb +8 -6
  19. data/lib/promiscuous/publisher/operation/transaction.rb +4 -56
  20. data/lib/promiscuous/publisher/transport.rb +14 -0
  21. data/lib/promiscuous/publisher/transport/batch.rb +138 -0
  22. data/lib/promiscuous/publisher/transport/persistence.rb +14 -0
  23. data/lib/promiscuous/publisher/transport/persistence/active_record.rb +33 -0
  24. data/lib/promiscuous/publisher/transport/persistence/mongoid.rb +22 -0
  25. data/lib/promiscuous/publisher/transport/worker.rb +36 -0
  26. data/lib/promiscuous/publisher/worker.rb +3 -12
  27. data/lib/promiscuous/redis.rb +5 -0
  28. data/lib/promiscuous/subscriber/message.rb +1 -29
  29. data/lib/promiscuous/subscriber/model/base.rb +3 -2
  30. data/lib/promiscuous/subscriber/model/mongoid.rb +16 -1
  31. data/lib/promiscuous/subscriber/model/observer.rb +0 -1
  32. data/lib/promiscuous/subscriber/operation.rb +9 -3
  33. data/lib/promiscuous/subscriber/unit_of_work.rb +7 -7
  34. data/lib/promiscuous/subscriber/worker/eventual_destroyer.rb +1 -1
  35. data/lib/promiscuous/version.rb +1 -1
  36. metadata +39 -35
  37. data/lib/promiscuous/dependency.rb +0 -78
  38. data/lib/promiscuous/error/dependency.rb +0 -116
@@ -1,25 +1,10 @@
1
1
  class Promiscuous::Publisher::Operation::Base
2
- mattr_accessor :recovery_mechanisms
3
- self.recovery_mechanisms = []
4
-
5
- def self.register_recovery_mechanism(method_name=nil, &block)
6
- self.recovery_mechanisms << (block || method(method_name))
7
- end
8
-
9
- def self.run_recovery_mechanisms
10
- self.recovery_mechanisms.each(&:call)
11
- end
12
-
13
- attr_accessor :operation
2
+ attr_accessor :operation, :recovering
14
3
 
15
4
  def initialize(options={})
16
5
  @operation = options[:operation]
17
6
  end
18
7
 
19
- def recovering?
20
- !!@recovery_data
21
- end
22
-
23
8
  def record_timestamp
24
9
  # Records the number of milliseconds since epoch, which we use send sending
25
10
  # the payload over. It's good for latency measurements.
@@ -27,345 +12,6 @@ class Promiscuous::Publisher::Operation::Base
27
12
  @timestamp = time.to_i * 1000 + time.usec / 1000
28
13
  end
29
14
 
30
- def self.rabbitmq_staging_set_key
31
- Promiscuous::Key.new(:pub).join('rabbitmq_staging').to_s
32
- end
33
-
34
- delegate :rabbitmq_staging_set_key, :to => self
35
-
36
- def on_rabbitmq_confirm
37
- # These requests could be parallelized, rabbitmq persisted the operation.
38
- # XXX TODO
39
- # Promiscuous::Redis.slave.del(@payload_recovery_key) if Promiscuous::Redis.slave
40
-
41
- @payload_recovery_node.multi do
42
- @payload_recovery_node.del(@payload_recovery_key)
43
- @payload_recovery_node.zrem(rabbitmq_staging_set_key, @payload_recovery_key)
44
- end
45
- end
46
-
47
- def publish_payload_in_rabbitmq_async
48
- Promiscuous::AMQP.publish(:key => Promiscuous::Config.app, :payload => @payload,
49
- :on_confirm => method(:on_rabbitmq_confirm))
50
- end
51
-
52
- def self.recover_payloads_for_rabbitmq
53
- return unless Promiscuous::Redis.master
54
- # This method is regularly called from a worker to resend payloads that
55
- # never got their confirm. We get the oldest queued message, and test if
56
- # it's old enough to for a republish (default 10 seconds).
57
- # Any sort of race is okay since we would just republish, and that's okay.
58
-
59
- Promiscuous::Redis.master.nodes.each do |node|
60
- loop do
61
- key, time = node.zrange(rabbitmq_staging_set_key, 0, 1, :with_scores => true).flatten
62
- break unless key && Time.now.to_i >= time.to_i + Promiscuous::Config.recovery_timeout
63
-
64
- # Refresh the score so we skip it next time we look for something to recover.
65
- node.zadd(rabbitmq_staging_set_key, Time.now.to_i, key)
66
- payload = node.get(key)
67
-
68
- # It's possible that the payload is nil as the message could be
69
- # recovered by another worker
70
- if payload
71
- Promiscuous.info "[payload recovery] #{payload}"
72
- new.instance_eval do
73
- @payload_recovery_node = node
74
- @payload_recovery_key = key
75
- @payload = payload
76
- @recovery = true
77
- publish_payload_in_rabbitmq_async
78
- end
79
- end
80
- end
81
- end
82
- end
83
- register_recovery_mechanism :recover_payloads_for_rabbitmq
84
-
85
- def publish_payload_in_redis
86
- # TODO Optimize and DRY this up
87
- w = @committed_write_deps
88
-
89
- # We identify a payload with a unique key (id:id_value:current_version:payload_recovery)
90
- # to avoid collisions with other updates on the same document.
91
- master_node = @op_lock.node
92
- @payload_recovery_node = master_node
93
- @payload_recovery_key = Promiscuous::Key.new(:pub).join('payload_recovery', @op_lock.token).to_s
94
-
95
- # We need to be able to recover from a redis failure. By sending the
96
- # payload to the slave first, we ensure that we can replay the lost
97
- # payloads if the master came to fail.
98
- # We still need to recover the lost operations. This can be done by doing a
99
- # version diff from what is stored in the database and the recovered redis slave.
100
- # XXX TODO
101
- # Promiscuous::Redis.slave.set(@payload_recovery_key, @payload) if Promiscuous::Redis.slave
102
-
103
- # We don't care if we get raced by someone recovering our operation. It can
104
- # happen if we lost the lock without knowing about it.
105
- # The payload can be sent twice, which is okay since the subscribers
106
- # tolerate it.
107
- operation_recovery_key = "#{@op_lock.key}:operation_recovery"
108
- versions_recovery_key = "#{operation_recovery_key}:versions"
109
-
110
- master_node.multi do
111
- master_node.set(@payload_recovery_key, @payload)
112
- master_node.zadd(rabbitmq_staging_set_key, Time.now.to_i, @payload_recovery_key)
113
- master_node.del(operation_recovery_key)
114
- master_node.del(versions_recovery_key)
115
- end
116
-
117
- # The payload is safe now. We can cleanup all the versions on the
118
- # secondary. There are no harmful races that can happen since the
119
- # secondary_operation_recovery_key is unique to the operation.
120
- # XXX The caveat is that if we die here, the
121
- # secondary_operation_recovery_key will never be cleaned up.
122
- w.map(&:redis_node).uniq
123
- .reject { |node| node == master_node }
124
- .each { |node| node.del(versions_recovery_key) }
125
- end
126
-
127
- def payload_for(instance)
128
- options = { :with_attributes => self.operation.in?([:create, :update]) }
129
- instance.promiscuous.payload(options).tap do |payload|
130
- payload[:operation] = self.operation
131
- end
132
- end
133
-
134
- def generate_payload
135
- payload = {}
136
- payload[:operations] = operation_payloads
137
- payload[:app] = Promiscuous::Config.app
138
- payload[:current_user_id] = Promiscuous.context.current_user.id if Promiscuous.context.current_user
139
- payload[:timestamp] = @timestamp
140
- payload[:generation] = Promiscuous::Config.generation
141
- payload[:host] = Socket.gethostname
142
- payload[:recovered_operation] = true if recovering?
143
- payload[:dependencies] = {}
144
- payload[:dependencies][:write] = @committed_write_deps
145
-
146
- @payload = MultiJson.dump(payload)
147
- end
148
-
149
- def self.recover_operation_from_lock(lock)
150
- # We happen to have acquired a never released lock.
151
- # The database instance is thus still pristine.
152
-
153
- master_node = lock.node
154
- recovery_data = master_node.get("#{lock.key}:operation_recovery")
155
-
156
- unless recovery_data.present?
157
- lock.unlock
158
- return
159
- end
160
-
161
- Promiscuous.info "[operation recovery] #{lock.key} -> #{recovery_data}"
162
-
163
- op_klass, operation, write_dependencies, recovery_arguments = *MultiJson.load(recovery_data)
164
-
165
- operation = operation.to_sym
166
- write_dependencies.map! { |k| Promiscuous::Dependency.parse(k.to_s, :type => :write) }
167
-
168
- begin
169
- op = op_klass.constantize.recover_operation(*recovery_arguments)
170
- rescue NameError
171
- raise "invalid recover operation class: #{op_klass}"
172
- end
173
-
174
- Thread.new do
175
- # We run the recovery in another thread to ensure that we get a new
176
- # database connection to avoid tampering with the current state of the
177
- # connection, which can be in an open transaction.
178
- # Thankfully, we are not in a fast path.
179
- # Note that any exceptions will be passed through the thread join() method.
180
- op.instance_eval do
181
- @operation = operation
182
- @write_dependencies = write_dependencies
183
- @op_lock = lock
184
- @recovery_data = recovery_data
185
-
186
- query = Promiscuous::Publisher::Operation::ProxyForQuery.new(self) { recover_db_operation }
187
- self.execute_instrumented(query)
188
- query.result
189
- end
190
- end.join
191
-
192
- rescue Exception => e
193
- message = "cannot recover #{lock.key}, failed to fetch recovery data"
194
- message = "cannot recover #{lock.key}, recovery data: #{recovery_data}" if recovery_data
195
- raise Promiscuous::Error::Recovery.new(message, e)
196
- end
197
-
198
- def increment_dependencies
199
- # We collapse all operations, ignoring the read/write interleaving.
200
- # It doesn't matter since all write operations are serialized, so the first
201
- # write in the transaction can have all the read dependencies.
202
- w = write_dependencies
203
-
204
- master_node = @op_lock.node
205
- operation_recovery_key = "#{@op_lock.key}:operation_recovery"
206
-
207
- # We group all the dependencies by their respective shards
208
- # The master node will have the responsibility to hold the recovery data.
209
- # We do the master node first. The secondaries can be done in parallel.
210
- @committed_write_deps = []
211
-
212
- # We need to do the increments always in the same node order, otherwise.
213
- # the subscriber can deadlock. But we must always put the recovery payload
214
- # on the master before touching anything.
215
- nodes_deps = w.group_by(&:redis_node)
216
- .sort_by { |node, deps| -Promiscuous::Redis.master.nodes.index(node) }
217
- if nodes_deps.first[0] != master_node
218
- nodes_deps = [[master_node, []]] + nodes_deps
219
- end
220
-
221
- nodes_deps.each do |node, deps|
222
- argv = []
223
- argv << Promiscuous::Key.new(:pub) # key prefixes
224
- argv << operation_recovery_key
225
-
226
- # Each shard have their own recovery payload. The master recovery node
227
- # has the full operation recovery, and the others just have their versions.
228
- # Note that the operation_recovery_key on the secondaries have the current
229
- # version of the instance appended to them. It's easier to cleanup when
230
- # locks get lost.
231
- if node == master_node && !self.recovering?
232
- # We are on the master node, which holds the recovery payload
233
- argv << MultiJson.dump([self.class.name, operation, w, self.recovery_payload])
234
- end
235
-
236
- # FIXME If the lock is lost, we need to backoff
237
-
238
- # We are going to store all the versions in redis, to be able to recover.
239
- # We store all our increments in a transaction_id key in JSON format.
240
- # Note that the transaction_id is the id of the current instance.
241
- @@increment_script ||= Promiscuous::Redis::Script.new <<-SCRIPT
242
- local prefix = ARGV[1] .. ':'
243
- local operation_recovery_key = ARGV[2]
244
- local versions_recovery_key = operation_recovery_key .. ':versions'
245
- local operation_recovery_payload = ARGV[3]
246
- local deps = KEYS
247
-
248
- local versions = {}
249
-
250
- if redis.call('exists', versions_recovery_key) == 1 then
251
- for i, dep in ipairs(deps) do
252
- versions[i] = tonumber(redis.call('hget', versions_recovery_key, dep))
253
- if not versions[i] then
254
- return redis.error_reply('Failed to read dependency ' .. dep .. ' during recovery')
255
- end
256
- end
257
-
258
- return { versions }
259
- end
260
-
261
- for i, dep in ipairs(deps) do
262
- local key = prefix .. dep
263
- versions[i] = redis.call('incr', key .. ':w')
264
- redis.call('hset', versions_recovery_key, dep, versions[i])
265
- end
266
-
267
- if operation_recovery_payload then
268
- redis.call('set', operation_recovery_key, operation_recovery_payload)
269
- end
270
-
271
- return { versions }
272
- SCRIPT
273
-
274
- versions = @@increment_script.eval(node, :argv => argv, :keys => deps)
275
-
276
- deps.zip(versions).each { |dep, version| dep.version = version }
277
-
278
- @committed_write_deps += deps
279
- end
280
-
281
- # The instance version must to be the first in the list to allow atomic
282
- # subscribers to do their magic.
283
- # TODO What happens with transactions with multiple operations?
284
- instance_dep_index = @committed_write_deps.index(write_dependencies.first)
285
- @committed_write_deps[0], @committed_write_deps[instance_dep_index] =
286
- @committed_write_deps[instance_dep_index], @committed_write_deps[0]
287
- end
288
-
289
- def self.lock_options
290
- {
291
- :timeout => 10.seconds, # after 10 seconds, we give up so we don't queue requests
292
- :sleep => 0.01.seconds, # polling every 10ms.
293
- :expire => 1.minute, # after one minute, we are considered dead
294
- :lock_set => Promiscuous::Key.new(:pub).join('lock_set').to_s
295
- }
296
- end
297
- delegate :lock_options, :to => self
298
-
299
- def dependency_for_op_lock
300
- query_dependencies.first
301
- end
302
-
303
- def get_new_op_lock
304
- dep = dependency_for_op_lock
305
- Promiscuous::Redis::Mutex.new(dep.key(:pub).to_s, lock_options.merge(:node => dep.redis_node))
306
- end
307
-
308
- def self._acquire_lock(mutex)
309
- loop do
310
- case mutex.lock
311
- # recover_operation_from_lock implicitely unlocks the lock.
312
- when :recovered then recover_operation_from_lock(mutex)
313
- when true then return true
314
- when false then return false
315
- end
316
- end
317
- end
318
-
319
- def acquire_op_lock
320
- @op_lock = get_new_op_lock
321
-
322
- unless self.class._acquire_lock(@op_lock)
323
- raise Promiscuous::Error::LockUnavailable.new(@op_lock.key)
324
- end
325
- end
326
-
327
- def release_op_lock
328
- @op_lock.unlock
329
- @op_lock = nil
330
- end
331
-
332
- def ensure_op_still_locked
333
- unless @op_lock.still_locked?
334
- # We lost the lock, let the recovery mechanism do its thing.
335
- raise Promiscuous::Error::LostLock.new(@op_lock.key)
336
- end
337
- end
338
-
339
- def self.recover_locks
340
- return unless Promiscuous::Redis.master
341
- # This method is regularly called from a worker to recover locks by doing a
342
- # locking/unlocking cycle.
343
-
344
- Promiscuous::Redis.master.nodes.each do |node|
345
- loop do
346
- key, time = node.zrange(lock_options[:lock_set], 0, 1, :with_scores => true).flatten
347
- break unless key && Time.now.to_i >= time.to_i + lock_options[:expire]
348
-
349
- mutex = Promiscuous::Redis::Mutex.new(key, lock_options.merge(:node => node))
350
- mutex.unlock if _acquire_lock(mutex)
351
- end
352
- end
353
- end
354
- register_recovery_mechanism :recover_locks
355
-
356
- def dependencies_for(instance, options={})
357
- return [] if instance.nil?
358
-
359
- # Note that tracked_dependencies will not return the id dependency if it
360
- # doesn't exist which can only happen for create operations and auto
361
- # generated ids.
362
- [instance.promiscuous.get_dependency]
363
- end
364
-
365
- def write_dependencies
366
- @write_dependencies ||= self.query_dependencies.uniq.each { |d| d.type = :write }
367
- end
368
-
369
15
  def should_instrument_query?
370
16
  !Promiscuous.disabled?
371
17
  end
@@ -382,37 +28,11 @@ class Promiscuous::Publisher::Operation::Base
382
28
  query.result
383
29
  end
384
30
 
385
- def query_dependencies
386
- # Returns the list of dependencies that are involved in the database query.
387
- # For an atomic write operation, the first one returned must be the one
388
- # corresponding to the primary key.
389
- raise
390
- end
391
-
392
31
  def execute_instrumented(db_operation)
393
32
  # Implemented by subclasses
394
33
  raise
395
34
  end
396
35
 
397
- def operation_payloads
398
- # subclass can use payloads_for to generate the payload
399
- raise
400
- end
401
-
402
- def recovery_payload
403
- # Overridden to be able to recover the operation
404
- []
405
- end
406
-
407
- def self.recover_operation(*recovery_payload)
408
- # Overridden to reconstruct the operation.
409
- end
410
-
411
- def recover_db_operation
412
- # Overridden to reexecute the db operation during recovery (or make sure that
413
- # it will never succeed).
414
- end
415
-
416
36
  def trace_operation
417
37
  if ENV['TRACE']
418
38
  msg = self.explain_operation(70)
@@ -423,4 +43,16 @@ class Promiscuous::Publisher::Operation::Base
423
43
  def explain_operation(max_width)
424
44
  "Unknown database operation"
425
45
  end
46
+
47
+ def create_transport_batch(operations)
48
+ Promiscuous::Publisher::Transport::Batch.new.tap do |batch|
49
+ operations.map do |operation|
50
+ batch.add operation.operation, operation.instances
51
+ end
52
+
53
+ if current_user = Promiscuous.context.current_user
54
+ batch.payload_attributes = { :current_user_id => current_user.id }
55
+ end
56
+ end
57
+ end
426
58
  end
@@ -1,14 +1,18 @@
1
- class Promiscuous::Publisher::Operation::Ephemeral < Promiscuous::Publisher::Operation::Atomic
2
- def execute
3
- super {}
1
+ class Promiscuous::Publisher::Operation::Ephemeral < Promiscuous::Publisher::Operation::Base
2
+ def initialize(options={})
3
+ super
4
+ @instance = options[:instance]
4
5
  end
5
6
 
6
- def yell_about_missing_instance
7
- # don't yell :)
7
+ def instances
8
+ [@instance].compact
8
9
  end
9
10
 
10
- def self.recover_operation(*recovery_payload)
11
- # no instance when we recover, it's okay
12
- new(:instance => nil)
11
+ def execute_instrumented(query)
12
+ create_transport_batch([self]).publish(true)
13
+ end
14
+
15
+ def increment_version_in_document
16
+ # No op
13
17
  end
14
18
  end