promiscuous 0.92.0 → 0.100.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (47) hide show
  1. checksums.yaml +4 -4
  2. data/lib/promiscuous.rb +2 -5
  3. data/lib/promiscuous/amqp.rb +1 -2
  4. data/lib/promiscuous/cli.rb +3 -43
  5. data/lib/promiscuous/config.rb +5 -7
  6. data/lib/promiscuous/error/dependency.rb +1 -3
  7. data/lib/promiscuous/publisher/context.rb +1 -1
  8. data/lib/promiscuous/publisher/context/base.rb +3 -34
  9. data/lib/promiscuous/publisher/model/base.rb +5 -25
  10. data/lib/promiscuous/publisher/model/mock.rb +5 -7
  11. data/lib/promiscuous/publisher/operation/active_record.rb +4 -69
  12. data/lib/promiscuous/publisher/operation/atomic.rb +1 -3
  13. data/lib/promiscuous/publisher/operation/base.rb +33 -123
  14. data/lib/promiscuous/publisher/operation/mongoid.rb +0 -67
  15. data/lib/promiscuous/publisher/operation/non_persistent.rb +0 -1
  16. data/lib/promiscuous/publisher/operation/transaction.rb +1 -3
  17. data/lib/promiscuous/railtie.rb +0 -31
  18. data/lib/promiscuous/subscriber.rb +1 -1
  19. data/lib/promiscuous/subscriber/{worker/message.rb → message.rb} +12 -40
  20. data/lib/promiscuous/subscriber/model/active_record.rb +1 -1
  21. data/lib/promiscuous/subscriber/model/base.rb +4 -4
  22. data/lib/promiscuous/subscriber/model/mongoid.rb +3 -3
  23. data/lib/promiscuous/subscriber/operation.rb +74 -3
  24. data/lib/promiscuous/subscriber/unit_of_work.rb +110 -0
  25. data/lib/promiscuous/subscriber/worker.rb +3 -7
  26. data/lib/promiscuous/subscriber/worker/eventual_destroyer.rb +2 -6
  27. data/lib/promiscuous/subscriber/worker/pump.rb +2 -11
  28. data/lib/promiscuous/version.rb +1 -1
  29. metadata +18 -36
  30. data/lib/promiscuous/error/missing_context.rb +0 -29
  31. data/lib/promiscuous/publisher/bootstrap.rb +0 -27
  32. data/lib/promiscuous/publisher/bootstrap/connection.rb +0 -25
  33. data/lib/promiscuous/publisher/bootstrap/data.rb +0 -127
  34. data/lib/promiscuous/publisher/bootstrap/mode.rb +0 -19
  35. data/lib/promiscuous/publisher/bootstrap/status.rb +0 -40
  36. data/lib/promiscuous/publisher/bootstrap/version.rb +0 -46
  37. data/lib/promiscuous/publisher/context/middleware.rb +0 -94
  38. data/lib/promiscuous/resque.rb +0 -12
  39. data/lib/promiscuous/sidekiq.rb +0 -15
  40. data/lib/promiscuous/subscriber/message_processor.rb +0 -4
  41. data/lib/promiscuous/subscriber/message_processor/base.rb +0 -54
  42. data/lib/promiscuous/subscriber/message_processor/bootstrap.rb +0 -17
  43. data/lib/promiscuous/subscriber/message_processor/regular.rb +0 -238
  44. data/lib/promiscuous/subscriber/operation/base.rb +0 -66
  45. data/lib/promiscuous/subscriber/operation/bootstrap.rb +0 -60
  46. data/lib/promiscuous/subscriber/operation/regular.rb +0 -19
  47. data/lib/promiscuous/subscriber/worker/message_synchronizer.rb +0 -333
@@ -76,7 +76,6 @@ class Promiscuous::Publisher::Operation::Atomic < Promiscuous::Publisher::Operat
76
76
  # this is a problem, but we need to publish.
77
77
  yell_about_missing_instance if @instance.nil?
78
78
  else
79
- generate_read_dependencies
80
79
  acquire_op_lock
81
80
 
82
81
  if @instance.nil?
@@ -93,7 +92,7 @@ class Promiscuous::Publisher::Operation::Atomic < Promiscuous::Publisher::Operat
93
92
  # old instance. This is a race that we tolerate.
94
93
  # XXX We also stash the document for create operations, so the recovery can
95
94
  # redo the create to avoid races when instances are getting partitioned.
96
- increment_read_and_write_dependencies
95
+ increment_dependencies
97
96
 
98
97
  # From this point, if we die, the one expiring our write locks must finish
99
98
  # the publish, either by sending a dummy, or by sending the real instance.
@@ -123,7 +122,6 @@ class Promiscuous::Publisher::Operation::Atomic < Promiscuous::Publisher::Operat
123
122
  ensure_op_still_locked
124
123
 
125
124
  generate_payload
126
- clear_previous_dependencies
127
125
 
128
126
  # As soon as we unlock the locks, the rescuer will not be able to assume
129
127
  # that the database instance is still pristine, and so we need to stash the
@@ -16,22 +16,10 @@ class Promiscuous::Publisher::Operation::Base
16
16
  @operation = options[:operation]
17
17
  end
18
18
 
19
- def read?
20
- @operation == :read
21
- end
22
-
23
- def write?
24
- !read?
25
- end
26
-
27
19
  def recovering?
28
20
  !!@recovery_data
29
21
  end
30
22
 
31
- def current_context
32
- @current_context ||= Promiscuous::Publisher::Context.current
33
- end
34
-
35
23
  def record_timestamp
36
24
  # Records the number of milliseconds since epoch, which we use send sending
37
25
  # the payload over. It's good for latency measurements.
@@ -96,7 +84,6 @@ class Promiscuous::Publisher::Operation::Base
96
84
 
97
85
  def publish_payload_in_redis
98
86
  # TODO Optimize and DRY this up
99
- r = @committed_read_deps
100
87
  w = @committed_write_deps
101
88
 
102
89
  # We identify a payload with a unique key (id:id_value:current_version:payload_recovery)
@@ -132,7 +119,7 @@ class Promiscuous::Publisher::Operation::Base
132
119
  # secondary_operation_recovery_key is unique to the operation.
133
120
  # XXX The caveat is that if we die here, the
134
121
  # secondary_operation_recovery_key will never be cleaned up.
135
- (w+r).map(&:redis_node).uniq
122
+ w.map(&:redis_node).uniq
136
123
  .reject { |node| node == master_node }
137
124
  .each { |node| node.del(versions_recovery_key) }
138
125
  end
@@ -148,28 +135,21 @@ class Promiscuous::Publisher::Operation::Base
148
135
  payload = {}
149
136
  payload[:operations] = operation_payloads
150
137
  payload[:app] = Promiscuous::Config.app
151
- payload[:context] = current_context.name
152
- payload[:current_user_id] = current_context.current_user.id if current_context.current_user
138
+ payload[:current_user_id] = Promiscuous.context.current_user.id if Promiscuous.context.current_user
153
139
  payload[:timestamp] = @timestamp
154
140
  payload[:generation] = Promiscuous::Config.generation
141
+ payload[:context] = "DEPRECATED"
155
142
  payload[:host] = Socket.gethostname
156
- payload[:was_during_bootstrap] = true if @was_during_bootstrap
157
143
  payload[:recovered_operation] = true if recovering?
158
144
  payload[:dependencies] = {}
159
- payload[:dependencies][:read] = @committed_read_deps if @committed_read_deps.present?
160
145
  payload[:dependencies][:write] = @committed_write_deps
161
146
 
162
147
  @payload = MultiJson.dump(payload)
163
148
  end
164
149
 
165
- def clear_previous_dependencies
166
- current_context.read_operations.clear
167
- current_context.extra_dependencies = [@committed_write_deps.first]
168
- end
169
-
170
150
  def self.recover_operation_from_lock(lock)
171
151
  # We happen to have acquired a never released lock.
172
- # The database instance is thus still prestine.
152
+ # The database instance is thus still pristine.
173
153
 
174
154
  master_node = lock.node
175
155
  recovery_data = master_node.get("#{lock.key}:operation_recovery")
@@ -181,11 +161,9 @@ class Promiscuous::Publisher::Operation::Base
181
161
 
182
162
  Promiscuous.info "[operation recovery] #{lock.key} -> #{recovery_data}"
183
163
 
184
- op_klass, operation, read_dependencies,
185
- write_dependencies, recovery_arguments = *MultiJson.load(recovery_data)
164
+ op_klass, operation, write_dependencies, recovery_arguments = *MultiJson.load(recovery_data)
186
165
 
187
166
  operation = operation.to_sym
188
- read_dependencies.map! { |k| Promiscuous::Dependency.parse(k.to_s, :type => :read) }
189
167
  write_dependencies.map! { |k| Promiscuous::Dependency.parse(k.to_s, :type => :write) }
190
168
 
191
169
  begin
@@ -196,22 +174,19 @@ class Promiscuous::Publisher::Operation::Base
196
174
 
197
175
  Thread.new do
198
176
  # We run the recovery in another thread to ensure that we get a new
199
- # database connection to avoid tempering with the current state of the
177
+ # database connection to avoid tampering with the current state of the
200
178
  # connection, which can be in an open transaction.
201
179
  # Thankfully, we are not in a fast path.
202
180
  # Note that any exceptions will be passed through the thread join() method.
203
- Promiscuous.context :operation_recovery do
204
- op.instance_eval do
205
- @operation = operation
206
- @read_dependencies = read_dependencies
207
- @write_dependencies = write_dependencies
208
- @op_lock = lock
209
- @recovery_data = recovery_data
210
-
211
- query = Promiscuous::Publisher::Operation::ProxyForQuery.new(self) { recover_db_operation }
212
- self.execute_instrumented(query)
213
- query.result
214
- end
181
+ op.instance_eval do
182
+ @operation = operation
183
+ @write_dependencies = write_dependencies
184
+ @op_lock = lock
185
+ @recovery_data = recovery_data
186
+
187
+ query = Promiscuous::Publisher::Operation::ProxyForQuery.new(self) { recover_db_operation }
188
+ self.execute_instrumented(query)
189
+ query.result
215
190
  end
216
191
  end.join
217
192
 
@@ -221,32 +196,25 @@ class Promiscuous::Publisher::Operation::Base
221
196
  raise Promiscuous::Error::Recovery.new(message, e)
222
197
  end
223
198
 
224
- def increment_read_and_write_dependencies
199
+ def increment_dependencies
225
200
  # We collapse all operations, ignoring the read/write interleaving.
226
201
  # It doesn't matter since all write operations are serialized, so the first
227
202
  # write in the transaction can have all the read dependencies.
228
- r = read_dependencies
229
203
  w = write_dependencies
230
204
 
231
- # We don't need to do a read dependency if we are writing to it, so we
232
- # prune them. The subscriber assumes the pruning (i.e. the intersection of
233
- # r and w is empty) when it calculates the happens before relationships.
234
- r -= w
235
-
236
205
  master_node = @op_lock.node
237
206
  operation_recovery_key = "#{@op_lock.key}:operation_recovery"
238
207
 
239
208
  # We group all the dependencies by their respective shards
240
- # The master node will have the responsability to hold the recovery data.
241
- # We do the master node first. The seconaries can be done in parallel.
242
- @committed_read_deps = []
209
+ # The master node will have the responsibility to hold the recovery data.
210
+ # We do the master node first. The secondaries can be done in parallel.
243
211
  @committed_write_deps = []
244
212
 
245
213
  # We need to do the increments always in the same node order, otherwise.
246
214
  # the subscriber can deadlock. But we must always put the recovery payload
247
215
  # on the master before touching anything.
248
- nodes_deps = (w+r).group_by(&:redis_node)
249
- .sort_by { |node, deps| -Promiscuous::Redis.master.nodes.index(node) }
216
+ nodes_deps = w.group_by(&:redis_node)
217
+ .sort_by { |node, deps| -Promiscuous::Redis.master.nodes.index(node) }
250
218
  if nodes_deps.first[0] != master_node
251
219
  nodes_deps = [[master_node, []]] + nodes_deps
252
220
  end
@@ -256,12 +224,6 @@ class Promiscuous::Publisher::Operation::Base
256
224
  argv << Promiscuous::Key.new(:pub) # key prefixes
257
225
  argv << operation_recovery_key
258
226
 
259
- # The index of the first write is then used to pass to redis along with the
260
- # dependencies. This is done because arguments to redis LUA scripts cannot
261
- # accept complex data types.
262
- first_read_index = deps.index(&:read?) || deps.length
263
- argv << first_read_index
264
-
265
227
  # Each shard have their own recovery payload. The master recovery node
266
228
  # has the full operation recovery, and the others just have their versions.
267
229
  # Note that the operation_recovery_key on the secondaries have the current
@@ -269,7 +231,7 @@ class Promiscuous::Publisher::Operation::Base
269
231
  # locks get lost.
270
232
  if node == master_node && !self.recovering?
271
233
  # We are on the master node, which holds the recovery payload
272
- argv << MultiJson.dump([self.class.name, operation, r, w, self.recovery_payload])
234
+ argv << MultiJson.dump([self.class.name, operation, w, self.recovery_payload])
273
235
  end
274
236
 
275
237
  # FIXME If the lock is lost, we need to backoff
@@ -281,18 +243,12 @@ class Promiscuous::Publisher::Operation::Base
281
243
  local prefix = ARGV[1] .. ':'
282
244
  local operation_recovery_key = ARGV[2]
283
245
  local versions_recovery_key = operation_recovery_key .. ':versions'
284
- local first_read_index = tonumber(ARGV[3]) + 1
285
- local operation_recovery_payload = ARGV[4]
246
+ local operation_recovery_payload = ARGV[3]
286
247
  local deps = KEYS
287
248
 
288
249
  local versions = {}
289
250
 
290
251
  if redis.call('exists', versions_recovery_key) == 1 then
291
- first_read_index = tonumber(redis.call('hget', versions_recovery_key, 'read_index'))
292
- if not first_read_index then
293
- return redis.error_reply('Failed to read dependency index during recovery')
294
- end
295
-
296
252
  for i, dep in ipairs(deps) do
297
253
  versions[i] = tonumber(redis.call('hget', versions_recovery_key, dep))
298
254
  if not versions[i] then
@@ -300,26 +256,12 @@ class Promiscuous::Publisher::Operation::Base
300
256
  end
301
257
  end
302
258
 
303
- return { first_read_index-1, versions }
304
- end
305
-
306
- if redis.call('exists', prefix .. 'bootstrap') == 1 then
307
- first_read_index = #deps + 1
308
- end
309
-
310
- if #deps ~= 0 then
311
- redis.call('hset', versions_recovery_key, 'read_index', first_read_index)
259
+ return { versions }
312
260
  end
313
261
 
314
262
  for i, dep in ipairs(deps) do
315
263
  local key = prefix .. dep
316
- local rw_version = redis.call('incr', key .. ':rw')
317
- if i < first_read_index then
318
- redis.call('set', key .. ':w', rw_version)
319
- versions[i] = rw_version
320
- else
321
- versions[i] = tonumber(redis.call('get', key .. ':w')) or 0
322
- end
264
+ versions[i] = redis.call('incr', key .. ':w')
323
265
  redis.call('hset', versions_recovery_key, dep, versions[i])
324
266
  end
325
267
 
@@ -327,16 +269,14 @@ class Promiscuous::Publisher::Operation::Base
327
269
  redis.call('set', operation_recovery_key, operation_recovery_payload)
328
270
  end
329
271
 
330
- return { first_read_index-1, versions }
272
+ return { versions }
331
273
  SCRIPT
332
274
 
333
- received_first_read_index, versions = @@increment_script.eval(node, :argv => argv, :keys => deps)
275
+ versions = @@increment_script.eval(node, :argv => argv, :keys => deps)
334
276
 
335
277
  deps.zip(versions).each { |dep, version| dep.version = version }
336
278
 
337
- @committed_write_deps += deps[0...received_first_read_index]
338
- @committed_read_deps += deps[received_first_read_index..-1]
339
- @was_during_bootstrap = true if first_read_index != received_first_read_index
279
+ @committed_write_deps += deps
340
280
  end
341
281
 
342
282
  # The instance version must to be the first in the list to allow atomic
@@ -417,54 +357,24 @@ class Promiscuous::Publisher::Operation::Base
417
357
  def dependencies_for(instance, options={})
418
358
  return [] if instance.nil?
419
359
 
420
- if read?
421
- # We want to use the smallest subset that we can depend on when doing
422
- # reads. tracked_dependencies comes sorted from the smallest subset to
423
- # the largest. For maximum performance on the subscriber side, we thus
424
- # pick the first one. In most cases, it should resolve to the id
425
- # dependency.
426
- # If we don't have any, the driver should track individual instances.
427
- best_dependency = instance.promiscuous.tracked_dependencies(:allow_missing_attributes => true).first
428
- [best_dependency].compact
429
- else
430
- # Note that tracked_dependencies will not return the id dependency if it
431
- # doesn't exist which can only happen for create operations and auto
432
- # generated ids.
433
- instance.promiscuous.tracked_dependencies
434
- end
435
- end
436
-
437
- def read_dependencies
438
- # We memoize the read dependencies not just for performance, but also
439
- # because we store the versions once incremented in these.
440
- return @read_dependencies if @read_dependencies
441
- read_dependencies = current_context.read_operations.map(&:query_dependencies).flatten
442
-
443
- # We add extra_dependencies, which can contain the latest write, or user
444
- # context, etc.
445
- current_context.extra_dependencies.each do |dep|
446
- dep.version = nil
447
- read_dependencies << dep
448
- end
449
-
450
- @read_dependencies = read_dependencies.uniq.each { |d| d.type = :read }
360
+ # Note that tracked_dependencies will not return the id dependency if it
361
+ # doesn't exist which can only happen for create operations and auto
362
+ # generated ids.
363
+ [instance.promiscuous.get_dependency]
451
364
  end
452
- alias generate_read_dependencies read_dependencies
453
365
 
454
366
  def write_dependencies
455
367
  @write_dependencies ||= self.query_dependencies.uniq.each { |d| d.type = :write }
456
368
  end
457
369
 
458
370
  def should_instrument_query?
459
- # current_context is later enforced for writes.
460
- !Promiscuous.disabled? && (current_context || write?)
371
+ !Promiscuous.disabled?
461
372
  end
462
373
 
463
374
  def execute(&query_config)
464
375
  query = Promiscuous::Publisher::Operation::ProxyForQuery.new(self, &query_config)
465
376
 
466
377
  if should_instrument_query?
467
- raise Promiscuous::Error::MissingContext if !current_context && write?
468
378
  execute_instrumented(query)
469
379
  else
470
380
  query.call_and_remember_result(:non_instrumented)
@@ -507,7 +417,7 @@ class Promiscuous::Publisher::Operation::Base
507
417
  def trace_operation
508
418
  if ENV['TRACE']
509
419
  msg = self.explain_operation(70)
510
- current_context.trace(msg, :color => self.read? ? '0;32' : '1;31')
420
+ Promiscuous.context.trace(msg, :color => '1;31')
511
421
  end
512
422
  end
513
423
 
@@ -231,40 +231,6 @@ class Moped::PromiscuousQueryWrapper < Moped::Query
231
231
 
232
232
  # Moped::Query
233
233
 
234
- def count(*args)
235
- promiscuous_read_operation(:operation_ext => :count).execute { super }.to_i
236
- end
237
-
238
- def distinct(key)
239
- promiscuous_read_operation(:operation_ext => :distinct).execute { super }
240
- end
241
-
242
- def each
243
- # The TLS is used to pass arguments to the Cursor so we don't hijack more than
244
- # necessary.
245
- old_moped_query, Thread.current[:moped_query] = Thread.current[:moped_query], self
246
- super
247
- ensure
248
- Thread.current[:moped_query] = old_moped_query
249
- end
250
- alias :cursor :each
251
-
252
- def first
253
- # FIXME If the the user is using something like .only(), we need to make
254
- # sure that we add the id, otherwise we are screwed.
255
- op = promiscuous_read_operation
256
-
257
- op.execute do |query|
258
- query.non_instrumented { super }
259
- query.instrumented do
260
- super.tap do |doc|
261
- op.instances = doc ? [Mongoid::Factory.from_db(op.model, doc)] : []
262
- end
263
- end
264
- end
265
- end
266
- alias :one :first
267
-
268
234
  def update(change, flags=nil)
269
235
  update_op = promiscuous_write_operation(:update, :change => change)
270
236
 
@@ -303,37 +269,6 @@ class Moped::PromiscuousQueryWrapper < Moped::Query
303
269
  end
304
270
  end
305
271
 
306
- class Moped::PromiscuousCursorWrapper < Moped::Cursor
307
- # Moped::Cursor
308
- def promiscuous_read_each(&block)
309
- op = Moped::PromiscuousQueryWrapper::PromiscuousReadOperation.new(
310
- :query => @query, :operation_ext => :each)
311
-
312
- op.execute do |query|
313
- query.non_instrumented { block.call.to_a }
314
- query.instrumented do
315
- block.call.to_a.tap do |docs|
316
- op.instances = docs.map { |doc| Mongoid::Factory.from_db(op.model, doc) }
317
- end
318
- end
319
- end
320
- end
321
-
322
- def load_docs
323
- promiscuous_read_each { super }
324
- end
325
-
326
- def get_more
327
- # TODO support batch_size
328
- promiscuous_read_each { super }
329
- end
330
-
331
- def initialize(session, query_operation)
332
- super
333
- @query = Thread.current[:moped_query]
334
- end
335
- end
336
-
337
272
  class Moped::PromiscuousDatabase < Moped::Database
338
273
  # TODO it might be safer to use the alias attribute method because promiscuous
339
274
  # may come late in the loading.
@@ -364,7 +299,5 @@ Moped.__send__(:remove_const, :Collection)
364
299
  Moped.__send__(:const_set, :Collection, Moped::PromiscuousCollectionWrapper)
365
300
  Moped.__send__(:remove_const, :Query)
366
301
  Moped.__send__(:const_set, :Query, Moped::PromiscuousQueryWrapper)
367
- Moped.__send__(:remove_const, :Cursor)
368
- Moped.__send__(:const_set, :Cursor, Moped::PromiscuousCursorWrapper)
369
302
  Moped.__send__(:remove_const, :Database)
370
303
  Moped.__send__(:const_set, :Database, Moped::PromiscuousDatabase)
@@ -12,7 +12,6 @@ class Promiscuous::Publisher::Operation::NonPersistent < Promiscuous::Publisher:
12
12
  db_operation.call_and_remember_result(:instrumented)
13
13
 
14
14
  unless db_operation.failed?
15
- current_context.read_operations << self if read?
16
15
  trace_operation
17
16
  end
18
17
  end
@@ -40,7 +40,6 @@ class Promiscuous::Publisher::Operation::Transaction < Promiscuous::Publisher::O
40
40
 
41
41
  def execute_instrumented(query)
42
42
  unless self.recovering?
43
- generate_read_dependencies
44
43
  acquire_op_lock
45
44
 
46
45
  # As opposed to atomic operations, we know the values of the instances
@@ -51,7 +50,7 @@ class Promiscuous::Publisher::Operation::Transaction < Promiscuous::Publisher::O
51
50
  query.call_and_remember_result(:prepare)
52
51
  end
53
52
 
54
- self.increment_read_and_write_dependencies
53
+ self.increment_dependencies
55
54
 
56
55
  query.call_and_remember_result(:instrumented)
57
56
 
@@ -67,7 +66,6 @@ class Promiscuous::Publisher::Operation::Transaction < Promiscuous::Publisher::O
67
66
  ensure_op_still_locked
68
67
 
69
68
  generate_payload
70
- clear_previous_dependencies
71
69
 
72
70
  publish_payload_in_redis
73
71
  release_op_lock