promiscuous 0.92.0 → 0.100.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. checksums.yaml +4 -4
  2. data/lib/promiscuous.rb +2 -5
  3. data/lib/promiscuous/amqp.rb +1 -2
  4. data/lib/promiscuous/cli.rb +3 -43
  5. data/lib/promiscuous/config.rb +5 -7
  6. data/lib/promiscuous/error/dependency.rb +1 -3
  7. data/lib/promiscuous/publisher/context.rb +1 -1
  8. data/lib/promiscuous/publisher/context/base.rb +3 -34
  9. data/lib/promiscuous/publisher/model/base.rb +5 -25
  10. data/lib/promiscuous/publisher/model/mock.rb +5 -7
  11. data/lib/promiscuous/publisher/operation/active_record.rb +4 -69
  12. data/lib/promiscuous/publisher/operation/atomic.rb +1 -3
  13. data/lib/promiscuous/publisher/operation/base.rb +33 -123
  14. data/lib/promiscuous/publisher/operation/mongoid.rb +0 -67
  15. data/lib/promiscuous/publisher/operation/non_persistent.rb +0 -1
  16. data/lib/promiscuous/publisher/operation/transaction.rb +1 -3
  17. data/lib/promiscuous/railtie.rb +0 -31
  18. data/lib/promiscuous/subscriber.rb +1 -1
  19. data/lib/promiscuous/subscriber/{worker/message.rb → message.rb} +12 -40
  20. data/lib/promiscuous/subscriber/model/active_record.rb +1 -1
  21. data/lib/promiscuous/subscriber/model/base.rb +4 -4
  22. data/lib/promiscuous/subscriber/model/mongoid.rb +3 -3
  23. data/lib/promiscuous/subscriber/operation.rb +74 -3
  24. data/lib/promiscuous/subscriber/unit_of_work.rb +110 -0
  25. data/lib/promiscuous/subscriber/worker.rb +3 -7
  26. data/lib/promiscuous/subscriber/worker/eventual_destroyer.rb +2 -6
  27. data/lib/promiscuous/subscriber/worker/pump.rb +2 -11
  28. data/lib/promiscuous/version.rb +1 -1
  29. metadata +18 -36
  30. data/lib/promiscuous/error/missing_context.rb +0 -29
  31. data/lib/promiscuous/publisher/bootstrap.rb +0 -27
  32. data/lib/promiscuous/publisher/bootstrap/connection.rb +0 -25
  33. data/lib/promiscuous/publisher/bootstrap/data.rb +0 -127
  34. data/lib/promiscuous/publisher/bootstrap/mode.rb +0 -19
  35. data/lib/promiscuous/publisher/bootstrap/status.rb +0 -40
  36. data/lib/promiscuous/publisher/bootstrap/version.rb +0 -46
  37. data/lib/promiscuous/publisher/context/middleware.rb +0 -94
  38. data/lib/promiscuous/resque.rb +0 -12
  39. data/lib/promiscuous/sidekiq.rb +0 -15
  40. data/lib/promiscuous/subscriber/message_processor.rb +0 -4
  41. data/lib/promiscuous/subscriber/message_processor/base.rb +0 -54
  42. data/lib/promiscuous/subscriber/message_processor/bootstrap.rb +0 -17
  43. data/lib/promiscuous/subscriber/message_processor/regular.rb +0 -238
  44. data/lib/promiscuous/subscriber/operation/base.rb +0 -66
  45. data/lib/promiscuous/subscriber/operation/bootstrap.rb +0 -60
  46. data/lib/promiscuous/subscriber/operation/regular.rb +0 -19
  47. data/lib/promiscuous/subscriber/worker/message_synchronizer.rb +0 -333
@@ -76,7 +76,6 @@ class Promiscuous::Publisher::Operation::Atomic < Promiscuous::Publisher::Operat
76
76
  # this is a problem, but we need to publish.
77
77
  yell_about_missing_instance if @instance.nil?
78
78
  else
79
- generate_read_dependencies
80
79
  acquire_op_lock
81
80
 
82
81
  if @instance.nil?
@@ -93,7 +92,7 @@ class Promiscuous::Publisher::Operation::Atomic < Promiscuous::Publisher::Operat
93
92
  # old instance. This is a race that we tolerate.
94
93
  # XXX We also stash the document for create operations, so the recovery can
95
94
  # redo the create to avoid races when instances are getting partitioned.
96
- increment_read_and_write_dependencies
95
+ increment_dependencies
97
96
 
98
97
  # From this point, if we die, the one expiring our write locks must finish
99
98
  # the publish, either by sending a dummy, or by sending the real instance.
@@ -123,7 +122,6 @@ class Promiscuous::Publisher::Operation::Atomic < Promiscuous::Publisher::Operat
123
122
  ensure_op_still_locked
124
123
 
125
124
  generate_payload
126
- clear_previous_dependencies
127
125
 
128
126
  # As soon as we unlock the locks, the rescuer will not be able to assume
129
127
  # that the database instance is still pristine, and so we need to stash the
@@ -16,22 +16,10 @@ class Promiscuous::Publisher::Operation::Base
16
16
  @operation = options[:operation]
17
17
  end
18
18
 
19
- def read?
20
- @operation == :read
21
- end
22
-
23
- def write?
24
- !read?
25
- end
26
-
27
19
  def recovering?
28
20
  !!@recovery_data
29
21
  end
30
22
 
31
- def current_context
32
- @current_context ||= Promiscuous::Publisher::Context.current
33
- end
34
-
35
23
  def record_timestamp
36
24
  # Records the number of milliseconds since epoch, which we use send sending
37
25
  # the payload over. It's good for latency measurements.
@@ -96,7 +84,6 @@ class Promiscuous::Publisher::Operation::Base
96
84
 
97
85
  def publish_payload_in_redis
98
86
  # TODO Optimize and DRY this up
99
- r = @committed_read_deps
100
87
  w = @committed_write_deps
101
88
 
102
89
  # We identify a payload with a unique key (id:id_value:current_version:payload_recovery)
@@ -132,7 +119,7 @@ class Promiscuous::Publisher::Operation::Base
132
119
  # secondary_operation_recovery_key is unique to the operation.
133
120
  # XXX The caveat is that if we die here, the
134
121
  # secondary_operation_recovery_key will never be cleaned up.
135
- (w+r).map(&:redis_node).uniq
122
+ w.map(&:redis_node).uniq
136
123
  .reject { |node| node == master_node }
137
124
  .each { |node| node.del(versions_recovery_key) }
138
125
  end
@@ -148,28 +135,21 @@ class Promiscuous::Publisher::Operation::Base
148
135
  payload = {}
149
136
  payload[:operations] = operation_payloads
150
137
  payload[:app] = Promiscuous::Config.app
151
- payload[:context] = current_context.name
152
- payload[:current_user_id] = current_context.current_user.id if current_context.current_user
138
+ payload[:current_user_id] = Promiscuous.context.current_user.id if Promiscuous.context.current_user
153
139
  payload[:timestamp] = @timestamp
154
140
  payload[:generation] = Promiscuous::Config.generation
141
+ payload[:context] = "DEPRECATED"
155
142
  payload[:host] = Socket.gethostname
156
- payload[:was_during_bootstrap] = true if @was_during_bootstrap
157
143
  payload[:recovered_operation] = true if recovering?
158
144
  payload[:dependencies] = {}
159
- payload[:dependencies][:read] = @committed_read_deps if @committed_read_deps.present?
160
145
  payload[:dependencies][:write] = @committed_write_deps
161
146
 
162
147
  @payload = MultiJson.dump(payload)
163
148
  end
164
149
 
165
- def clear_previous_dependencies
166
- current_context.read_operations.clear
167
- current_context.extra_dependencies = [@committed_write_deps.first]
168
- end
169
-
170
150
  def self.recover_operation_from_lock(lock)
171
151
  # We happen to have acquired a never released lock.
172
- # The database instance is thus still prestine.
152
+ # The database instance is thus still pristine.
173
153
 
174
154
  master_node = lock.node
175
155
  recovery_data = master_node.get("#{lock.key}:operation_recovery")
@@ -181,11 +161,9 @@ class Promiscuous::Publisher::Operation::Base
181
161
 
182
162
  Promiscuous.info "[operation recovery] #{lock.key} -> #{recovery_data}"
183
163
 
184
- op_klass, operation, read_dependencies,
185
- write_dependencies, recovery_arguments = *MultiJson.load(recovery_data)
164
+ op_klass, operation, write_dependencies, recovery_arguments = *MultiJson.load(recovery_data)
186
165
 
187
166
  operation = operation.to_sym
188
- read_dependencies.map! { |k| Promiscuous::Dependency.parse(k.to_s, :type => :read) }
189
167
  write_dependencies.map! { |k| Promiscuous::Dependency.parse(k.to_s, :type => :write) }
190
168
 
191
169
  begin
@@ -196,22 +174,19 @@ class Promiscuous::Publisher::Operation::Base
196
174
 
197
175
  Thread.new do
198
176
  # We run the recovery in another thread to ensure that we get a new
199
- # database connection to avoid tempering with the current state of the
177
+ # database connection to avoid tampering with the current state of the
200
178
  # connection, which can be in an open transaction.
201
179
  # Thankfully, we are not in a fast path.
202
180
  # Note that any exceptions will be passed through the thread join() method.
203
- Promiscuous.context :operation_recovery do
204
- op.instance_eval do
205
- @operation = operation
206
- @read_dependencies = read_dependencies
207
- @write_dependencies = write_dependencies
208
- @op_lock = lock
209
- @recovery_data = recovery_data
210
-
211
- query = Promiscuous::Publisher::Operation::ProxyForQuery.new(self) { recover_db_operation }
212
- self.execute_instrumented(query)
213
- query.result
214
- end
181
+ op.instance_eval do
182
+ @operation = operation
183
+ @write_dependencies = write_dependencies
184
+ @op_lock = lock
185
+ @recovery_data = recovery_data
186
+
187
+ query = Promiscuous::Publisher::Operation::ProxyForQuery.new(self) { recover_db_operation }
188
+ self.execute_instrumented(query)
189
+ query.result
215
190
  end
216
191
  end.join
217
192
 
@@ -221,32 +196,25 @@ class Promiscuous::Publisher::Operation::Base
221
196
  raise Promiscuous::Error::Recovery.new(message, e)
222
197
  end
223
198
 
224
- def increment_read_and_write_dependencies
199
+ def increment_dependencies
225
200
  # We collapse all operations, ignoring the read/write interleaving.
226
201
  # It doesn't matter since all write operations are serialized, so the first
227
202
  # write in the transaction can have all the read dependencies.
228
- r = read_dependencies
229
203
  w = write_dependencies
230
204
 
231
- # We don't need to do a read dependency if we are writing to it, so we
232
- # prune them. The subscriber assumes the pruning (i.e. the intersection of
233
- # r and w is empty) when it calculates the happens before relationships.
234
- r -= w
235
-
236
205
  master_node = @op_lock.node
237
206
  operation_recovery_key = "#{@op_lock.key}:operation_recovery"
238
207
 
239
208
  # We group all the dependencies by their respective shards
240
- # The master node will have the responsability to hold the recovery data.
241
- # We do the master node first. The seconaries can be done in parallel.
242
- @committed_read_deps = []
209
+ # The master node will have the responsibility to hold the recovery data.
210
+ # We do the master node first. The secondaries can be done in parallel.
243
211
  @committed_write_deps = []
244
212
 
245
213
  # We need to do the increments always in the same node order, otherwise.
246
214
  # the subscriber can deadlock. But we must always put the recovery payload
247
215
  # on the master before touching anything.
248
- nodes_deps = (w+r).group_by(&:redis_node)
249
- .sort_by { |node, deps| -Promiscuous::Redis.master.nodes.index(node) }
216
+ nodes_deps = w.group_by(&:redis_node)
217
+ .sort_by { |node, deps| -Promiscuous::Redis.master.nodes.index(node) }
250
218
  if nodes_deps.first[0] != master_node
251
219
  nodes_deps = [[master_node, []]] + nodes_deps
252
220
  end
@@ -256,12 +224,6 @@ class Promiscuous::Publisher::Operation::Base
256
224
  argv << Promiscuous::Key.new(:pub) # key prefixes
257
225
  argv << operation_recovery_key
258
226
 
259
- # The index of the first write is then used to pass to redis along with the
260
- # dependencies. This is done because arguments to redis LUA scripts cannot
261
- # accept complex data types.
262
- first_read_index = deps.index(&:read?) || deps.length
263
- argv << first_read_index
264
-
265
227
  # Each shard have their own recovery payload. The master recovery node
266
228
  # has the full operation recovery, and the others just have their versions.
267
229
  # Note that the operation_recovery_key on the secondaries have the current
@@ -269,7 +231,7 @@ class Promiscuous::Publisher::Operation::Base
269
231
  # locks get lost.
270
232
  if node == master_node && !self.recovering?
271
233
  # We are on the master node, which holds the recovery payload
272
- argv << MultiJson.dump([self.class.name, operation, r, w, self.recovery_payload])
234
+ argv << MultiJson.dump([self.class.name, operation, w, self.recovery_payload])
273
235
  end
274
236
 
275
237
  # FIXME If the lock is lost, we need to backoff
@@ -281,18 +243,12 @@ class Promiscuous::Publisher::Operation::Base
281
243
  local prefix = ARGV[1] .. ':'
282
244
  local operation_recovery_key = ARGV[2]
283
245
  local versions_recovery_key = operation_recovery_key .. ':versions'
284
- local first_read_index = tonumber(ARGV[3]) + 1
285
- local operation_recovery_payload = ARGV[4]
246
+ local operation_recovery_payload = ARGV[3]
286
247
  local deps = KEYS
287
248
 
288
249
  local versions = {}
289
250
 
290
251
  if redis.call('exists', versions_recovery_key) == 1 then
291
- first_read_index = tonumber(redis.call('hget', versions_recovery_key, 'read_index'))
292
- if not first_read_index then
293
- return redis.error_reply('Failed to read dependency index during recovery')
294
- end
295
-
296
252
  for i, dep in ipairs(deps) do
297
253
  versions[i] = tonumber(redis.call('hget', versions_recovery_key, dep))
298
254
  if not versions[i] then
@@ -300,26 +256,12 @@ class Promiscuous::Publisher::Operation::Base
300
256
  end
301
257
  end
302
258
 
303
- return { first_read_index-1, versions }
304
- end
305
-
306
- if redis.call('exists', prefix .. 'bootstrap') == 1 then
307
- first_read_index = #deps + 1
308
- end
309
-
310
- if #deps ~= 0 then
311
- redis.call('hset', versions_recovery_key, 'read_index', first_read_index)
259
+ return { versions }
312
260
  end
313
261
 
314
262
  for i, dep in ipairs(deps) do
315
263
  local key = prefix .. dep
316
- local rw_version = redis.call('incr', key .. ':rw')
317
- if i < first_read_index then
318
- redis.call('set', key .. ':w', rw_version)
319
- versions[i] = rw_version
320
- else
321
- versions[i] = tonumber(redis.call('get', key .. ':w')) or 0
322
- end
264
+ versions[i] = redis.call('incr', key .. ':w')
323
265
  redis.call('hset', versions_recovery_key, dep, versions[i])
324
266
  end
325
267
 
@@ -327,16 +269,14 @@ class Promiscuous::Publisher::Operation::Base
327
269
  redis.call('set', operation_recovery_key, operation_recovery_payload)
328
270
  end
329
271
 
330
- return { first_read_index-1, versions }
272
+ return { versions }
331
273
  SCRIPT
332
274
 
333
- received_first_read_index, versions = @@increment_script.eval(node, :argv => argv, :keys => deps)
275
+ versions = @@increment_script.eval(node, :argv => argv, :keys => deps)
334
276
 
335
277
  deps.zip(versions).each { |dep, version| dep.version = version }
336
278
 
337
- @committed_write_deps += deps[0...received_first_read_index]
338
- @committed_read_deps += deps[received_first_read_index..-1]
339
- @was_during_bootstrap = true if first_read_index != received_first_read_index
279
+ @committed_write_deps += deps
340
280
  end
341
281
 
342
282
  # The instance version must to be the first in the list to allow atomic
@@ -417,54 +357,24 @@ class Promiscuous::Publisher::Operation::Base
417
357
  def dependencies_for(instance, options={})
418
358
  return [] if instance.nil?
419
359
 
420
- if read?
421
- # We want to use the smallest subset that we can depend on when doing
422
- # reads. tracked_dependencies comes sorted from the smallest subset to
423
- # the largest. For maximum performance on the subscriber side, we thus
424
- # pick the first one. In most cases, it should resolve to the id
425
- # dependency.
426
- # If we don't have any, the driver should track individual instances.
427
- best_dependency = instance.promiscuous.tracked_dependencies(:allow_missing_attributes => true).first
428
- [best_dependency].compact
429
- else
430
- # Note that tracked_dependencies will not return the id dependency if it
431
- # doesn't exist which can only happen for create operations and auto
432
- # generated ids.
433
- instance.promiscuous.tracked_dependencies
434
- end
435
- end
436
-
437
- def read_dependencies
438
- # We memoize the read dependencies not just for performance, but also
439
- # because we store the versions once incremented in these.
440
- return @read_dependencies if @read_dependencies
441
- read_dependencies = current_context.read_operations.map(&:query_dependencies).flatten
442
-
443
- # We add extra_dependencies, which can contain the latest write, or user
444
- # context, etc.
445
- current_context.extra_dependencies.each do |dep|
446
- dep.version = nil
447
- read_dependencies << dep
448
- end
449
-
450
- @read_dependencies = read_dependencies.uniq.each { |d| d.type = :read }
360
+ # Note that tracked_dependencies will not return the id dependency if it
361
+ # doesn't exist which can only happen for create operations and auto
362
+ # generated ids.
363
+ [instance.promiscuous.get_dependency]
451
364
  end
452
- alias generate_read_dependencies read_dependencies
453
365
 
454
366
  def write_dependencies
455
367
  @write_dependencies ||= self.query_dependencies.uniq.each { |d| d.type = :write }
456
368
  end
457
369
 
458
370
  def should_instrument_query?
459
- # current_context is later enforced for writes.
460
- !Promiscuous.disabled? && (current_context || write?)
371
+ !Promiscuous.disabled?
461
372
  end
462
373
 
463
374
  def execute(&query_config)
464
375
  query = Promiscuous::Publisher::Operation::ProxyForQuery.new(self, &query_config)
465
376
 
466
377
  if should_instrument_query?
467
- raise Promiscuous::Error::MissingContext if !current_context && write?
468
378
  execute_instrumented(query)
469
379
  else
470
380
  query.call_and_remember_result(:non_instrumented)
@@ -507,7 +417,7 @@ class Promiscuous::Publisher::Operation::Base
507
417
  def trace_operation
508
418
  if ENV['TRACE']
509
419
  msg = self.explain_operation(70)
510
- current_context.trace(msg, :color => self.read? ? '0;32' : '1;31')
420
+ Promiscuous.context.trace(msg, :color => '1;31')
511
421
  end
512
422
  end
513
423
 
@@ -231,40 +231,6 @@ class Moped::PromiscuousQueryWrapper < Moped::Query
231
231
 
232
232
  # Moped::Query
233
233
 
234
- def count(*args)
235
- promiscuous_read_operation(:operation_ext => :count).execute { super }.to_i
236
- end
237
-
238
- def distinct(key)
239
- promiscuous_read_operation(:operation_ext => :distinct).execute { super }
240
- end
241
-
242
- def each
243
- # The TLS is used to pass arguments to the Cursor so we don't hijack more than
244
- # necessary.
245
- old_moped_query, Thread.current[:moped_query] = Thread.current[:moped_query], self
246
- super
247
- ensure
248
- Thread.current[:moped_query] = old_moped_query
249
- end
250
- alias :cursor :each
251
-
252
- def first
253
- # FIXME If the the user is using something like .only(), we need to make
254
- # sure that we add the id, otherwise we are screwed.
255
- op = promiscuous_read_operation
256
-
257
- op.execute do |query|
258
- query.non_instrumented { super }
259
- query.instrumented do
260
- super.tap do |doc|
261
- op.instances = doc ? [Mongoid::Factory.from_db(op.model, doc)] : []
262
- end
263
- end
264
- end
265
- end
266
- alias :one :first
267
-
268
234
  def update(change, flags=nil)
269
235
  update_op = promiscuous_write_operation(:update, :change => change)
270
236
 
@@ -303,37 +269,6 @@ class Moped::PromiscuousQueryWrapper < Moped::Query
303
269
  end
304
270
  end
305
271
 
306
- class Moped::PromiscuousCursorWrapper < Moped::Cursor
307
- # Moped::Cursor
308
- def promiscuous_read_each(&block)
309
- op = Moped::PromiscuousQueryWrapper::PromiscuousReadOperation.new(
310
- :query => @query, :operation_ext => :each)
311
-
312
- op.execute do |query|
313
- query.non_instrumented { block.call.to_a }
314
- query.instrumented do
315
- block.call.to_a.tap do |docs|
316
- op.instances = docs.map { |doc| Mongoid::Factory.from_db(op.model, doc) }
317
- end
318
- end
319
- end
320
- end
321
-
322
- def load_docs
323
- promiscuous_read_each { super }
324
- end
325
-
326
- def get_more
327
- # TODO support batch_size
328
- promiscuous_read_each { super }
329
- end
330
-
331
- def initialize(session, query_operation)
332
- super
333
- @query = Thread.current[:moped_query]
334
- end
335
- end
336
-
337
272
  class Moped::PromiscuousDatabase < Moped::Database
338
273
  # TODO it might be safer to use the alias attribute method because promiscuous
339
274
  # may come late in the loading.
@@ -364,7 +299,5 @@ Moped.__send__(:remove_const, :Collection)
364
299
  Moped.__send__(:const_set, :Collection, Moped::PromiscuousCollectionWrapper)
365
300
  Moped.__send__(:remove_const, :Query)
366
301
  Moped.__send__(:const_set, :Query, Moped::PromiscuousQueryWrapper)
367
- Moped.__send__(:remove_const, :Cursor)
368
- Moped.__send__(:const_set, :Cursor, Moped::PromiscuousCursorWrapper)
369
302
  Moped.__send__(:remove_const, :Database)
370
303
  Moped.__send__(:const_set, :Database, Moped::PromiscuousDatabase)
@@ -12,7 +12,6 @@ class Promiscuous::Publisher::Operation::NonPersistent < Promiscuous::Publisher:
12
12
  db_operation.call_and_remember_result(:instrumented)
13
13
 
14
14
  unless db_operation.failed?
15
- current_context.read_operations << self if read?
16
15
  trace_operation
17
16
  end
18
17
  end
@@ -40,7 +40,6 @@ class Promiscuous::Publisher::Operation::Transaction < Promiscuous::Publisher::O
40
40
 
41
41
  def execute_instrumented(query)
42
42
  unless self.recovering?
43
- generate_read_dependencies
44
43
  acquire_op_lock
45
44
 
46
45
  # As opposed to atomic operations, we know the values of the instances
@@ -51,7 +50,7 @@ class Promiscuous::Publisher::Operation::Transaction < Promiscuous::Publisher::O
51
50
  query.call_and_remember_result(:prepare)
52
51
  end
53
52
 
54
- self.increment_read_and_write_dependencies
53
+ self.increment_dependencies
55
54
 
56
55
  query.call_and_remember_result(:instrumented)
57
56
 
@@ -67,7 +66,6 @@ class Promiscuous::Publisher::Operation::Transaction < Promiscuous::Publisher::O
67
66
  ensure_op_still_locked
68
67
 
69
68
  generate_payload
70
- clear_previous_dependencies
71
69
 
72
70
  publish_payload_in_redis
73
71
  release_op_lock