promiscuous 0.91.0 → 0.92.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. checksums.yaml +4 -4
  2. data/lib/promiscuous.rb +1 -1
  3. data/lib/promiscuous/amqp.rb +1 -1
  4. data/lib/promiscuous/amqp/fake.rb +0 -3
  5. data/lib/promiscuous/amqp/file.rb +81 -0
  6. data/lib/promiscuous/amqp/null.rb +0 -3
  7. data/lib/promiscuous/cli.rb +35 -25
  8. data/lib/promiscuous/config.rb +8 -3
  9. data/lib/promiscuous/error.rb +1 -2
  10. data/lib/promiscuous/key.rb +1 -12
  11. data/lib/promiscuous/mongoid.rb +7 -0
  12. data/lib/promiscuous/publisher/context/base.rb +4 -4
  13. data/lib/promiscuous/publisher/context/middleware.rb +2 -23
  14. data/lib/promiscuous/publisher/model/ephemeral.rb +5 -1
  15. data/lib/promiscuous/publisher/model/mock.rb +9 -7
  16. data/lib/promiscuous/publisher/model/mongoid.rb +3 -1
  17. data/lib/promiscuous/publisher/operation.rb +1 -1
  18. data/lib/promiscuous/publisher/operation/atomic.rb +44 -32
  19. data/lib/promiscuous/publisher/operation/base.rb +14 -9
  20. data/lib/promiscuous/publisher/operation/ephemeral.rb +14 -0
  21. data/lib/promiscuous/publisher/operation/mongoid.rb +4 -12
  22. data/lib/promiscuous/subscriber/message_processor/base.rb +17 -1
  23. data/lib/promiscuous/subscriber/message_processor/regular.rb +94 -48
  24. data/lib/promiscuous/subscriber/model/active_record.rb +25 -0
  25. data/lib/promiscuous/subscriber/model/base.rb +17 -13
  26. data/lib/promiscuous/subscriber/model/mongoid.rb +20 -1
  27. data/lib/promiscuous/subscriber/model/observer.rb +4 -0
  28. data/lib/promiscuous/subscriber/operation/base.rb +14 -16
  29. data/lib/promiscuous/subscriber/operation/bootstrap.rb +7 -1
  30. data/lib/promiscuous/subscriber/operation/regular.rb +6 -0
  31. data/lib/promiscuous/subscriber/worker.rb +6 -2
  32. data/lib/promiscuous/subscriber/worker/eventual_destroyer.rb +85 -0
  33. data/lib/promiscuous/subscriber/worker/message.rb +9 -15
  34. data/lib/promiscuous/subscriber/worker/message_synchronizer.rb +24 -78
  35. data/lib/promiscuous/subscriber/worker/runner.rb +6 -2
  36. data/lib/promiscuous/subscriber/worker/stats.rb +11 -7
  37. data/lib/promiscuous/version.rb +1 -1
  38. metadata +66 -63
  39. data/lib/promiscuous/error/already_processed.rb +0 -5
@@ -25,7 +25,7 @@ class Promiscuous::Publisher::Operation::Base
25
25
  end
26
26
 
27
27
  def recovering?
28
- !!@recovering
28
+ !!@recovery_data
29
29
  end
30
30
 
31
31
  def current_context
@@ -147,11 +147,14 @@ class Promiscuous::Publisher::Operation::Base
147
147
  def generate_payload
148
148
  payload = {}
149
149
  payload[:operations] = operation_payloads
150
- payload[:context] = current_context.name
151
150
  payload[:app] = Promiscuous::Config.app
151
+ payload[:context] = current_context.name
152
+ payload[:current_user_id] = current_context.current_user.id if current_context.current_user
152
153
  payload[:timestamp] = @timestamp
154
+ payload[:generation] = Promiscuous::Config.generation
153
155
  payload[:host] = Socket.gethostname
154
- payload[:current_user_id] = Thread.current[:promiscuous_context].try(:current_user_id)
156
+ payload[:was_during_bootstrap] = true if @was_during_bootstrap
157
+ payload[:recovered_operation] = true if recovering?
155
158
  payload[:dependencies] = {}
156
159
  payload[:dependencies][:read] = @committed_read_deps if @committed_read_deps.present?
157
160
  payload[:dependencies][:write] = @committed_write_deps
@@ -203,10 +206,10 @@ class Promiscuous::Publisher::Operation::Base
203
206
  @read_dependencies = read_dependencies
204
207
  @write_dependencies = write_dependencies
205
208
  @op_lock = lock
206
- @recovering = true
209
+ @recovery_data = recovery_data
207
210
 
208
211
  query = Promiscuous::Publisher::Operation::ProxyForQuery.new(self) { recover_db_operation }
209
- execute_instrumented(query)
212
+ self.execute_instrumented(query)
210
213
  query.result
211
214
  end
212
215
  end
@@ -256,7 +259,8 @@ class Promiscuous::Publisher::Operation::Base
256
259
  # The index of the first write is then used to pass to redis along with the
257
260
  # dependencies. This is done because arguments to redis LUA scripts cannot
258
261
  # accept complex data types.
259
- argv << (deps.index(&:read?) || deps.length)
262
+ first_read_index = deps.index(&:read?) || deps.length
263
+ argv << first_read_index
260
264
 
261
265
  # Each shard have their own recovery payload. The master recovery node
262
266
  # has the full operation recovery, and the others just have their versions.
@@ -326,12 +330,13 @@ class Promiscuous::Publisher::Operation::Base
326
330
  return { first_read_index-1, versions }
327
331
  SCRIPT
328
332
 
329
- first_read_index, versions = @@increment_script.eval(node, :argv => argv, :keys => deps)
333
+ received_first_read_index, versions = @@increment_script.eval(node, :argv => argv, :keys => deps)
330
334
 
331
335
  deps.zip(versions).each { |dep, version| dep.version = version }
332
336
 
333
- @committed_write_deps += deps[0...first_read_index]
334
- @committed_read_deps += deps[first_read_index..-1]
337
+ @committed_write_deps += deps[0...received_first_read_index]
338
+ @committed_read_deps += deps[received_first_read_index..-1]
339
+ @was_during_bootstrap = true if first_read_index != received_first_read_index
335
340
  end
336
341
 
337
342
  # The instance version must to be the first in the list to allow atomic
@@ -0,0 +1,14 @@
1
+ class Promiscuous::Publisher::Operation::Ephemeral < Promiscuous::Publisher::Operation::Atomic
2
+ def execute
3
+ super {}
4
+ end
5
+
6
+ def yell_about_missing_instance
7
+ # don't yell :)
8
+ end
9
+
10
+ def self.recover_operation(*recovery_payload)
11
+ # no instance when we recover, it's okay
12
+ new(:instance => nil)
13
+ end
14
+ end
@@ -157,16 +157,16 @@ class Moped::PromiscuousQueryWrapper < Moped::Query
157
157
  @query.selector = selector
158
158
  end
159
159
 
160
- def stash_version_in_document(version)
161
- @change['$set'] ||= {}
162
- @change['$set'][Promiscuous::Config.version_field] = version
160
+ def increment_version_in_document
161
+ @change['$inc'] ||= {}
162
+ @change['$inc'][Promiscuous::Config.version_field] = 1
163
163
  end
164
164
 
165
165
  def execute_instrumented(query)
166
166
  # We are trying to be optimistic for the locking. We are trying to figure
167
167
  # out our dependencies with the selector upfront to avoid an extra read
168
168
  # from reload_instance.
169
- @instance ||= get_selector_instance
169
+ @instance ||= get_selector_instance unless recovering? && operation == :update
170
170
  super
171
171
  end
172
172
 
@@ -360,14 +360,6 @@ class Mongoid::Validations::UniquenessValidator
360
360
  end
361
361
  end
362
362
 
363
- class Moped::BSON::ObjectId
364
- # No {"$oid": "123"}, it's horrible.
365
- # TODO Document this shit.
366
- def to_json(*args)
367
- "\"#{to_s}\""
368
- end
369
- end
370
-
371
363
  Moped.__send__(:remove_const, :Collection)
372
364
  Moped.__send__(:const_set, :Collection, Moped::PromiscuousCollectionWrapper)
373
365
  Moped.__send__(:remove_const, :Query)
@@ -14,7 +14,7 @@ class Promiscuous::Subscriber::MessageProcessor::Base
14
14
 
15
15
  begin
16
16
  self.current = new(*args)
17
- self.current.on_message
17
+ self.current.process_message
18
18
  ensure
19
19
  self.current = nil
20
20
  end
@@ -28,6 +28,22 @@ class Promiscuous::Subscriber::MessageProcessor::Base
28
28
  Thread.current[:promiscuous_message_processor] = value
29
29
  end
30
30
 
31
+ def process_message
32
+ begin
33
+ on_message
34
+ rescue Exception => e
35
+ @fail_count ||= 0; @fail_count += 1
36
+
37
+ if @fail_count <= Promiscuous::Config.max_retries
38
+ Promiscuous.warn("[receive] #{e.message} #{@fail_count.ordinalize} retry: #{@message}")
39
+ sleep @fail_count ** 2
40
+ process_message
41
+ else
42
+ raise e
43
+ end
44
+ end
45
+ end
46
+
31
47
  def on_message
32
48
  raise "Must be implemented"
33
49
  end
@@ -33,32 +33,14 @@ class Promiscuous::Subscriber::MessageProcessor::Regular < Promiscuous::Subscrib
33
33
 
34
34
  # XXX TODO Code is not tolerant to losing a lock.
35
35
 
36
- def update_dependencies_on_node(node_with_deps, options={})
37
- # Read and write dependencies are not handled the same way:
38
- # * Read dependencies are just incremented (which allow parallelization).
39
- # * Write dependencies are set to be max(current_version, received_version).
40
- # This allow the version bootstrapping process to be non-atomic.
41
- # Publishers upgrade their reads dependencies to write dependencies
42
- # during bootstrapping to permit the mechanism to function properly.
43
-
44
- # TODO Evaluate the performance hit of this heavy mechanism, and see if it's
45
- # worth optimizing it for the non-bootstrap case.
46
-
47
- node = node_with_deps[0]
48
- r_deps = node_with_deps[1].select(&:read?)
49
- w_deps = node_with_deps[1].select(&:write?)
50
-
51
- if options[:only_write_dependencies]
52
- r_deps = []
53
- end
54
-
36
+ def update_dependencies_non_atomic_bootstrap(node, r_deps, w_deps, options={})
55
37
  argv = []
56
38
  argv << MultiJson.dump([r_deps.map { |dep| dep.key(:sub) },
57
39
  w_deps.map { |dep| dep.key(:sub) },
58
40
  w_deps.map { |dep| dep.version }])
59
41
  argv << recovery_key if options[:with_recovery]
60
42
 
61
- @@update_script_secondary ||= Promiscuous::Redis::Script.new <<-SCRIPT
43
+ @@update_script_bootstrap ||= Promiscuous::Redis::Script.new <<-SCRIPT
62
44
  local _args = cjson.decode(ARGV[1])
63
45
  local read_deps = _args[1]
64
46
  local write_deps = _args[2]
@@ -90,7 +72,56 @@ class Promiscuous::Subscriber::MessageProcessor::Regular < Promiscuous::Subscrib
90
72
  end
91
73
  SCRIPT
92
74
 
93
- @@update_script_secondary.eval(node, :argv => argv)
75
+ @@update_script_bootstrap.eval(node, :argv => argv)
76
+ end
77
+
78
+ def update_dependencies_fast(node, r_deps, w_deps, options={})
79
+ keys = (r_deps + w_deps).map { |dep| dep.key(:sub) }
80
+ argv = options[:with_recovery] ? [recovery_key] : []
81
+
82
+ @@update_script_fast ||= Promiscuous::Redis::Script.new <<-SCRIPT
83
+ local deps = KEYS
84
+ local recovery_key = ARGV[1]
85
+
86
+ if recovery_key and redis.call('exists', recovery_key) == 1 then
87
+ return
88
+ end
89
+
90
+ for i, _key in ipairs(deps) do
91
+ local key = _key .. ':rw'
92
+ local v = redis.call('incr', key)
93
+ redis.call('publish', key, v)
94
+ end
95
+
96
+ if recovery_key then
97
+ redis.call('set', recovery_key, 'done')
98
+ end
99
+ SCRIPT
100
+
101
+ @@update_script_fast.eval(node, :keys => keys, :argv => argv)
102
+ end
103
+
104
+ def update_dependencies_on_node(node_with_deps, options={})
105
+ # Read and write dependencies are not handled the same way:
106
+ # * Read dependencies are just incremented (which allow parallelization).
107
+ # * Write dependencies are set to be max(current_version, received_version).
108
+ # This allow the version bootstrapping process to be non-atomic.
109
+ # Publishers upgrade their reads dependencies to write dependencies
110
+ # during bootstrapping to permit the mechanism to function properly.
111
+
112
+ # TODO Evaluate the performance hit of this heavy mechanism, and see if it's
113
+ # worth optimizing it for the non-bootstrap case.
114
+
115
+ node = node_with_deps[0]
116
+ r_deps = node_with_deps[1].select(&:read?)
117
+ w_deps = node_with_deps[1].select(&:write?)
118
+
119
+ if message.was_during_bootstrap?
120
+ raise "Message should not have any read deps" unless r_deps.empty?
121
+ update_dependencies_non_atomic_bootstrap(node, r_deps, w_deps, options)
122
+ else
123
+ update_dependencies_fast(node, r_deps, w_deps, options)
124
+ end
94
125
  end
95
126
 
96
127
  def update_dependencies_master(options={})
@@ -124,7 +155,7 @@ class Promiscuous::Subscriber::MessageProcessor::Regular < Promiscuous::Subscrib
124
155
  cleanup_dependency_secondaries
125
156
  end
126
157
 
127
- def check_for_duplicated_message
158
+ def duplicate_message?
128
159
  unless instance_dep.version >= get_current_instance_version + 1
129
160
  # We happen to get a duplicate message, or we are recovering a dead
130
161
  # worker. During regular operations, we just need to cleanup the 2pc (from
@@ -142,12 +173,11 @@ class Promiscuous::Subscriber::MessageProcessor::Regular < Promiscuous::Subscrib
142
173
  # seldom: either (1) the publisher recovered a payload that didn't need
143
174
  # recovery, or (2) a subscriber worker died after # update_dependencies_master,
144
175
  # but before the message acking).
145
- # It is thus okay to assume the worse and be inefficient.
146
- update_dependencies(:only_write_dependencies => true)
147
-
148
- message.ack
176
+ update_dependencies if message.was_during_bootstrap?
149
177
 
150
- raise Promiscuous::Error::AlreadyProcessed
178
+ true
179
+ else
180
+ false
151
181
  end
152
182
  end
153
183
 
@@ -155,35 +185,51 @@ class Promiscuous::Subscriber::MessageProcessor::Regular < Promiscuous::Subscrib
155
185
  :sleep => 0.1, # polling every 100ms.
156
186
  :expire => 1.minute } # after one minute, we are considered dead
157
187
 
158
- def synchronize_and_update_dependencies
159
- if Promiscuous::Config.bootstrap
160
- else
161
- lock_options = LOCK_OPTIONS.merge(:node => master_node)
162
- mutex = Promiscuous::Redis::Mutex.new(instance_dep.key(:sub).to_s, lock_options)
188
+ def check_duplicate_and_update_dependencies
189
+ if duplicate_message?
190
+ Promiscuous.debug "[receive] Skipping message (already processed) #{message}"
191
+ return
192
+ end
163
193
 
164
- unless mutex.lock
165
- raise Promiscuous::Error::LockUnavailable.new(mutex.key)
166
- end
194
+ yield
167
195
 
168
- begin
169
- check_for_duplicated_message
170
- yield
171
- update_dependencies
172
- message.ack
173
- ensure
174
- unless mutex.unlock
175
- # TODO Be safe in case we have a duplicate message and lost the lock on it
176
- raise "The subscriber lost the lock during its operation. It means that someone else\n"+
177
- "received a duplicate message, and we got screwed.\n"
178
- end
196
+ update_dependencies
197
+ end
198
+
199
+ def with_instance_locked(&block)
200
+ return yield unless message.has_dependencies?
201
+
202
+ lock_options = LOCK_OPTIONS.merge(:node => master_node)
203
+ mutex = Promiscuous::Redis::Mutex.new(instance_dep.key(:sub).to_s, lock_options)
204
+
205
+ unless mutex.lock
206
+ raise Promiscuous::Error::LockUnavailable.new(mutex.key)
207
+ end
208
+
209
+ begin
210
+ yield
211
+ ensure
212
+ unless mutex.unlock
213
+ # TODO Be safe in case we have a duplicate message and lost the lock on it
214
+ raise "The subscriber lost the lock during its operation. It means that someone else\n"+
215
+ "received a duplicate message, and we got screwed.\n"
179
216
  end
180
217
  end
181
218
  end
182
219
 
220
+ def execute_operations
221
+ self.operations.each(&:execute)
222
+ end
223
+
183
224
  def on_message
184
- self.synchronize_and_update_dependencies do
185
- self.operations.each(&:execute)
225
+ with_instance_locked do
226
+ if Promiscuous::Config.consistency == :causal && message.has_dependencies?
227
+ self.check_duplicate_and_update_dependencies { execute_operations }
228
+ else
229
+ execute_operations
230
+ end
186
231
  end
232
+ message.ack
187
233
  end
188
234
 
189
235
  def operation_class
@@ -2,9 +2,34 @@ module Promiscuous::Subscriber::Model::ActiveRecord
2
2
  extend ActiveSupport::Concern
3
3
  include Promiscuous::Subscriber::Model::Base
4
4
 
5
+ included do
6
+ if Promiscuous::Config.consistency == :eventual && !self.columns.collect(&:name).include?("_v")
7
+ raise <<-help
8
+ #{self} must include a _v column. Create the following migration:
9
+ change_table :#{self.table_name} do |t|
10
+ t.integer :_v, :limit => 8
11
+ end
12
+ help
13
+ end
14
+ end
15
+
5
16
  module ClassMethods
6
17
  def __promiscuous_missing_record_exception
7
18
  ActiveRecord::RecordNotFound
8
19
  end
20
+
21
+ def __promiscuous_duplicate_key_exception?(e)
22
+ # TODO Ensure that it's on the pk
23
+ e.is_a?(ActiveRecord::RecordNotUnique)
24
+ end
25
+
26
+ def __promiscuous_fetch_existing(id)
27
+ key = subscribe_foreign_key
28
+ if promiscuous_root_class.respond_to?("find_by_#{key}!")
29
+ promiscuous_root_class.__send__("find_by_#{key}!", id)
30
+ elsif respond_to?("find_by")
31
+ promiscuous_root_class.find_by(key => id)
32
+ end
33
+ end
9
34
  end
10
35
  end
@@ -1,6 +1,23 @@
1
1
  module Promiscuous::Subscriber::Model::Base
2
2
  extend ActiveSupport::Concern
3
3
 
4
+ def __promiscuous_eventual_consistency_update(operation)
5
+ return true unless Promiscuous::Config.consistency == :eventual
6
+ return true unless operation.message.has_dependencies?
7
+
8
+ version = operation.message_processor.instance_dep.version
9
+ generation = operation.message.generation
10
+ version = (generation << 50) | version
11
+
12
+ if self.attributes[Promiscuous::Config.version_field].to_i <= version
13
+ self.send("#{Promiscuous::Config.version_field}=", version)
14
+ true
15
+ else
16
+ Promiscuous.debug "[receive] out of order message #{self.class}/#{id}/g:#{generation},v:#{version}"
17
+ false
18
+ end
19
+ end
20
+
4
21
  def __promiscuous_update(payload, options={})
5
22
  self.class.subscribed_attrs.map(&:to_s).each do |attr|
6
23
  unless payload.attributes.has_key?(attr)
@@ -84,18 +101,5 @@ module Promiscuous::Subscriber::Model::Base
84
101
  def __promiscuous_fetch_new(id)
85
102
  new.tap { |m| m.__send__("#{subscribe_foreign_key}=", id) }
86
103
  end
87
-
88
- def __promiscuous_fetch_existing(id)
89
- key = subscribe_foreign_key
90
- if promiscuous_root_class.respond_to?("find_by_#{key}!")
91
- promiscuous_root_class.__send__("find_by_#{key}!", id)
92
- elsif respond_to?("find_by")
93
- promiscuous_root_class.find_by(key => id)
94
- else
95
- instance = promiscuous_root_class.where(key => id).first
96
- raise __promiscuous_missing_record_exception.new(promiscuous_root_class, id) if instance.nil?
97
- instance
98
- end
99
- end
100
104
  end
101
105
  end
@@ -8,6 +8,10 @@ module Promiscuous::Subscriber::Model::Mongoid
8
8
  !self.embedded? || options[:old_value] != self
9
9
  end
10
10
 
11
+ included do
12
+ field :_v if Promiscuous::Config.consistency == :eventual
13
+ end
14
+
11
15
  module ClassMethods
12
16
  def subscribe(*args, &block)
13
17
  super
@@ -38,6 +42,21 @@ module Promiscuous::Subscriber::Model::Mongoid
38
42
  def __promiscuous_missing_record_exception
39
43
  Mongoid::Errors::DocumentNotFound
40
44
  end
45
+
46
+ def __promiscuous_duplicate_key_exception?(e)
47
+ e.message =~ /E11000/
48
+ end
49
+
50
+ def __promiscuous_fetch_existing(id)
51
+ key = subscribe_foreign_key
52
+ if respond_to?("find_by")
53
+ promiscuous_root_class.find_by(key => id)
54
+ else
55
+ instance = promiscuous_root_class.where(key => id).first
56
+ raise __promiscuous_missing_record_exception.new(promiscuous_root_class, id) if instance.nil?
57
+ instance
58
+ end
59
+ end
41
60
  end
42
61
 
43
62
  class EmbeddedDocs
@@ -71,7 +90,7 @@ module Promiscuous::Subscriber::Model::Mongoid
71
90
  # create all the new ones
72
91
  new_embeddeds.reject { |new_e| new_e['existed'] }.each do |new_e|
73
92
  payload = Promiscuous::Subscriber::Operation::Regular.new(new_e)
74
- new_e_instance = payload.model. __promiscuous_fetch_new(payload.id)
93
+ new_e_instance = payload.model.__promiscuous_fetch_new(payload.id)
75
94
  new_e_instance.__promiscuous_update(payload)
76
95
  options[:parent].__send__(old_embeddeds.metadata[:name]) << new_e_instance
77
96
  end