promiscuous 0.90.0 → 0.91.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (65) hide show
  1. checksums.yaml +7 -0
  2. data/lib/promiscuous/amqp/bunny.rb +63 -36
  3. data/lib/promiscuous/amqp/fake.rb +3 -1
  4. data/lib/promiscuous/amqp/hot_bunnies.rb +26 -16
  5. data/lib/promiscuous/amqp/null.rb +1 -0
  6. data/lib/promiscuous/amqp.rb +12 -12
  7. data/lib/promiscuous/cli.rb +70 -29
  8. data/lib/promiscuous/config.rb +54 -29
  9. data/lib/promiscuous/convenience.rb +1 -1
  10. data/lib/promiscuous/dependency.rb +25 -6
  11. data/lib/promiscuous/error/connection.rb +11 -9
  12. data/lib/promiscuous/error/dependency.rb +8 -1
  13. data/lib/promiscuous/loader.rb +4 -2
  14. data/lib/promiscuous/publisher/bootstrap/connection.rb +25 -0
  15. data/lib/promiscuous/publisher/bootstrap/data.rb +127 -0
  16. data/lib/promiscuous/publisher/bootstrap/mode.rb +19 -0
  17. data/lib/promiscuous/publisher/bootstrap/status.rb +40 -0
  18. data/lib/promiscuous/publisher/bootstrap/version.rb +46 -0
  19. data/lib/promiscuous/publisher/bootstrap.rb +27 -0
  20. data/lib/promiscuous/publisher/context/base.rb +67 -0
  21. data/lib/promiscuous/{middleware.rb → publisher/context/middleware.rb} +16 -13
  22. data/lib/promiscuous/publisher/context/transaction.rb +36 -0
  23. data/lib/promiscuous/publisher/context.rb +4 -88
  24. data/lib/promiscuous/publisher/mock_generator.rb +9 -9
  25. data/lib/promiscuous/publisher/model/active_record.rb +7 -7
  26. data/lib/promiscuous/publisher/model/base.rb +29 -29
  27. data/lib/promiscuous/publisher/model/ephemeral.rb +5 -3
  28. data/lib/promiscuous/publisher/model/mock.rb +9 -5
  29. data/lib/promiscuous/publisher/model/mongoid.rb +5 -22
  30. data/lib/promiscuous/publisher/operation/active_record.rb +360 -0
  31. data/lib/promiscuous/publisher/operation/atomic.rb +167 -0
  32. data/lib/promiscuous/publisher/operation/base.rb +279 -474
  33. data/lib/promiscuous/publisher/operation/mongoid.rb +153 -145
  34. data/lib/promiscuous/publisher/operation/non_persistent.rb +28 -0
  35. data/lib/promiscuous/publisher/operation/proxy_for_query.rb +42 -0
  36. data/lib/promiscuous/publisher/operation/transaction.rb +85 -0
  37. data/lib/promiscuous/publisher/operation.rb +1 -1
  38. data/lib/promiscuous/publisher/worker.rb +7 -7
  39. data/lib/promiscuous/publisher.rb +1 -1
  40. data/lib/promiscuous/railtie.rb +20 -5
  41. data/lib/promiscuous/redis.rb +104 -56
  42. data/lib/promiscuous/subscriber/message_processor/base.rb +38 -0
  43. data/lib/promiscuous/subscriber/message_processor/bootstrap.rb +17 -0
  44. data/lib/promiscuous/subscriber/message_processor/regular.rb +192 -0
  45. data/lib/promiscuous/subscriber/message_processor.rb +4 -0
  46. data/lib/promiscuous/subscriber/model/base.rb +20 -15
  47. data/lib/promiscuous/subscriber/model/mongoid.rb +4 -4
  48. data/lib/promiscuous/subscriber/model/observer.rb +16 -2
  49. data/lib/promiscuous/subscriber/operation/base.rb +68 -0
  50. data/lib/promiscuous/subscriber/operation/bootstrap.rb +54 -0
  51. data/lib/promiscuous/subscriber/operation/regular.rb +13 -0
  52. data/lib/promiscuous/subscriber/operation.rb +3 -166
  53. data/lib/promiscuous/subscriber/worker/message.rb +61 -35
  54. data/lib/promiscuous/subscriber/worker/message_synchronizer.rb +90 -59
  55. data/lib/promiscuous/subscriber/worker/pump.rb +17 -5
  56. data/lib/promiscuous/subscriber/worker/recorder.rb +4 -1
  57. data/lib/promiscuous/subscriber/worker/runner.rb +49 -9
  58. data/lib/promiscuous/subscriber/worker/stats.rb +2 -2
  59. data/lib/promiscuous/subscriber/worker.rb +6 -0
  60. data/lib/promiscuous/subscriber.rb +1 -1
  61. data/lib/promiscuous/timer.rb +31 -18
  62. data/lib/promiscuous/version.rb +1 -1
  63. data/lib/promiscuous.rb +23 -3
  64. metadata +104 -89
  65. data/lib/promiscuous/subscriber/payload.rb +0 -34
@@ -0,0 +1,68 @@
1
+ class Promiscuous::Subscriber::Operation::Base
2
+ attr_accessor :model, :id, :operation, :attributes
3
+ delegate :message, :to => :message_processor
4
+
5
+ def initialize(payload)
6
+ if payload.is_a?(Hash)
7
+ self.id = payload['id']
8
+ self.operation = payload['operation'].try(:to_sym)
9
+ self.attributes = payload['attributes']
10
+ self.model = self.get_subscribed_model(payload) if payload['types']
11
+ end
12
+ end
13
+
14
+ def get_subscribed_model(payload)
15
+ [message.app, '*'].each do |app|
16
+ app_mapping = Promiscuous::Subscriber::Model.mapping[app] || {}
17
+ payload['types'].to_a.each do |ancestor|
18
+ model = app_mapping[ancestor]
19
+ return model if model
20
+ end
21
+ end
22
+ nil
23
+ end
24
+
25
+ def warn(msg)
26
+ Promiscuous.warn "[receive] #{msg} #{message.payload}"
27
+ end
28
+
29
+ def create(options={})
30
+ model.__promiscuous_fetch_new(id).tap do |instance|
31
+ instance.__promiscuous_update(self)
32
+ instance.save!
33
+ end
34
+ rescue Exception => e
35
+ # TODO Abstract the duplicated index error message
36
+ dup_index_error = true if defined?(Mongoid) && e.message =~ /E11000/
37
+ # # TODO Ensure that it's on the pk
38
+ dup_index_error = true if defined?(ActiveRecord) && e.is_a?(ActiveRecord::RecordNotUnique)
39
+
40
+ if dup_index_error
41
+ if options[:upsert]
42
+ update
43
+ else
44
+ warn "ignoring already created record"
45
+ end
46
+ else
47
+ raise e
48
+ end
49
+ end
50
+
51
+ def update
52
+ model.__promiscuous_fetch_existing(id).tap do |instance|
53
+ instance.__promiscuous_update(self)
54
+ instance.save!
55
+ end
56
+ rescue model.__promiscuous_missing_record_exception
57
+ warn "upserting #{message.payload}"
58
+ create
59
+ end
60
+
61
+ def destroy
62
+ model.__promiscuous_fetch_existing(id).tap do |instance|
63
+ instance.destroy
64
+ end
65
+ rescue model.__promiscuous_missing_record_exception
66
+ warn "ignoring missing record"
67
+ end
68
+ end
@@ -0,0 +1,54 @@
1
+ class Promiscuous::Subscriber::Operation::Bootstrap < Promiscuous::Subscriber::Operation::Base
2
+ # TODO Here's what's left to do:
3
+ # - Automatic switching from pass1, pass2, live
4
+ # - Unbinding the bootstrap exchange when going live, and reset prefetch
5
+ # during the version bootstrap phase.
6
+ # - CLI interface and progress bars
7
+
8
+ def bootstrap_versions
9
+ operations = message.parsed_payload['operations']
10
+
11
+ operations.map { |op| op['keys'] }.flatten.map { |k| Promiscuous::Dependency.parse(k, :owner => message.app) }.group_by(&:redis_node).each do |node, deps|
12
+ node.mset(deps.map { |dep| [dep.key(:sub).join('rw').to_s, dep.version] }.flatten)
13
+ end
14
+ end
15
+
16
+ def bootstrap_data
17
+ create(:upsert => true)
18
+ end
19
+
20
+ def on_bootstrap_operation(wanted_operation, options={})
21
+ if operation == wanted_operation
22
+ yield
23
+ options[:always_postpone] ? message.postpone : message.ack
24
+ else
25
+ message.postpone
26
+ end
27
+ end
28
+
29
+ def execute
30
+ case Promiscuous::Config.bootstrap
31
+ when :pass1
32
+ # The first thing to do is to receive and save an non atomic snapshot of
33
+ # the publisher's versions.
34
+ on_bootstrap_operation(:bootstrap_versions) { bootstrap_versions }
35
+
36
+ when :pass2
37
+ # Then we move on to save the raw data, but skipping the message if we get
38
+ # a mismatch on the version.
39
+ on_bootstrap_operation(:bootstrap_data) { bootstrap_data }
40
+
41
+ when :pass3
42
+ # Finally, we create the rows that we've skipped, we postpone them to make
43
+ # our lives easier. We'll detect the message as duplicates when re-processed.
44
+ # on_bootstrap_operation(:update, :always_postpone => true) { bootstrap_missing_data if model }
45
+ # TODO unbind the bootstrap exchange
46
+ else
47
+ raise "Invalid operation received: #{operation}"
48
+ end
49
+ end
50
+
51
+ def message_processor
52
+ @message_processor ||= Promiscuous::Subscriber::MessageProcessor::Bootstrap.current
53
+ end
54
+ end
@@ -0,0 +1,13 @@
1
+ class Promiscuous::Subscriber::Operation::Regular < Promiscuous::Subscriber::Operation::Base
2
+ def execute
3
+ case operation
4
+ when :create then create if model
5
+ when :update then update if model
6
+ when :destroy then destroy if model
7
+ end
8
+ end
9
+
10
+ def message_processor
11
+ @message_processor ||= Promiscuous::Subscriber::MessageProcessor::Regular.current
12
+ end
13
+ end
@@ -1,167 +1,4 @@
1
- class Promiscuous::Subscriber::Operation
2
- attr_accessor :payload
3
-
4
- delegate :model, :id, :operation, :message, :to => :payload
5
-
6
- def initialize(payload)
7
- self.payload = payload
8
- end
9
-
10
- # XXX TODO Code is not tolerent to losing a lock.
11
-
12
- def update_dependencies_single(node_with_deps)
13
- master_node = node_with_deps[0]
14
- deps = node_with_deps[1]
15
-
16
- @@update_script ||= Promiscuous::Redis::Script.new <<-SCRIPT
17
- for i, key in ipairs(KEYS) do
18
- local v = redis.call('incr', key .. ':rw')
19
- redis.call('publish', key .. ':rw', v)
20
- end
21
- SCRIPT
22
- keys = deps.map { |dep| dep.key(:sub).to_s }
23
- @@update_script.eval(master_node, :keys => keys)
24
- end
25
-
26
- def update_dependencies_multi(nodes_with_deps, options={})
27
- # With multi nodes, we have to do a 2pc for the lock recovery mechanism:
28
- # 1) We do the secondaries first, with a recovery payload.
29
- # 2) Then we do the master.
30
- # 3) Then we cleanup the secondaries.
31
- # We use a recovery_key unique to the operation to avoid any trouble of
32
- # touching another operation.
33
- secondary_nodes_with_deps = nodes_with_deps[1..-1]
34
- recovery_key = @instance_dep.key(:sub).join(@instance_dep.version).to_s
35
-
36
- secondary_nodes_with_deps.each do |node, deps|
37
- @@update_script_secondary ||= Promiscuous::Redis::Script.new <<-SCRIPT
38
- local recovery_key = ARGV[1]
39
-
40
- if redis.call('get', recovery_key) == 'done' then
41
- return
42
- end
43
-
44
- for i, key in ipairs(KEYS) do
45
- local v = redis.call('incr', key .. ':rw')
46
- redis.call('publish', key .. ':rw', v)
47
- end
48
-
49
- redis.call('set', recovery_key, 'done')
50
- SCRIPT
51
- keys = deps.map { |dep| dep.key(:sub).to_s }
52
- @@update_script_secondary.eval(node, :keys => keys, :argv => [recovery_key])
53
- after_secondary_update_hook
54
- end
55
-
56
- update_dependencies_single(nodes_with_deps.first) unless options[:skip_master]
57
-
58
- secondary_nodes_with_deps.each do |node, deps|
59
- node.del(recovery_key)
60
- end
61
- end
62
-
63
- def after_secondary_update_hook; end # for tests
64
-
65
- def update_dependencies
66
- nodes_with_deps.size == 1 ? update_dependencies_single(nodes_with_deps.first) :
67
- update_dependencies_multi(nodes_with_deps)
68
- end
69
-
70
- def verify_dependencies
71
- key = @instance_dep.key(:sub).join('rw').to_s
72
-
73
- if @instance_dep.redis_node.get(key).to_i + 1 > @instance_dep.version
74
- if nodes_with_deps.size != 1
75
- update_dependencies_multi(nodes_with_deps, :skip_master => true)
76
- end
77
-
78
- raise Promiscuous::Error::AlreadyProcessed
79
- end
80
- end
81
-
82
- def nodes_with_deps
83
- @nodes_with_deps ||= dependencies.group_by(&:redis_node).to_a
84
- end
85
-
86
- def dependencies
87
- @dependencies ||= message.dependencies[:write] + message.dependencies[:read]
88
- end
89
-
90
- LOCK_OPTIONS = { :timeout => 1.5.minute, # after 1.5 minute , we give up
91
- :sleep => 0.1, # polling every 100ms.
92
- :expire => 1.minute } # after one minute, we are considered dead
93
-
94
- def with_instance_dependencies
95
- return yield unless message && message.has_dependencies?
96
-
97
- @instance_dep = message.dependencies[:write].first
98
- lock_options = LOCK_OPTIONS.merge(:node => @instance_dep.redis_node)
99
- mutex = Promiscuous::Redis::Mutex.new(@instance_dep.key(:sub).to_s, lock_options)
100
-
101
- unless mutex.lock
102
- raise Promiscuous::Error::LockUnavailable.new(mutex.key)
103
- end
104
-
105
- begin
106
- verify_dependencies
107
- result = yield
108
- update_dependencies
109
- result
110
- ensure
111
- unless mutex.unlock
112
- # TODO Be safe in case we have a duplicate message and lost the lock on it
113
- raise "The subscriber lost the lock during its operation. It means that someone else\n"+
114
- "received a duplicate message, and we got screwed.\n"
115
- end
116
- end
117
- end
118
-
119
- def create
120
- model.__promiscuous_fetch_new(id).tap do |instance|
121
- instance.__promiscuous_update(payload)
122
- instance.save!
123
- end
124
- rescue Exception => e
125
- # TODO Abstract the duplicated index error message
126
- if e.message =~ /E11000 duplicate key error index: .*\.\$_id_ +dup key/
127
- Promiscuous.warn "[receive] ignoring already created record #{message.payload}"
128
- else
129
- raise e
130
- end
131
- end
132
-
133
- def update
134
- model.__promiscuous_fetch_existing(id).tap do |instance|
135
- instance.__promiscuous_update(payload)
136
- instance.save!
137
- end
138
- rescue model.__promiscuous_missing_record_exception
139
- Promiscuous.warn "[receive] upserting #{message.payload}"
140
- create
141
- end
142
-
143
- def destroy
144
- model.__promiscuous_fetch_existing(id).tap do |instance|
145
- instance.destroy
146
- end
147
- rescue model.__promiscuous_missing_record_exception
148
- Promiscuous.warn "[receive] ignoring missing record #{message.payload}"
149
- end
150
-
151
- def operation
152
- # We must process messages with versions to stay in sync even if we
153
- # don't have a subscriber.
154
- payload.model.nil? ? :dummy : payload.operation
155
- end
156
-
157
- def commit
158
- with_instance_dependencies do
159
- case operation
160
- when :create then create
161
- when :update then update
162
- when :destroy then destroy
163
- when :dummy then nil
164
- end
165
- end
166
- end
1
+ module Promiscuous::Subscriber::Operation
2
+ extend Promiscuous::Autoload
3
+ autoload :Base, :Regular, :Bootstrap
167
4
  end
@@ -11,8 +11,12 @@ class Promiscuous::Subscriber::Worker::Message
11
11
  @parsed_payload ||= MultiJson.load(payload)
12
12
  end
13
13
 
14
- def endpoint
15
- parsed_payload['__amqp__']
14
+ def context
15
+ parsed_payload['context']
16
+ end
17
+
18
+ def app
19
+ parsed_payload['app']
16
20
  end
17
21
 
18
22
  def timestamp
@@ -20,52 +24,70 @@ class Promiscuous::Subscriber::Worker::Message
20
24
  end
21
25
 
22
26
  def dependencies
23
- return @dependencies if @dependencies
24
- @dependencies = parsed_payload['dependencies'].try(:symbolize_keys) || {}
25
- @dependencies[:read] ||= []
26
- @dependencies[:write] ||= []
27
- @dependencies[:read].map! { |dep| Promiscuous::Dependency.parse(dep) }
28
- @dependencies[:write].map! { |dep| Promiscuous::Dependency.parse(dep) }
29
- @dependencies
27
+ @dependencies ||= begin
28
+ dependencies = parsed_payload['dependencies'] || {}
29
+ deps = dependencies['read'].to_a.map { |dep| Promiscuous::Dependency.parse(dep, :type => :read, :owner => app) } +
30
+ dependencies['write'].to_a.map { |dep| Promiscuous::Dependency.parse(dep, :type => :write, :owner => app) }
31
+
32
+ deps
33
+ end
30
34
  end
31
35
 
32
- def happens_before_dependencies
33
- return @happens_before_dependencies if @happens_before_dependencies
36
+ def write_dependencies
37
+ @write_dependencies ||= dependencies.select(&:write?)
38
+ end
34
39
 
35
- deps = []
36
- deps += dependencies[:read]
37
- deps += dependencies[:write].map { |dep| dep.dup.tap { |d| d.version -= 1 } }
40
+ def read_dependencies
41
+ @read_dependencies ||= dependencies.select(&:read?)
42
+ end
43
+
44
+ def happens_before_dependencies
45
+ @happens_before_dependencies ||= begin
46
+ deps = []
47
+ deps += read_dependencies
48
+ deps += write_dependencies.map { |dep| dep.dup.tap { |d| d.version -= 1 } }
38
49
 
39
- # We return the most difficult condition to satisfy first
40
- @happens_before_dependencies = deps.uniq.reverse
50
+ # We return the most difficult condition to satisfy first
51
+ deps.uniq.reverse
52
+ end
41
53
  end
42
54
 
43
55
  def has_dependencies?
44
- return false if Promiscuous::Config.bareback
45
- dependencies[:read].present? || dependencies[:write].present?
56
+ return false if Promiscuous::Config.no_deps
57
+ dependencies.present?
46
58
  end
47
59
 
48
60
  def to_s
49
- "#{endpoint} -> #{happens_before_dependencies.join(', ')}"
61
+ "#{app}/#{context} -> #{happens_before_dependencies.join(', ')}"
50
62
  end
51
63
 
52
64
  def ack
53
65
  time = Time.now
54
- @metadata.ack
55
- @root_worker.stats.notify_processed_message(self, time)
66
+ Promiscuous.debug "[receive] #{payload}"
67
+ @metadata.try(:ack)
68
+ @root_worker.stats.notify_processed_message(self, time) if @root_worker
56
69
  rescue Exception => e
57
70
  # We don't care if we fail, the message will be redelivered at some point
58
71
  Promiscuous.warn "[receive] Some exception happened, but it's okay: #{e}\n#{e.backtrace.join("\n")}"
59
- Promiscuous::Config.error_notifier.try(:call, e)
72
+ Promiscuous::Config.error_notifier.call(e)
73
+ end
74
+
75
+ def postpone
76
+ # Only used during bootstrapping
77
+ @metadata.postpone
78
+ rescue Exception => e
79
+ # We don't care if we fail
80
+ Promiscuous.warn "[receive] (postpone) Some exception happened, but it's okay: #{e}\n#{e.backtrace.join("\n")}"
81
+ Promiscuous::Config.error_notifier.call(e)
60
82
  end
61
83
 
62
84
  def unit_of_work(type, &block)
63
85
  # type is used by the new relic agent, by monkey patching.
64
86
  # middleware?
65
87
  if defined?(Mongoid)
66
- Mongoid.unit_of_work { yield }
88
+ Mongoid.unit_of_work { yield_and_catch_already_processsed(&block) }
67
89
  else
68
- yield
90
+ yield_and_catch_already_processsed(&block)
69
91
  end
70
92
  ensure
71
93
  if defined?(ActiveRecord)
@@ -73,20 +95,24 @@ class Promiscuous::Subscriber::Worker::Message
73
95
  end
74
96
  end
75
97
 
76
- def process
77
- Promiscuous.debug "[receive] #{payload}"
78
- unit_of_work(endpoint) do
79
- payload = Promiscuous::Subscriber::Payload.new(parsed_payload, self)
80
- Promiscuous::Subscriber::Operation.new(payload).commit
81
- end
82
- ack
98
+ def yield_and_catch_already_processsed
99
+ Promiscuous.context { yield }
83
100
  rescue Promiscuous::Error::AlreadyProcessed => orig_e
84
101
  e = Promiscuous::Error::Subscriber.new(orig_e, :payload => payload)
85
- Promiscuous.debug "[receive] #{e}"
86
- ack
102
+ Promiscuous.debug "[receive] #{payload} #{e}\n#{e.backtrace.join("\n")}"
103
+ end
104
+
105
+ def process
106
+ unit_of_work(context) do
107
+ if Promiscuous::Config.bootstrap
108
+ Promiscuous::Subscriber::MessageProcessor::Bootstrap.process(self)
109
+ else
110
+ Promiscuous::Subscriber::MessageProcessor::Regular.process(self)
111
+ end
112
+ end
87
113
  rescue Exception => orig_e
88
114
  e = Promiscuous::Error::Subscriber.new(orig_e, :payload => payload)
89
- Promiscuous.warn "[receive] #{e} #{e.backtrace.join("\n")}"
90
- Promiscuous::Config.error_notifier.try(:call, e)
115
+ Promiscuous.warn "[receive] #{payload} #{e}\n#{e.backtrace.join("\n")}"
116
+ Promiscuous::Config.error_notifier.call(e)
91
117
  end
92
118
  end