promiscuous 0.53.1 → 0.90.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (106) hide show
  1. data/lib/promiscuous.rb +25 -28
  2. data/lib/promiscuous/amqp.rb +27 -8
  3. data/lib/promiscuous/amqp/bunny.rb +131 -16
  4. data/lib/promiscuous/amqp/fake.rb +52 -0
  5. data/lib/promiscuous/amqp/hot_bunnies.rb +56 -0
  6. data/lib/promiscuous/amqp/null.rb +6 -6
  7. data/lib/promiscuous/cli.rb +108 -24
  8. data/lib/promiscuous/config.rb +73 -12
  9. data/lib/promiscuous/convenience.rb +18 -0
  10. data/lib/promiscuous/dependency.rb +59 -0
  11. data/lib/promiscuous/dsl.rb +36 -0
  12. data/lib/promiscuous/error.rb +3 -1
  13. data/lib/promiscuous/error/already_processed.rb +5 -0
  14. data/lib/promiscuous/error/base.rb +1 -0
  15. data/lib/promiscuous/error/connection.rb +7 -5
  16. data/lib/promiscuous/error/dependency.rb +111 -0
  17. data/lib/promiscuous/error/lock_unavailable.rb +12 -0
  18. data/lib/promiscuous/error/lost_lock.rb +12 -0
  19. data/lib/promiscuous/error/missing_context.rb +29 -0
  20. data/lib/promiscuous/error/publisher.rb +5 -15
  21. data/lib/promiscuous/error/recovery.rb +7 -0
  22. data/lib/promiscuous/error/subscriber.rb +2 -4
  23. data/lib/promiscuous/key.rb +36 -0
  24. data/lib/promiscuous/loader.rb +12 -16
  25. data/lib/promiscuous/middleware.rb +112 -0
  26. data/lib/promiscuous/publisher.rb +7 -4
  27. data/lib/promiscuous/publisher/context.rb +92 -0
  28. data/lib/promiscuous/publisher/mock_generator.rb +72 -0
  29. data/lib/promiscuous/publisher/model.rb +3 -86
  30. data/lib/promiscuous/publisher/model/active_record.rb +8 -15
  31. data/lib/promiscuous/publisher/model/base.rb +136 -0
  32. data/lib/promiscuous/publisher/model/ephemeral.rb +69 -0
  33. data/lib/promiscuous/publisher/model/mock.rb +61 -0
  34. data/lib/promiscuous/publisher/model/mongoid.rb +57 -100
  35. data/lib/promiscuous/{common/lint.rb → publisher/operation.rb} +1 -1
  36. data/lib/promiscuous/publisher/operation/base.rb +707 -0
  37. data/lib/promiscuous/publisher/operation/mongoid.rb +370 -0
  38. data/lib/promiscuous/publisher/worker.rb +22 -0
  39. data/lib/promiscuous/railtie.rb +21 -3
  40. data/lib/promiscuous/redis.rb +132 -40
  41. data/lib/promiscuous/resque.rb +12 -0
  42. data/lib/promiscuous/sidekiq.rb +15 -0
  43. data/lib/promiscuous/subscriber.rb +9 -20
  44. data/lib/promiscuous/subscriber/model.rb +4 -104
  45. data/lib/promiscuous/subscriber/model/active_record.rb +10 -0
  46. data/lib/promiscuous/subscriber/model/base.rb +96 -0
  47. data/lib/promiscuous/subscriber/model/mongoid.rb +86 -0
  48. data/lib/promiscuous/subscriber/model/observer.rb +37 -0
  49. data/lib/promiscuous/subscriber/operation.rb +167 -0
  50. data/lib/promiscuous/subscriber/payload.rb +34 -0
  51. data/lib/promiscuous/subscriber/worker.rb +22 -18
  52. data/lib/promiscuous/subscriber/worker/message.rb +48 -25
  53. data/lib/promiscuous/subscriber/worker/message_synchronizer.rb +273 -181
  54. data/lib/promiscuous/subscriber/worker/pump.rb +17 -43
  55. data/lib/promiscuous/subscriber/worker/recorder.rb +24 -0
  56. data/lib/promiscuous/subscriber/worker/runner.rb +24 -3
  57. data/lib/promiscuous/subscriber/worker/stats.rb +62 -0
  58. data/lib/promiscuous/timer.rb +38 -0
  59. data/lib/promiscuous/version.rb +1 -1
  60. metadata +98 -143
  61. data/README.md +0 -33
  62. data/lib/promiscuous/amqp/ruby_amqp.rb +0 -140
  63. data/lib/promiscuous/common.rb +0 -4
  64. data/lib/promiscuous/common/class_helpers.rb +0 -12
  65. data/lib/promiscuous/common/lint/base.rb +0 -24
  66. data/lib/promiscuous/common/options.rb +0 -51
  67. data/lib/promiscuous/ephemeral.rb +0 -14
  68. data/lib/promiscuous/error/recover.rb +0 -1
  69. data/lib/promiscuous/observer.rb +0 -5
  70. data/lib/promiscuous/publisher/active_record.rb +0 -7
  71. data/lib/promiscuous/publisher/amqp.rb +0 -18
  72. data/lib/promiscuous/publisher/attributes.rb +0 -32
  73. data/lib/promiscuous/publisher/base.rb +0 -23
  74. data/lib/promiscuous/publisher/class.rb +0 -36
  75. data/lib/promiscuous/publisher/envelope.rb +0 -7
  76. data/lib/promiscuous/publisher/ephemeral.rb +0 -9
  77. data/lib/promiscuous/publisher/lint.rb +0 -35
  78. data/lib/promiscuous/publisher/lint/amqp.rb +0 -14
  79. data/lib/promiscuous/publisher/lint/attributes.rb +0 -12
  80. data/lib/promiscuous/publisher/lint/base.rb +0 -5
  81. data/lib/promiscuous/publisher/lint/class.rb +0 -15
  82. data/lib/promiscuous/publisher/lint/polymorphic.rb +0 -22
  83. data/lib/promiscuous/publisher/mock.rb +0 -79
  84. data/lib/promiscuous/publisher/mongoid.rb +0 -33
  85. data/lib/promiscuous/publisher/mongoid/embedded.rb +0 -27
  86. data/lib/promiscuous/publisher/mongoid/embedded_many.rb +0 -12
  87. data/lib/promiscuous/publisher/polymorphic.rb +0 -8
  88. data/lib/promiscuous/subscriber/active_record.rb +0 -11
  89. data/lib/promiscuous/subscriber/amqp.rb +0 -25
  90. data/lib/promiscuous/subscriber/attributes.rb +0 -35
  91. data/lib/promiscuous/subscriber/base.rb +0 -29
  92. data/lib/promiscuous/subscriber/class.rb +0 -29
  93. data/lib/promiscuous/subscriber/dummy.rb +0 -19
  94. data/lib/promiscuous/subscriber/envelope.rb +0 -18
  95. data/lib/promiscuous/subscriber/lint.rb +0 -30
  96. data/lib/promiscuous/subscriber/lint/amqp.rb +0 -21
  97. data/lib/promiscuous/subscriber/lint/attributes.rb +0 -21
  98. data/lib/promiscuous/subscriber/lint/base.rb +0 -14
  99. data/lib/promiscuous/subscriber/lint/class.rb +0 -13
  100. data/lib/promiscuous/subscriber/lint/polymorphic.rb +0 -39
  101. data/lib/promiscuous/subscriber/mongoid.rb +0 -27
  102. data/lib/promiscuous/subscriber/mongoid/embedded.rb +0 -17
  103. data/lib/promiscuous/subscriber/mongoid/embedded_many.rb +0 -44
  104. data/lib/promiscuous/subscriber/observer.rb +0 -26
  105. data/lib/promiscuous/subscriber/polymorphic.rb +0 -36
  106. data/lib/promiscuous/subscriber/upsert.rb +0 -12
@@ -0,0 +1,34 @@
1
+ class Promiscuous::Subscriber::Payload
2
+ attr_accessor :message, :id, :operation, :attributes, :model
3
+
4
+ def initialize(payload, message=nil)
5
+ self.message = message
6
+
7
+ if payload.is_a?(Hash) && payload['__amqp__']
8
+ self.id = payload['id']
9
+ self.operation = payload['operation'].try(:to_sym)
10
+ self.attributes = payload['payload'] # TODO payload payload... not great.
11
+ self.model = self.class.get_subscribed_model(payload)
12
+ end
13
+ end
14
+
15
+ def self.get_subscribed_model(payload)
16
+ # TODO test the regexp source
17
+ mapping = Promiscuous::Subscriber::Model.mapping
18
+ model = mapping.select { |from| payload['__amqp__'] =~ from }.values.first
19
+ model = get_subscribed_subclass(model, payload) if model
20
+ model
21
+ end
22
+
23
+ def self.get_subscribed_subclass(root_model, payload)
24
+ # TODO remove 'type' (backward compatibility)
25
+ received_ancestors = [payload['ancestors'] || payload['type']].flatten.compact
26
+ # TODO test the ancestor chain
27
+ subscriber_subclasses = [root_model] + root_model.descendants
28
+ received_ancestors.each do |ancestor|
29
+ model = subscriber_subclasses.select { |klass| klass.subscribe_as == ancestor }.first
30
+ return model if model
31
+ end
32
+ root_model
33
+ end
34
+ end
@@ -1,23 +1,27 @@
1
- require 'celluloid'
2
- require 'celluloid/io'
3
-
4
- class Promiscuous::Subscriber::Worker < Celluloid::SupervisionGroup
1
+ class Promiscuous::Subscriber::Worker
5
2
  extend Promiscuous::Autoload
6
- autoload :Message, :Pump, :MessageSynchronizer, :Runner
3
+ autoload :Message, :Pump, :MessageSynchronizer, :Runner, :Stats, :Recorder
7
4
 
8
- pool Runner, :as => :runners, :size => 10
9
- supervise MessageSynchronizer, :as => :message_synchronizer
10
- supervise Pump, :as => :pump
5
+ attr_accessor :message_synchronizer, :pump, :runner, :stats
11
6
 
12
- def finalize
13
- # The order matters as actors depend on each other.
14
- # This is fixed in the new celluloid, but the gem is not published yet.
15
- [:pump, :message_synchronizer, :runners].each do |actor_name|
16
- Celluloid::Actor[actor_name].terminate
17
- end
7
+ def initialize
8
+ @message_synchronizer = MessageSynchronizer.new(self)
9
+ @pump = Pump.new(self)
10
+ @runner = Runner.new(self)
11
+ @stats = Stats.new
12
+ end
13
+
14
+ def start
15
+ @message_synchronizer.connect
16
+ @pump.connect
17
+ @runner.start
18
+ @stats.connect
18
19
  end
19
- end
20
20
 
21
- # Find a better place to put this
22
- Celluloid.exception_handler { |e| Promiscuous::Config.error_notifier.try(:call, e) }
23
- Celluloid.logger = Promiscuous::Config.logger
21
+ def stop
22
+ @stats.disconnect
23
+ @runner.stop
24
+ @pump.disconnect
25
+ @message_synchronizer.disconnect
26
+ end
27
+ end
@@ -1,41 +1,62 @@
1
1
  class Promiscuous::Subscriber::Worker::Message
2
- attr_accessor :metadata, :payload, :parsed_payload
2
+ attr_accessor :payload, :parsed_payload
3
3
 
4
- def initialize(metadata, payload)
5
- self.metadata = metadata
4
+ def initialize(payload, options={})
6
5
  self.payload = payload
6
+ @metadata = options[:metadata]
7
+ @root_worker = options[:root_worker]
7
8
  end
8
9
 
9
10
  def parsed_payload
10
- @parsed_payload ||= JSON.parse(payload)
11
+ @parsed_payload ||= MultiJson.load(payload)
11
12
  end
12
13
 
13
- def queue_name
14
+ def endpoint
14
15
  parsed_payload['__amqp__']
15
16
  end
16
17
 
17
- def version
18
- return nil unless parsed_payload['version'].is_a? Hash # TODO remove once migrated
19
- @version ||= parsed_payload['version'].try(:symbolize_keys)
18
+ def timestamp
19
+ parsed_payload['timestamp'].to_i
20
20
  end
21
21
 
22
- def global_version
23
- version.try(:[], :global)
22
+ def dependencies
23
+ return @dependencies if @dependencies
24
+ @dependencies = parsed_payload['dependencies'].try(:symbolize_keys) || {}
25
+ @dependencies[:read] ||= []
26
+ @dependencies[:write] ||= []
27
+ @dependencies[:read].map! { |dep| Promiscuous::Dependency.parse(dep) }
28
+ @dependencies[:write].map! { |dep| Promiscuous::Dependency.parse(dep) }
29
+ @dependencies
30
+ end
31
+
32
+ def happens_before_dependencies
33
+ return @happens_before_dependencies if @happens_before_dependencies
34
+
35
+ deps = []
36
+ deps += dependencies[:read]
37
+ deps += dependencies[:write].map { |dep| dep.dup.tap { |d| d.version -= 1 } }
38
+
39
+ # We return the most difficult condition to satisfy first
40
+ @happens_before_dependencies = deps.uniq.reverse
24
41
  end
25
42
 
26
43
  def has_dependencies?
27
44
  return false if Promiscuous::Config.bareback
28
- !!global_version
45
+ dependencies[:read].present? || dependencies[:write].present?
46
+ end
47
+
48
+ def to_s
49
+ "#{endpoint} -> #{happens_before_dependencies.join(', ')}"
29
50
  end
30
51
 
31
52
  def ack
32
- EM.next_tick do
33
- begin
34
- metadata.ack if metadata.channel.open?
35
- rescue
36
- # We don't care if we fail, the message will be redelivered at some point
37
- end
38
- end
53
+ time = Time.now
54
+ @metadata.ack
55
+ @root_worker.stats.notify_processed_message(self, time)
56
+ rescue Exception => e
57
+ # We don't care if we fail, the message will be redelivered at some point
58
+ Promiscuous.warn "[receive] Some exception happened, but it's okay: #{e}\n#{e.backtrace.join("\n")}"
59
+ Promiscuous::Config.error_notifier.try(:call, e)
39
60
  end
40
61
 
41
62
  def unit_of_work(type, &block)
@@ -53,16 +74,18 @@ class Promiscuous::Subscriber::Worker::Message
53
74
  end
54
75
 
55
76
  def process
56
- #return if worker.stopped?
57
-
58
77
  Promiscuous.debug "[receive] #{payload}"
59
- unit_of_work(queue_name) do
60
- Promiscuous::Subscriber.process(parsed_payload, :message => self)
78
+ unit_of_work(endpoint) do
79
+ payload = Promiscuous::Subscriber::Payload.new(parsed_payload, self)
80
+ Promiscuous::Subscriber::Operation.new(payload).commit
61
81
  end
62
-
63
82
  ack
64
- rescue Exception => e
65
- e = Promiscuous::Error::Subscriber.new(e, :payload => payload)
83
+ rescue Promiscuous::Error::AlreadyProcessed => orig_e
84
+ e = Promiscuous::Error::Subscriber.new(orig_e, :payload => payload)
85
+ Promiscuous.debug "[receive] #{e}"
86
+ ack
87
+ rescue Exception => orig_e
88
+ e = Promiscuous::Error::Subscriber.new(orig_e, :payload => payload)
66
89
  Promiscuous.warn "[receive] #{e} #{e.backtrace.join("\n")}"
67
90
  Promiscuous::Config.error_notifier.try(:call, e)
68
91
  end
@@ -2,262 +2,354 @@ module ::Containers; end
2
2
  require 'containers/priority_queue'
3
3
 
4
4
  class Promiscuous::Subscriber::Worker::MessageSynchronizer
5
- include Celluloid::IO
5
+ RECONNECT_INTERVAL = 2
6
+ CLEANUP_INTERVAL = 100 # messages
7
+ QUEUE_MAX_AGE = 100 # messages
6
8
 
7
- attr_accessor :redis
9
+ attr_accessor :redis, :node_synchronizers, :num_processed_messages, :num_queued_messages
8
10
 
9
- def initialize
10
- connect
11
- async.main_loop
11
+ def initialize(root)
12
+ @root = root
13
+ @node_synchronizers = {}
14
+ @lock = Mutex.new
15
+ @reconnect_timer = Promiscuous::Timer.new
12
16
  end
13
17
 
14
- def stop
15
- terminate
18
+ def connected?
19
+ !!@redis
16
20
  end
17
21
 
18
- def finalize
19
- disconnect
22
+ def connect
23
+ @lock.synchronize do
24
+ return unless !connected?
25
+
26
+ @num_processed_messages = 0
27
+ @num_queued_messages = 0
28
+ redis = Promiscuous::Redis.new_blocking_connection
29
+ redis.nodes.each { |node| @node_synchronizers[node] = NodeSynchronizer.new(self, node) }
30
+ @redis = redis
31
+ end
32
+ @root.pump.recover
20
33
  end
21
34
 
22
- def connect
23
- @queued_messages = 0
24
- @subscriptions = {}
25
- self.redis = Promiscuous::Redis.new_celluloid_connection
35
+ def disconnect
36
+ @lock.synchronize do
37
+ return unless connected?
38
+
39
+ @redis, redis = nil, @redis
40
+ @node_synchronizers.values.each { |node_synchronizer| node_synchronizer.stop_main_loop }
41
+ @node_synchronizers.clear
42
+ redis.quit
43
+ end
44
+ rescue Exception
26
45
  end
27
46
 
28
- def connected?
29
- !!self.redis
47
+ def reconnect
48
+ self.disconnect
49
+ self.connect
50
+ @reconnect_timer.reset
51
+ Promiscuous.warn "[redis] Reconnected"
30
52
  end
31
53
 
32
54
  def rescue_connection
33
- disconnect
34
55
  e = Promiscuous::Redis.lost_connection_exception
35
56
 
36
57
  Promiscuous.warn "[redis] #{e}. Reconnecting..."
37
58
  Promiscuous::Config.error_notifier.try(:call, e)
38
59
 
39
60
  # TODO stop the pump to unack all messages
40
- reconnect_later
41
- end
42
-
43
- def disconnect
44
- self.redis.client.connection.disconnect if connected?
45
- rescue
46
- ensure
47
- self.redis = nil
48
- end
49
-
50
- def reconnect
51
- @reconnect_timer.try(:reset)
52
- @reconnect_timer = nil
53
-
54
- unless connected?
55
- self.connect
56
- main_loop!
57
-
58
- Promiscuous.warn "[redis] Reconnected"
59
- Celluloid::Actor[:pump].recover
60
- end
61
- rescue
62
- reconnect_later
63
- end
64
-
65
- def reconnect_later
66
- @reconnect_timer ||= after(2.seconds) { reconnect }
67
- end
68
-
69
- def main_loop
70
- redis_client = self.redis.client
71
- loop do
72
- reply = redis_client.read
73
- raise reply if reply.is_a?(Redis::CommandError)
74
- type, subscription, arg = reply
75
-
76
- case type
77
- when 'subscribe'
78
- find_subscription(subscription).finalize_subscription
79
- when 'unsubscribe'
80
- when 'message'
81
- find_subscription(subscription).signal_version(arg)
82
- end
83
- end
84
- rescue EOFError
85
- # Unwanted disconnection
86
- rescue_connection
87
- rescue IOError => e
88
- unless redis_client == self.redis.client
89
- # We were told to disconnect
90
- else
91
- raise e
92
- end
93
- rescue Celluloid::Task::TerminatedError
94
- rescue Exception => e
95
- Promiscuous.warn "[redis] #{e} #{e.backtrace.join("\n")}"
96
-
97
- #Promiscuous::Worker.stop TODO
98
- Promiscuous::Config.error_notifier.try(:call, e)
61
+ @reconnect_timer.run_every(RECONNECT_INTERVAL) { reconnect }
99
62
  end
100
63
 
101
64
  # process_when_ready() is called by the AMQP pump. This is what happens:
102
65
  # 1. First, we subscribe to redis and wait for the confirmation.
103
66
  # 2. Then we check if the version in redis is old enough to process the message.
104
67
  # If not we bail out and rely on the subscription to kick the processing.
105
- # Because we subscribed in advanced, we will not miss the notification, but
106
- # extra care needs to be taken to avoid processing the message twice (see
107
- # perform()).
68
+ # Because we subscribed in advanced, we will not miss the notification.
108
69
  def process_when_ready(msg)
109
- # Dropped messages will be redelivered as we reconnect
110
- # when calling worker.pump.start
70
+ # Dropped messages will be redelivered as we (re)connect
111
71
  return unless self.redis
112
72
 
113
- bump_message_counter!
114
-
115
- return process_message!(msg) unless msg.has_dependencies?
73
+ @lock.synchronize do
74
+ @num_queued_messages += 1
75
+ end
116
76
 
117
- # The message synchronizer only takes care of happens before (>=) dependencies.
118
- # The message will handle the skip logic in case of duplicates.
119
- on_version Promiscuous::Redis.sub_key('global'), msg.global_version do
77
+ if msg.has_dependencies?
78
+ process_message_proc = proc { process_message!(msg) }
79
+ msg.happens_before_dependencies.reduce(process_message_proc) do |chain, dep|
80
+ get_redis = dep.redis_node
81
+ subscriber_redis = dep.redis_node(@redis)
82
+
83
+ key = dep.key(:sub).join('rw').to_s
84
+ version = dep.version
85
+ node_synchronizer = @node_synchronizers[subscriber_redis]
86
+ proc { node_synchronizer.on_version(subscriber_redis, get_redis, key, version, msg) { chain.call } }
87
+ end.call
88
+ else
120
89
  process_message!(msg)
121
90
  end
122
91
  end
123
92
 
124
- def bump_message_counter!
125
- @queued_messages += 1
126
- maybe_recover
93
+ def process_message!(msg)
94
+ @root.runner.messages_to_process << msg
95
+
96
+ cleanup = false
97
+ @lock.synchronize do
98
+ @num_queued_messages -= 1
99
+ @num_processed_messages += 1
100
+ cleanup = @num_processed_messages % CLEANUP_INTERVAL == 0
101
+ end
102
+ @node_synchronizers.values.each(&:cleanup_if_old) if @node_synchronizers && cleanup
127
103
  end
128
104
 
129
105
  def maybe_recover
130
- return unless Promiscuous::Config.recovery
131
-
132
- if @queued_messages == Promiscuous::Config.prefetch
106
+ if @num_queued_messages == Promiscuous::Config.prefetch
133
107
  # We've reached the amount of messages the amqp queue is willing to give us.
134
- # We also know that we are not processing messages (@queued_messages is
135
- # decremented before we send the message to the runners).
108
+ # We also know that we are not processing messages (@num_queued_messages is
109
+ # decremented before we send the message to the runners), and we are called
110
+ # after adding a pending callback.
136
111
  recover
137
112
  end
138
113
  end
139
114
 
140
115
  def recover
141
- global_key = Promiscuous::Redis.sub_key('global')
142
- current_version = Promiscuous::Redis.get(global_key).to_i
143
-
144
- version_to_allow_progress = get_subscription(global_key).callbacks.next.version - 1
145
- num_messages_to_skip = version_to_allow_progress - current_version
146
-
147
- if num_messages_to_skip > 0
148
- recovery_msg = "Recovering. Moving current version from #{current_version} " +
149
- "to #{version_to_allow_progress}. " +
150
- "Skipping #{num_messages_to_skip} messages..."
151
- else
152
- recovery_msg = "Not recovering. current version is #{current_version}, " +
153
- "while we just need #{version_to_allow_progress}. " +
154
- "Offset is #{num_messages_to_skip} message."
155
- end
156
-
157
- e = Promiscuous::Error::Recover.new(recovery_msg)
158
- if current_version > 0
159
- Promiscuous.error "[receive] #{e}"
160
- Promiscuous::Config.error_notifier.try(:call, e)
161
- else
162
- Promiscuous.info "[receive] #{e}"
163
- # Initial sync, nothing to worry about
164
- end
165
-
166
- if num_messages_to_skip > 0
167
- Promiscuous::Redis.set(global_key, version_to_allow_progress)
168
- Promiscuous::Redis.publish(global_key, version_to_allow_progress)
169
- end
116
+ # XXX This recovery mechanism only works with one worker.
117
+ # We are taking the earliest message to unblock, but in reality we should
118
+ # do the DAG of the happens before dependencies, take root nodes
119
+ # of the disconnected graphs, and sort by timestamps if needed.
120
+ msg = blocked_messages.first
121
+
122
+ versions_to_skip = msg.happens_before_dependencies.map do |dep|
123
+ key = dep.key(:sub).join('rw').to_s
124
+ to_skip = dep.version - dep.redis_node.get(key).to_i
125
+ [dep, key, to_skip] if to_skip > 0
126
+ end.compact
127
+
128
+ return not_recovering if versions_to_skip.blank?
129
+
130
+ recovery_msg = "Skipping "
131
+ recovery_msg += versions_to_skip.map do |dep, key, to_skip|
132
+ dep.redis_node.set(key, dep.version)
133
+ dep.redis_node.publish(key, dep.version)
134
+
135
+ # Note: the skipped message would have a write dependency with dep.to_s
136
+ "#{to_skip} message(s) on #{dep}"
137
+ end.join(", ")
138
+
139
+ e = Promiscuous::Error::Recovery.new(recovery_msg)
140
+ Promiscuous.error "[synchronization recovery] #{e}"
141
+ # TODO Don't report when doing the initial sync
142
+ Promiscuous::Config.error_notifier.try(:call, e)
170
143
  end
171
144
 
172
- def process_message!(msg)
173
- @queued_messages -= 1
174
- Celluloid::Actor[:runners].async.process(msg)
145
+ def blocked_messages
146
+ @node_synchronizers.values
147
+ .map { |node_synchronizer| node_synchronizer.blocked_messages }
148
+ .flatten
149
+ .uniq
150
+ .sort_by { |msg| msg.timestamp }
175
151
  end
176
152
 
177
- def on_version(key, version, &callback)
178
- return unless @subscriptions
179
- sub = get_subscription(key).subscribe
180
- sub.add_callback(Subscription::Callback.new(version, callback))
181
- sub.signal_version(Promiscuous::Redis.get(key))
153
+ def not_recovering
154
+ Promiscuous.warn "[synchronization recovery] Nothing to recover from"
182
155
  end
183
156
 
184
- def find_subscription(key)
185
- raise "Fatal error (redis sub)" unless @subscriptions[key]
186
- @subscriptions[key]
187
- end
157
+ class NodeSynchronizer
158
+ attr_accessor :node, :subscriptions, :root_synchronizer
188
159
 
189
- def get_subscription(key)
190
- @subscriptions[key] ||= Subscription.new(self, key)
191
- end
160
+ def initialize(root_synchronizer, node)
161
+ @root_synchronizer = root_synchronizer
162
+ @node = node
163
+ @subscriptions = {}
164
+ @subscriptions_lock = Mutex.new
165
+ @thread = Thread.new { main_loop }
166
+ end
192
167
 
193
- class Subscription
194
- attr_accessor :parent, :key, :callbacks
168
+ def main_loop
169
+ redis_client = @node.client
195
170
 
196
- def initialize(parent, key)
197
- self.parent = parent
198
- self.key = key
171
+ loop do
172
+ reply = redis_client.read
173
+ raise reply if reply.is_a?(Redis::CommandError)
174
+ type, subscription, arg = reply
175
+
176
+ case type
177
+ when 'subscribe'
178
+ notify_subscription(subscription)
179
+ when 'unsubscribe'
180
+ when 'message'
181
+ notify_key_change(subscription, arg)
182
+ end
183
+ end
184
+ rescue EOFError, Errno::ECONNRESET
185
+ # Unwanted disconnection
186
+ @root_synchronizer.rescue_connection unless @stop
187
+ rescue IOError => e
188
+ raise e unless @stop
189
+ rescue Exception => e
190
+ Promiscuous.warn "[redis] #{e.class} #{e.message}"
191
+ Promiscuous.warn "[redis] #{e} #{e.backtrace.join("\n")}"
199
192
 
200
- @subscription_requested = false
201
- @subscribed_to_redis = false
202
- # We use a priority queue that returns the smallest value first
203
- @callbacks = Containers::PriorityQueue.new { |x, y| x < y }
193
+ Promiscuous::Config.error_notifier.try(:call, e)
204
194
  end
205
195
 
206
- def subscribe
207
- request_subscription
196
+ def stop_main_loop
197
+ @stop = true
198
+ @thread.kill
199
+ end
208
200
 
209
- loop do
210
- break if @subscribed_to_redis
211
- parent.wait :subscription
201
+ def on_version(subscriber_redis, get_redis, key, version, message, &callback)
202
+ # subscriber_redis and get_redis are different connections to the
203
+ # same node.
204
+ if version == 0
205
+ callback.call
206
+ else
207
+ sub = get_subscription(subscriber_redis, get_redis, key)
208
+ sub.subscribe
209
+ sub.add_callback(Subscription::Callback.new(version, callback, message))
212
210
  end
213
- self
214
211
  end
215
212
 
216
- def request_subscription
217
- return if @subscription_requested
218
- parent.redis.client.process([[:subscribe, key]])
219
- @subscription_requested = true
213
+ def blocked_messages
214
+ @subscriptions_lock.synchronize do
215
+ @subscriptions.values
216
+ .map(&:callbacks)
217
+ .map(&:next)
218
+ .compact
219
+ .map(&:message)
220
+ end
220
221
  end
221
222
 
222
- def finalize_subscription
223
- @subscribed_to_redis = true
224
- parent.signal :subscription
223
+ def notify_subscription(key)
224
+ find_subscription(key).try(:finalize_subscription)
225
225
  end
226
226
 
227
- def destroy
228
- # TODO parent.redis_client_call(:unsubscribe, key)
227
+ def notify_key_change(key, version)
228
+ find_subscription(key).try(:signal_version, version)
229
229
  end
230
230
 
231
- def signal_version(current_version)
232
- current_version = current_version.to_i
233
- loop do
234
- next_cb = @callbacks.next
235
- return unless next_cb && next_cb.can_perform?(current_version)
231
+ def remove_subscription(key)
232
+ @subscriptions_lock.synchronize do
233
+ @subscriptions.delete(key)
234
+ end
235
+ end
236
236
 
237
- @callbacks.pop
238
- next_cb.perform
237
+ def find_subscription(key)
238
+ @subscriptions_lock.synchronize do
239
+ @subscriptions[key]
239
240
  end
240
241
  end
241
242
 
242
- def add_callback(callback)
243
- callback.subscription = self
244
- @callbacks.push(callback, callback.version)
243
+ def get_subscription(subscriber_redis, get_redis, key)
244
+ @subscriptions_lock.synchronize do
245
+ @subscriptions[key] ||= Subscription.new(self, subscriber_redis, get_redis, key)
246
+ end
245
247
  end
246
248
 
247
- class Callback
248
- attr_accessor :subscription, :version, :callback, :token
249
+ def cleanup_if_old
250
+ @subscriptions_lock.synchronize do
251
+ @subscriptions.values.each(&:cleanup_if_old)
252
+ end
253
+ end
249
254
 
250
- def initialize(version, callback)
251
- self.version = version
252
- self.callback = callback
255
+ class Subscription
256
+ attr_accessor :node_synchronizer, :subscriber_redis, :get_redis, :key, :callbacks, :last_version
257
+
258
+ def initialize(node_synchronizer, subscriber_redis, get_redis, key)
259
+ self.node_synchronizer = node_synchronizer
260
+ self.subscriber_redis = subscriber_redis
261
+ self.get_redis = get_redis
262
+ self.key = key
263
+
264
+ @subscription_requested = false
265
+ # We use a priority queue that returns the smallest value first
266
+ @callbacks = Containers::PriorityQueue.new { |x, y| x < y }
267
+ @last_version = 0
268
+ @lock = Mutex.new
269
+
270
+ refresh_activity
253
271
  end
254
272
 
255
- def can_perform?(current_version)
256
- current_version + 1 >= self.version
273
+ def total_num_processed_messages
274
+ node_synchronizer.root_synchronizer.num_processed_messages
257
275
  end
258
276
 
259
- def perform
260
- callback.call
277
+ def refresh_activity
278
+ @last_activity_at = total_num_processed_messages
279
+ end
280
+
281
+ def is_old?
282
+ delta = total_num_processed_messages - @last_activity_at
283
+ @callbacks.empty? && delta >= QUEUE_MAX_AGE
284
+ end
285
+
286
+ def cleanup_if_old
287
+ if is_old?
288
+ subscriber_redis.client.process([[:unsubscribe, key]])
289
+ node_synchronizer.subscriptions.delete(key) # lock is already held
290
+ end
291
+ end
292
+
293
+ def subscribe
294
+ @lock.synchronize do
295
+ return if @subscription_requested
296
+ @subscription_requested = true
297
+ end
298
+
299
+ subscriber_redis.client.process([[:subscribe, key]])
300
+ end
301
+
302
+ def finalize_subscription
303
+ signal_version(get_redis.get(key))
304
+ end
305
+
306
+ def signal_version(current_version)
307
+ current_version = current_version.to_i
308
+ @lock.synchronize do
309
+ return if current_version < @last_version
310
+ @last_version = current_version
311
+ end
312
+
313
+ loop do
314
+ next_cb = nil
315
+ @lock.synchronize do
316
+ next_cb = @callbacks.next
317
+ return unless next_cb && next_cb.can_perform?(@last_version)
318
+ @callbacks.pop
319
+ end
320
+ next_cb.perform
321
+ end
322
+ end
323
+
324
+ def add_callback(callback)
325
+ refresh_activity
326
+
327
+ can_perform_immediately = false
328
+ @lock.synchronize do
329
+ if callback.can_perform?(@last_version)
330
+ can_perform_immediately = true
331
+ else
332
+ @callbacks.push(callback, callback.version)
333
+ end
334
+ end
335
+
336
+ if can_perform_immediately
337
+ callback.perform
338
+ else
339
+ node_synchronizer.root_synchronizer.maybe_recover if Promiscuous::Config.recovery
340
+ end
341
+ end
342
+
343
+ class Callback < Struct.new(:version, :callback, :message)
344
+ # message is just here for debugging, not used in the happy path
345
+ def can_perform?(current_version)
346
+ # The message synchronizer takes care of happens before dependencies.
347
+ current_version >= self.version
348
+ end
349
+
350
+ def perform
351
+ callback.call
352
+ end
261
353
  end
262
354
  end
263
355
  end