promiscuous 0.31.1 → 0.32.0
Sign up to get free protection for your applications and to get access to all the features.
- data/lib/promiscuous.rb +9 -0
- data/lib/promiscuous/amqp.rb +14 -1
- data/lib/promiscuous/amqp/ruby_amqp.rb +14 -13
- data/lib/promiscuous/error/connection.rb +13 -4
- data/lib/promiscuous/publisher/model.rb +19 -20
- data/lib/promiscuous/redis.rb +14 -1
- data/lib/promiscuous/subscriber/model.rb +53 -11
- data/lib/promiscuous/subscriber/observer.rb +5 -8
- data/lib/promiscuous/subscriber/worker.rb +4 -3
- data/lib/promiscuous/subscriber/worker/message.rb +20 -12
- data/lib/promiscuous/subscriber/worker/message_synchronizer.rb +82 -19
- data/lib/promiscuous/subscriber/worker/pump.rb +11 -15
- data/lib/promiscuous/version.rb +1 -1
- data/lib/promiscuous/worker.rb +3 -3
- metadata +43 -43
data/lib/promiscuous.rb
CHANGED
@@ -25,5 +25,14 @@ module Promiscuous
|
|
25
25
|
desc.reject! { |klass| klass.name =~ /^Promiscuous::/ }
|
26
26
|
desc.each { |klass| klass.setup_class_binding }
|
27
27
|
end
|
28
|
+
|
29
|
+
def healthy?
|
30
|
+
AMQP.ensure_connected
|
31
|
+
Redis.ensure_connected
|
32
|
+
rescue
|
33
|
+
false
|
34
|
+
else
|
35
|
+
true
|
36
|
+
end
|
28
37
|
end
|
29
38
|
end
|
data/lib/promiscuous/amqp.rb
CHANGED
@@ -11,6 +11,19 @@ module Promiscuous::AMQP
|
|
11
11
|
@backend = "Promiscuous::AMQP::#{value.to_s.camelize.gsub(/amqp/, 'AMQP')}".constantize unless value.nil?
|
12
12
|
end
|
13
13
|
|
14
|
-
|
14
|
+
def lost_connection_exception
|
15
|
+
Promiscuous::Error::Connection.new(:service => :amqp)
|
16
|
+
end
|
17
|
+
|
18
|
+
def ensure_connected
|
19
|
+
raise lost_connection_exception unless connected?
|
20
|
+
end
|
21
|
+
|
22
|
+
def publish(options={})
|
23
|
+
ensure_connected
|
24
|
+
backend.publish(options)
|
25
|
+
end
|
26
|
+
|
27
|
+
delegate :connect, :disconnect, :connected?, :open_queue, :to => :backend
|
15
28
|
end
|
16
29
|
end
|
@@ -24,24 +24,30 @@ module Promiscuous::AMQP::RubyAMQP
|
|
24
24
|
|
25
25
|
connection.on_tcp_connection_loss do |conn|
|
26
26
|
unless conn.reconnecting?
|
27
|
-
|
28
|
-
|
27
|
+
e = Promiscuous::AMQP.lost_connection_exception
|
28
|
+
Promiscuous.warn "[amqp] #{e}. Reconnecting..."
|
29
|
+
Promiscuous::Config.error_notifier.try(:call, e)
|
29
30
|
|
30
|
-
|
31
|
-
|
32
|
-
|
31
|
+
worker = Promiscuous::Worker.workers.first
|
32
|
+
worker.message_synchronizer.disconnect if worker
|
33
|
+
|
34
|
+
conn.periodically_reconnect(2.seconds)
|
33
35
|
end
|
34
36
|
end
|
35
37
|
|
36
38
|
connection.on_recovery do |conn|
|
37
|
-
Promiscuous.warn "[
|
38
|
-
|
39
|
+
Promiscuous.warn "[amqp] Reconnected"
|
40
|
+
|
41
|
+
worker = Promiscuous::Worker.workers.first
|
42
|
+
worker.message_synchronizer.reconnect if worker
|
43
|
+
|
44
|
+
Promiscuous::AMQP::RubyAMQP.channel.recover
|
39
45
|
end
|
40
46
|
|
41
47
|
connection.on_error do |conn, conn_close|
|
42
48
|
# No need to handle CONNECTION_FORCED since on_tcp_connection_loss takes
|
43
49
|
# care of it.
|
44
|
-
Promiscuous.warn "[
|
50
|
+
Promiscuous.warn "[amqp] #{conn_close.reply_text}"
|
45
51
|
end
|
46
52
|
end
|
47
53
|
|
@@ -74,11 +80,6 @@ module Promiscuous::AMQP::RubyAMQP
|
|
74
80
|
|
75
81
|
def self.publish(options={})
|
76
82
|
info_msg = "(#{options[:exchange_name]}) #{options[:key]} -> #{options[:payload]}"
|
77
|
-
|
78
|
-
unless channel.connection.connected?
|
79
|
-
raise Promiscuous::Error::Connection.new(:amqp, 'Not connected')
|
80
|
-
end
|
81
|
-
|
82
83
|
Promiscuous.debug "[publish] #{info_msg}"
|
83
84
|
|
84
85
|
EM.next_tick do
|
@@ -1,8 +1,17 @@
|
|
1
1
|
class Promiscuous::Error::Connection < RuntimeError
|
2
|
-
attr_accessor :
|
2
|
+
attr_accessor :service, :url
|
3
3
|
|
4
|
-
def initialize(
|
5
|
-
|
6
|
-
|
4
|
+
def initialize(options={})
|
5
|
+
super(nil)
|
6
|
+
self.service = options[:service]
|
7
|
+
self.url = Promiscuous::Config.__send__("#{service}_url")
|
8
|
+
end
|
9
|
+
|
10
|
+
def message
|
11
|
+
"Lost connection with #{url}"
|
12
|
+
end
|
13
|
+
|
14
|
+
def to_s
|
15
|
+
message
|
7
16
|
end
|
8
17
|
end
|
@@ -22,20 +22,16 @@ module Promiscuous::Publisher::Model
|
|
22
22
|
end
|
23
23
|
end
|
24
24
|
|
25
|
-
def version
|
26
|
-
{:global => @global_version}
|
27
|
-
end
|
28
|
-
|
29
25
|
def payload
|
30
|
-
|
31
|
-
{:version => version, :operation => :dummy}
|
32
|
-
else
|
33
|
-
super.merge(:id => instance.id, :operation => operation, :version => version)
|
34
|
-
end
|
26
|
+
super.merge(:id => instance.id, :operation => operation, :version => version)
|
35
27
|
end
|
36
28
|
|
37
29
|
def include_attributes?
|
38
|
-
operation
|
30
|
+
!operation.in? [:destroy, :dummy]
|
31
|
+
end
|
32
|
+
|
33
|
+
def instance
|
34
|
+
@new_instance || super
|
39
35
|
end
|
40
36
|
|
41
37
|
def with_lock(&block)
|
@@ -43,28 +39,32 @@ module Promiscuous::Publisher::Model
|
|
43
39
|
return yield if operation == :create
|
44
40
|
|
45
41
|
key = Promiscuous::Redis.pub_key(instance.id)
|
46
|
-
|
42
|
+
# We'll block for 60 seconds before raising an exception
|
43
|
+
::RedisLock.new(Promiscuous::Redis, key).retry(300).every(0.2).lock_for_update(&block)
|
47
44
|
end
|
48
45
|
|
49
|
-
def
|
50
|
-
|
46
|
+
def version
|
47
|
+
{:global => @global_version}
|
48
|
+
end
|
49
|
+
|
50
|
+
def update_dependencies
|
51
|
+
@global_version = Promiscuous::Redis.incr(Promiscuous::Redis.pub_key('global'))
|
51
52
|
end
|
52
53
|
|
53
54
|
def commit
|
54
55
|
ret = nil
|
55
56
|
exception = nil
|
56
57
|
|
57
|
-
|
58
|
-
raise Promiscuous::Error::Connection.new(:amqp, 'Not connected')
|
59
|
-
end
|
58
|
+
Promiscuous::AMQP.ensure_connected
|
60
59
|
|
61
60
|
with_lock do
|
62
|
-
|
61
|
+
update_dependencies
|
63
62
|
begin
|
64
63
|
ret = yield
|
65
64
|
rescue Exception => e
|
66
|
-
#
|
67
|
-
|
65
|
+
# we must publish something so the subscriber can sync
|
66
|
+
# with the updated dependencies
|
67
|
+
options[:operation] = :dummy
|
68
68
|
exception = e
|
69
69
|
end
|
70
70
|
|
@@ -75,7 +75,6 @@ module Promiscuous::Publisher::Model
|
|
75
75
|
end
|
76
76
|
end
|
77
77
|
|
78
|
-
# We always need to publish so that the subscriber can keep up
|
79
78
|
publish
|
80
79
|
|
81
80
|
raise exception if exception
|
data/lib/promiscuous/redis.rb
CHANGED
@@ -10,7 +10,10 @@ module Promiscuous::Redis
|
|
10
10
|
def self.new_connection
|
11
11
|
return Null.new if Promiscuous::Config.backend == :null
|
12
12
|
|
13
|
-
::Redis.new(:url => Promiscuous::Config.redis_url
|
13
|
+
redis = ::Redis.new(:url => Promiscuous::Config.redis_url,
|
14
|
+
:tcp_keepalive => 60)
|
15
|
+
redis.client.connect
|
16
|
+
redis
|
14
17
|
end
|
15
18
|
|
16
19
|
def self.new_celluloid_connection
|
@@ -32,6 +35,16 @@ module Promiscuous::Redis
|
|
32
35
|
end
|
33
36
|
end
|
34
37
|
|
38
|
+
def self.lost_connection_exception
|
39
|
+
Promiscuous::Error::Connection.new(:service => :redis)
|
40
|
+
end
|
41
|
+
|
42
|
+
def self.ensure_connected
|
43
|
+
Promiscuous::Redis.master.ping
|
44
|
+
rescue
|
45
|
+
raise lost_connection_exception
|
46
|
+
end
|
47
|
+
|
35
48
|
def self.method_missing(name, *args, &block)
|
36
49
|
self.master.__send__(name, *args, &block)
|
37
50
|
end
|
@@ -1,3 +1,5 @@
|
|
1
|
+
require 'crowdtap_redis_lock'
|
2
|
+
|
1
3
|
module Promiscuous::Subscriber::Model
|
2
4
|
extend ActiveSupport::Concern
|
3
5
|
include Promiscuous::Subscriber::Envelope
|
@@ -31,28 +33,68 @@ module Promiscuous::Subscriber::Model
|
|
31
33
|
when :create then fetch_new
|
32
34
|
when :update then fetch_existing
|
33
35
|
when :destroy then fetch_existing
|
36
|
+
when :dummy then fetch_new
|
34
37
|
end
|
35
38
|
end
|
36
39
|
|
37
40
|
def process_attributes?
|
38
|
-
operation
|
41
|
+
!operation.in? [:destroy, :dummy]
|
39
42
|
end
|
40
43
|
|
41
|
-
def
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
44
|
+
def message
|
45
|
+
options[:message]
|
46
|
+
end
|
47
|
+
|
48
|
+
def with_lock(&block)
|
49
|
+
return yield if Promiscuous::Config.backend == :null
|
50
|
+
|
51
|
+
key = Promiscuous::Redis.sub_key(instance.id)
|
52
|
+
# We'll block for 60 seconds before raising an exception
|
53
|
+
::RedisLock.new(Promiscuous::Redis, key).retry(300).every(0.2).lock_for_update(&block)
|
54
|
+
end
|
55
|
+
|
56
|
+
def verify_dependencies
|
57
|
+
@global_key = Promiscuous::Redis.sub_key('global')
|
58
|
+
Promiscuous::Redis.get(@global_key).to_i + 1 == message.global_version
|
59
|
+
end
|
60
|
+
|
61
|
+
def update_dependencies
|
62
|
+
Promiscuous::Redis.set(@global_key, message.global_version)
|
63
|
+
@changed_global_key = true
|
64
|
+
end
|
65
|
+
|
66
|
+
def publish_dependencies
|
67
|
+
Promiscuous::Redis.publish(@global_key, message.global_version) if @changed_global_key
|
68
|
+
end
|
69
|
+
|
70
|
+
def with_dependencies
|
71
|
+
return yield unless message && message.has_dependencies?
|
72
|
+
|
73
|
+
with_lock do
|
74
|
+
if verify_dependencies
|
75
|
+
yield
|
76
|
+
update_dependencies
|
77
|
+
else
|
78
|
+
Promiscuous.info "[receive] (skipped, already processed) #{message.payload}"
|
79
|
+
end
|
47
80
|
end
|
81
|
+
|
82
|
+
publish_dependencies
|
48
83
|
end
|
49
84
|
|
50
|
-
def
|
51
|
-
|
85
|
+
def process
|
86
|
+
super
|
87
|
+
commit
|
52
88
|
end
|
53
89
|
|
54
|
-
def
|
55
|
-
|
90
|
+
def commit
|
91
|
+
with_dependencies do
|
92
|
+
case operation
|
93
|
+
when :create then instance.save!
|
94
|
+
when :update then instance.save!
|
95
|
+
when :destroy then instance.destroy
|
96
|
+
end
|
97
|
+
end
|
56
98
|
end
|
57
99
|
|
58
100
|
included do
|
@@ -4,19 +4,16 @@ class Promiscuous::Subscriber::Observer < Promiscuous::Subscriber::Base
|
|
4
4
|
include Promiscuous::Subscriber::Polymorphic
|
5
5
|
include Promiscuous::Subscriber::AMQP
|
6
6
|
include Promiscuous::Subscriber::Envelope
|
7
|
+
include Promiscuous::Subscriber::Model
|
7
8
|
|
8
9
|
def fetch
|
9
10
|
klass.new.tap { |o| o.id = id if o.respond_to?(:id=) }
|
10
11
|
end
|
11
12
|
|
12
|
-
def
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
# XXX destroy callbacks will not set attributes (they are not sent)
|
18
|
-
def process_attributes?
|
19
|
-
operation != :destroy
|
13
|
+
def commit
|
14
|
+
with_dependencies do
|
15
|
+
instance.run_callbacks operation unless operation == :dummy
|
16
|
+
end
|
20
17
|
end
|
21
18
|
|
22
19
|
def self.subscribe(options)
|
@@ -17,13 +17,13 @@ class Promiscuous::Subscriber::Worker
|
|
17
17
|
self.pump = Pump.new(self)
|
18
18
|
end
|
19
19
|
|
20
|
-
def
|
20
|
+
def start
|
21
21
|
return unless self.stopped
|
22
22
|
self.stopped = false
|
23
23
|
self.runners = Runner.pool
|
24
24
|
self.message_synchronizer = MessageSynchronizer.new(self)
|
25
|
-
self.message_synchronizer.
|
26
|
-
self.pump.
|
25
|
+
self.message_synchronizer.start
|
26
|
+
self.pump.start
|
27
27
|
end
|
28
28
|
|
29
29
|
def stop
|
@@ -34,6 +34,7 @@ class Promiscuous::Subscriber::Worker
|
|
34
34
|
self.runners.terminate
|
35
35
|
self.runners = nil
|
36
36
|
self.stopped = true
|
37
|
+
# TODO wait for the runners to finish
|
37
38
|
end
|
38
39
|
|
39
40
|
def unit_of_work(type, &block)
|
@@ -15,29 +15,37 @@ class Promiscuous::Subscriber::Worker::Message
|
|
15
15
|
parsed_payload['__amqp__']
|
16
16
|
end
|
17
17
|
|
18
|
-
def has_version?
|
19
|
-
!!version
|
20
|
-
end
|
21
|
-
|
22
18
|
def version
|
23
19
|
@version ||= parsed_payload['version'].try(:symbolize_keys)
|
24
20
|
end
|
25
21
|
|
26
|
-
def
|
27
|
-
|
28
|
-
|
29
|
-
|
22
|
+
def global_version
|
23
|
+
version.try(:[], :global)
|
24
|
+
end
|
25
|
+
|
26
|
+
def has_dependencies?
|
27
|
+
!!global_version
|
28
|
+
end
|
29
|
+
|
30
|
+
def ack
|
31
|
+
EM.next_tick do
|
32
|
+
begin
|
33
|
+
metadata.ack
|
34
|
+
rescue
|
35
|
+
# We don't care if we fail, the message will be redelivered at some point
|
36
|
+
end
|
37
|
+
end
|
30
38
|
end
|
31
39
|
|
32
40
|
def process
|
33
41
|
return if worker.stopped?
|
34
42
|
|
35
43
|
Promiscuous.debug "[receive] #{payload}"
|
44
|
+
worker.unit_of_work(queue_name) do
|
45
|
+
Promiscuous::Subscriber.process(parsed_payload, :message => self)
|
46
|
+
end
|
36
47
|
|
37
|
-
|
38
|
-
|
39
|
-
update_dependencies
|
40
|
-
metadata.ack
|
48
|
+
ack
|
41
49
|
rescue Exception => e
|
42
50
|
e = Promiscuous::Error::Subscriber.new(e, :payload => payload)
|
43
51
|
Promiscuous.warn "[receive] #{e} #{e.backtrace.join("\n")}"
|
@@ -5,11 +5,10 @@ class Promiscuous::Subscriber::Worker::MessageSynchronizer
|
|
5
5
|
|
6
6
|
def initialize(worker)
|
7
7
|
self.worker = worker
|
8
|
-
@subscriptions = {}
|
9
8
|
end
|
10
9
|
|
11
|
-
def
|
12
|
-
|
10
|
+
def start
|
11
|
+
connect
|
13
12
|
main_loop!
|
14
13
|
end
|
15
14
|
|
@@ -18,13 +17,59 @@ class Promiscuous::Subscriber::Worker::MessageSynchronizer
|
|
18
17
|
end
|
19
18
|
|
20
19
|
def finalize
|
21
|
-
|
20
|
+
disconnect
|
21
|
+
end
|
22
|
+
|
23
|
+
def connect
|
24
|
+
@subscriptions = {}
|
25
|
+
self.redis = Promiscuous::Redis.new_celluloid_connection
|
26
|
+
end
|
27
|
+
|
28
|
+
def connected?
|
29
|
+
!!self.redis
|
30
|
+
end
|
31
|
+
|
32
|
+
def rescue_connection
|
33
|
+
disconnect
|
34
|
+
e = Promiscuous::Redis.lost_connection_exception
|
35
|
+
|
36
|
+
Promiscuous.warn "[redis] #{e}. Reconnecting..."
|
37
|
+
Promiscuous::Config.error_notifier.try(:call, e)
|
38
|
+
|
39
|
+
EM.next_tick { worker.pump.stop }
|
40
|
+
reconnect_later
|
41
|
+
end
|
42
|
+
|
43
|
+
def disconnect
|
44
|
+
self.redis.client.connection.disconnect if connected?
|
45
|
+
rescue
|
46
|
+
ensure
|
47
|
+
self.redis = nil
|
48
|
+
end
|
49
|
+
|
50
|
+
def reconnect
|
51
|
+
@reconnect_timer.try(:reset)
|
52
|
+
@reconnect_timer = nil
|
53
|
+
|
54
|
+
unless connected?
|
55
|
+
self.connect
|
56
|
+
main_loop!
|
57
|
+
|
58
|
+
Promiscuous.warn "[redis] Reconnected"
|
59
|
+
EM.next_tick { worker.pump.start }
|
60
|
+
end
|
22
61
|
rescue
|
62
|
+
reconnect_later
|
63
|
+
end
|
64
|
+
|
65
|
+
def reconnect_later
|
66
|
+
@reconnect_timer ||= after(2.seconds) { reconnect }
|
23
67
|
end
|
24
68
|
|
25
69
|
def main_loop
|
70
|
+
redis_client = self.redis.client
|
26
71
|
loop do
|
27
|
-
reply =
|
72
|
+
reply = redis_client.read
|
28
73
|
raise reply if reply.is_a?(Redis::CommandError)
|
29
74
|
type, subscription, arg = reply
|
30
75
|
|
@@ -36,11 +81,19 @@ class Promiscuous::Subscriber::Worker::MessageSynchronizer
|
|
36
81
|
find_subscription(subscription).maybe_perform_callbacks(arg)
|
37
82
|
end
|
38
83
|
end
|
84
|
+
rescue EOFError
|
85
|
+
# Unwanted disconnection
|
86
|
+
rescue_connection
|
87
|
+
rescue IOError => e
|
88
|
+
unless redis_client == self.redis.client
|
89
|
+
# We were told to disconnect
|
90
|
+
else
|
91
|
+
raise e
|
92
|
+
end
|
39
93
|
rescue Celluloid::Task::TerminatedError
|
40
94
|
rescue Exception => e
|
41
95
|
Promiscuous.warn "[redis] #{e} #{e.backtrace.join("\n")}"
|
42
96
|
|
43
|
-
e = Promiscuous::Error::Connection.new(:redis, 'Lost connection')
|
44
97
|
Promiscuous::Worker.stop
|
45
98
|
Promiscuous::Config.error_notifier.try(:call, e)
|
46
99
|
end
|
@@ -53,9 +106,15 @@ class Promiscuous::Subscriber::Worker::MessageSynchronizer
|
|
53
106
|
# extra care needs to be taken to avoid processing the message twice (see
|
54
107
|
# perform()).
|
55
108
|
def process_when_ready(msg)
|
56
|
-
|
109
|
+
# Dropped messages will be redelivered as we reconnect
|
110
|
+
# when calling worker.pump.start
|
111
|
+
return unless self.redis
|
112
|
+
|
113
|
+
return worker.runners.process!(msg) unless msg.has_dependencies?
|
57
114
|
|
58
|
-
|
115
|
+
# The message synchronizer only takes care of happens before (>=) dependencies.
|
116
|
+
# The message will handle the skip logic in case of duplicates.
|
117
|
+
on_version Promiscuous::Redis.sub_key('global'), msg.global_version do
|
59
118
|
worker.runners.process!(msg)
|
60
119
|
end
|
61
120
|
end
|
@@ -64,16 +123,14 @@ class Promiscuous::Subscriber::Worker::MessageSynchronizer
|
|
64
123
|
return unless @subscriptions
|
65
124
|
cb = Subscription::Callback.new(version, callback)
|
66
125
|
get_subscription(key).subscribe.add_callback(version, cb)
|
67
|
-
cb.
|
126
|
+
cb.current_version = Promiscuous::Redis.get(key)
|
68
127
|
end
|
69
128
|
|
70
|
-
# state_lock must be taken before calling find_subscription()
|
71
129
|
def find_subscription(key)
|
72
130
|
raise "Fatal error (redis sub)" unless @subscriptions[key]
|
73
131
|
@subscriptions[key]
|
74
132
|
end
|
75
133
|
|
76
|
-
# state_lock must be taken before calling find_subscription()
|
77
134
|
def get_subscription(key)
|
78
135
|
@subscriptions[key] ||= Subscription.new(self, key)
|
79
136
|
end
|
@@ -90,7 +147,6 @@ class Promiscuous::Subscriber::Worker::MessageSynchronizer
|
|
90
147
|
@callbacks = {}
|
91
148
|
end
|
92
149
|
|
93
|
-
# subscribe() is called with the state_lock of the parent held
|
94
150
|
def subscribe
|
95
151
|
request_subscription
|
96
152
|
|
@@ -102,8 +158,6 @@ class Promiscuous::Subscriber::Worker::MessageSynchronizer
|
|
102
158
|
end
|
103
159
|
|
104
160
|
def request_subscription
|
105
|
-
# We will not send two subscription requests, since we are holding
|
106
|
-
# the state_lock of the parent.
|
107
161
|
return if @subscription_requested
|
108
162
|
parent.redis.client.process([[:subscribe, key]])
|
109
163
|
@subscription_requested = true
|
@@ -130,7 +184,7 @@ class Promiscuous::Subscriber::Worker::MessageSynchronizer
|
|
130
184
|
|
131
185
|
def maybe_perform_callbacks(current_version)
|
132
186
|
@callbacks.values.each do |cb|
|
133
|
-
cb.
|
187
|
+
cb.current_version = current_version
|
134
188
|
end
|
135
189
|
end
|
136
190
|
|
@@ -145,13 +199,22 @@ class Promiscuous::Subscriber::Worker::MessageSynchronizer
|
|
145
199
|
attr_accessor :subscription, :version, :callback, :token
|
146
200
|
|
147
201
|
def initialize(version, callback)
|
202
|
+
@current_version = 0
|
148
203
|
self.version = version
|
149
204
|
self.callback = callback
|
150
205
|
@token = self.class.get_next_token
|
151
206
|
end
|
152
207
|
|
153
|
-
def
|
154
|
-
current_version
|
208
|
+
def current_version=(value)
|
209
|
+
@current_version = value.to_i
|
210
|
+
maybe_perform
|
211
|
+
value
|
212
|
+
end
|
213
|
+
|
214
|
+
private
|
215
|
+
|
216
|
+
def can_perform?
|
217
|
+
@current_version + 1 >= self.version
|
155
218
|
end
|
156
219
|
|
157
220
|
def perform
|
@@ -160,8 +223,8 @@ class Promiscuous::Subscriber::Worker::MessageSynchronizer
|
|
160
223
|
callback.call if subscription.remove_callback(@token)
|
161
224
|
end
|
162
225
|
|
163
|
-
def maybe_perform
|
164
|
-
perform if can_perform?
|
226
|
+
def maybe_perform
|
227
|
+
perform if can_perform?
|
165
228
|
end
|
166
229
|
end
|
167
230
|
end
|
@@ -1,33 +1,29 @@
|
|
1
1
|
class Promiscuous::Subscriber::Worker::Pump
|
2
|
+
# TODO Make this celluloid happy
|
2
3
|
attr_accessor :worker
|
3
4
|
|
4
5
|
def initialize(worker)
|
5
6
|
self.worker = worker
|
6
7
|
end
|
7
8
|
|
8
|
-
def
|
9
|
-
if @queue
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
else
|
17
|
-
Promiscuous::AMQP.open_queue(queue_bindings) do |queue|
|
18
|
-
@queue = queue
|
19
|
-
@queue.subscribe({:ack => true}, &method(:process_payload))
|
9
|
+
def start
|
10
|
+
return if @queue
|
11
|
+
Promiscuous::AMQP.open_queue(queue_bindings) do |queue|
|
12
|
+
@queue = queue
|
13
|
+
@queue.subscribe :ack => true do |metadata, payload|
|
14
|
+
# we drop the payload if we switched to another queue,
|
15
|
+
# duplicate messages could hurt us.
|
16
|
+
process_payload(metadata, payload) if queue == @queue
|
20
17
|
end
|
21
18
|
end
|
22
19
|
end
|
23
20
|
|
24
21
|
def stop
|
25
|
-
|
22
|
+
queue, @queue = @queue, nil
|
23
|
+
queue.unsubscribe if queue rescue nil
|
26
24
|
end
|
27
25
|
|
28
26
|
def process_payload(metadata, payload)
|
29
|
-
return if worker.stopped?
|
30
|
-
|
31
27
|
msg = Promiscuous::Subscriber::Worker::Message.new(worker, metadata, payload)
|
32
28
|
worker.message_synchronizer.process_when_ready(msg)
|
33
29
|
end
|
data/lib/promiscuous/version.rb
CHANGED
data/lib/promiscuous/worker.rb
CHANGED
@@ -3,7 +3,7 @@ module Promiscuous::Worker
|
|
3
3
|
self.workers = []
|
4
4
|
|
5
5
|
def self.replicate(options={})
|
6
|
-
self.workers << Promiscuous::Subscriber::Worker.new(options).tap { |w| w.
|
6
|
+
self.workers << Promiscuous::Subscriber::Worker.new(options).tap { |w| w.start }
|
7
7
|
end
|
8
8
|
|
9
9
|
def self.kill
|
@@ -16,7 +16,7 @@ module Promiscuous::Worker
|
|
16
16
|
workers.each(&:stop)
|
17
17
|
end
|
18
18
|
|
19
|
-
def self.
|
20
|
-
workers.each(&:
|
19
|
+
def self.start
|
20
|
+
workers.each(&:start)
|
21
21
|
end
|
22
22
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: promiscuous
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.32.0
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -10,7 +10,7 @@ authors:
|
|
10
10
|
autorequire:
|
11
11
|
bindir: bin
|
12
12
|
cert_chain: []
|
13
|
-
date: 2013-02-
|
13
|
+
date: 2013-02-03 00:00:00.000000000 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: activesupport
|
@@ -19,7 +19,7 @@ dependencies:
|
|
19
19
|
requirements:
|
20
20
|
- - ! '>='
|
21
21
|
- !ruby/object:Gem::Version
|
22
|
-
version:
|
22
|
+
version: 3.0.0
|
23
23
|
type: :runtime
|
24
24
|
prerelease: false
|
25
25
|
version_requirements: !ruby/object:Gem::Requirement
|
@@ -27,7 +27,7 @@ dependencies:
|
|
27
27
|
requirements:
|
28
28
|
- - ! '>='
|
29
29
|
- !ruby/object:Gem::Version
|
30
|
-
version:
|
30
|
+
version: 3.0.0
|
31
31
|
- !ruby/object:Gem::Dependency
|
32
32
|
name: activemodel
|
33
33
|
requirement: !ruby/object:Gem::Requirement
|
@@ -35,7 +35,7 @@ dependencies:
|
|
35
35
|
requirements:
|
36
36
|
- - ! '>='
|
37
37
|
- !ruby/object:Gem::Version
|
38
|
-
version:
|
38
|
+
version: 3.0.0
|
39
39
|
type: :runtime
|
40
40
|
prerelease: false
|
41
41
|
version_requirements: !ruby/object:Gem::Requirement
|
@@ -43,71 +43,71 @@ dependencies:
|
|
43
43
|
requirements:
|
44
44
|
- - ! '>='
|
45
45
|
- !ruby/object:Gem::Version
|
46
|
-
version:
|
46
|
+
version: 3.0.0
|
47
47
|
- !ruby/object:Gem::Dependency
|
48
48
|
name: bunny
|
49
49
|
requirement: !ruby/object:Gem::Requirement
|
50
50
|
none: false
|
51
51
|
requirements:
|
52
|
-
- -
|
52
|
+
- - ~>
|
53
53
|
- !ruby/object:Gem::Version
|
54
|
-
version:
|
54
|
+
version: 0.8.0
|
55
55
|
type: :runtime
|
56
56
|
prerelease: false
|
57
57
|
version_requirements: !ruby/object:Gem::Requirement
|
58
58
|
none: false
|
59
59
|
requirements:
|
60
|
-
- -
|
60
|
+
- - ~>
|
61
61
|
- !ruby/object:Gem::Version
|
62
|
-
version:
|
62
|
+
version: 0.8.0
|
63
63
|
- !ruby/object:Gem::Dependency
|
64
64
|
name: amqp
|
65
65
|
requirement: !ruby/object:Gem::Requirement
|
66
66
|
none: false
|
67
67
|
requirements:
|
68
|
-
- -
|
68
|
+
- - ~>
|
69
69
|
- !ruby/object:Gem::Version
|
70
|
-
version:
|
70
|
+
version: 0.9.8
|
71
71
|
type: :runtime
|
72
72
|
prerelease: false
|
73
73
|
version_requirements: !ruby/object:Gem::Requirement
|
74
74
|
none: false
|
75
75
|
requirements:
|
76
|
-
- -
|
76
|
+
- - ~>
|
77
77
|
- !ruby/object:Gem::Version
|
78
|
-
version:
|
78
|
+
version: 0.9.8
|
79
79
|
- !ruby/object:Gem::Dependency
|
80
80
|
name: em-synchrony
|
81
81
|
requirement: !ruby/object:Gem::Requirement
|
82
82
|
none: false
|
83
83
|
requirements:
|
84
|
-
- -
|
84
|
+
- - ~>
|
85
85
|
- !ruby/object:Gem::Version
|
86
|
-
version:
|
86
|
+
version: 1.0.3
|
87
87
|
type: :runtime
|
88
88
|
prerelease: false
|
89
89
|
version_requirements: !ruby/object:Gem::Requirement
|
90
90
|
none: false
|
91
91
|
requirements:
|
92
|
-
- -
|
92
|
+
- - ~>
|
93
93
|
- !ruby/object:Gem::Version
|
94
|
-
version:
|
94
|
+
version: 1.0.3
|
95
95
|
- !ruby/object:Gem::Dependency
|
96
96
|
name: ruby-progressbar
|
97
97
|
requirement: !ruby/object:Gem::Requirement
|
98
98
|
none: false
|
99
99
|
requirements:
|
100
|
-
- -
|
100
|
+
- - ~>
|
101
101
|
- !ruby/object:Gem::Version
|
102
|
-
version:
|
102
|
+
version: 1.0.2
|
103
103
|
type: :runtime
|
104
104
|
prerelease: false
|
105
105
|
version_requirements: !ruby/object:Gem::Requirement
|
106
106
|
none: false
|
107
107
|
requirements:
|
108
|
-
- -
|
108
|
+
- - ~>
|
109
109
|
- !ruby/object:Gem::Version
|
110
|
-
version:
|
110
|
+
version: 1.0.2
|
111
111
|
- !ruby/object:Gem::Dependency
|
112
112
|
name: redis
|
113
113
|
requirement: !ruby/object:Gem::Requirement
|
@@ -129,49 +129,49 @@ dependencies:
|
|
129
129
|
requirement: !ruby/object:Gem::Requirement
|
130
130
|
none: false
|
131
131
|
requirements:
|
132
|
-
- -
|
132
|
+
- - ~>
|
133
133
|
- !ruby/object:Gem::Version
|
134
|
-
version:
|
134
|
+
version: 0.1.2
|
135
135
|
type: :runtime
|
136
136
|
prerelease: false
|
137
137
|
version_requirements: !ruby/object:Gem::Requirement
|
138
138
|
none: false
|
139
139
|
requirements:
|
140
|
-
- -
|
140
|
+
- - ~>
|
141
141
|
- !ruby/object:Gem::Version
|
142
|
-
version:
|
142
|
+
version: 0.1.2
|
143
143
|
- !ruby/object:Gem::Dependency
|
144
144
|
name: celluloid
|
145
145
|
requirement: !ruby/object:Gem::Requirement
|
146
146
|
none: false
|
147
147
|
requirements:
|
148
|
-
- -
|
148
|
+
- - ~>
|
149
149
|
- !ruby/object:Gem::Version
|
150
|
-
version:
|
150
|
+
version: 0.12.4
|
151
151
|
type: :runtime
|
152
152
|
prerelease: false
|
153
153
|
version_requirements: !ruby/object:Gem::Requirement
|
154
154
|
none: false
|
155
155
|
requirements:
|
156
|
-
- -
|
156
|
+
- - ~>
|
157
157
|
- !ruby/object:Gem::Version
|
158
|
-
version:
|
158
|
+
version: 0.12.4
|
159
159
|
- !ruby/object:Gem::Dependency
|
160
160
|
name: celluloid-io
|
161
161
|
requirement: !ruby/object:Gem::Requirement
|
162
162
|
none: false
|
163
163
|
requirements:
|
164
|
-
- -
|
164
|
+
- - ~>
|
165
165
|
- !ruby/object:Gem::Version
|
166
|
-
version:
|
166
|
+
version: 0.12.1
|
167
167
|
type: :runtime
|
168
168
|
prerelease: false
|
169
169
|
version_requirements: !ruby/object:Gem::Requirement
|
170
170
|
none: false
|
171
171
|
requirements:
|
172
|
-
- -
|
172
|
+
- - ~>
|
173
173
|
- !ruby/object:Gem::Version
|
174
|
-
version:
|
174
|
+
version: 0.12.1
|
175
175
|
description: Replicate your Mongoid/ActiveRecord models across your applications
|
176
176
|
email:
|
177
177
|
- nicolas@viennot.biz
|
@@ -181,9 +181,9 @@ executables:
|
|
181
181
|
extensions: []
|
182
182
|
extra_rdoc_files: []
|
183
183
|
files:
|
184
|
+
- lib/promiscuous/amqp/null.rb
|
184
185
|
- lib/promiscuous/amqp/bunny.rb
|
185
186
|
- lib/promiscuous/amqp/ruby_amqp.rb
|
186
|
-
- lib/promiscuous/amqp/null.rb
|
187
187
|
- lib/promiscuous/common/lint/base.rb
|
188
188
|
- lib/promiscuous/common/class_helpers.rb
|
189
189
|
- lib/promiscuous/common/options.rb
|
@@ -207,8 +207,8 @@ files:
|
|
207
207
|
- lib/promiscuous/publisher/base.rb
|
208
208
|
- lib/promiscuous/publisher/lint.rb
|
209
209
|
- lib/promiscuous/publisher/mongoid.rb
|
210
|
-
- lib/promiscuous/publisher/model.rb
|
211
210
|
- lib/promiscuous/publisher/amqp.rb
|
211
|
+
- lib/promiscuous/publisher/model.rb
|
212
212
|
- lib/promiscuous/subscriber/lint/amqp.rb
|
213
213
|
- lib/promiscuous/subscriber/lint/base.rb
|
214
214
|
- lib/promiscuous/subscriber/lint/class.rb
|
@@ -216,26 +216,26 @@ files:
|
|
216
216
|
- lib/promiscuous/subscriber/lint/attributes.rb
|
217
217
|
- lib/promiscuous/subscriber/mongoid/embedded_many.rb
|
218
218
|
- lib/promiscuous/subscriber/mongoid/embedded.rb
|
219
|
-
- lib/promiscuous/subscriber/worker/message.rb
|
220
|
-
- lib/promiscuous/subscriber/worker/runner.rb
|
221
219
|
- lib/promiscuous/subscriber/worker/pump.rb
|
220
|
+
- lib/promiscuous/subscriber/worker/runner.rb
|
222
221
|
- lib/promiscuous/subscriber/worker/message_synchronizer.rb
|
222
|
+
- lib/promiscuous/subscriber/worker/message.rb
|
223
223
|
- lib/promiscuous/subscriber/active_record.rb
|
224
224
|
- lib/promiscuous/subscriber/envelope.rb
|
225
225
|
- lib/promiscuous/subscriber/upsert.rb
|
226
|
-
- lib/promiscuous/subscriber/observer.rb
|
227
226
|
- lib/promiscuous/subscriber/polymorphic.rb
|
228
227
|
- lib/promiscuous/subscriber/amqp.rb
|
229
228
|
- lib/promiscuous/subscriber/attributes.rb
|
230
|
-
- lib/promiscuous/subscriber/model.rb
|
231
229
|
- lib/promiscuous/subscriber/base.rb
|
232
230
|
- lib/promiscuous/subscriber/class.rb
|
233
231
|
- lib/promiscuous/subscriber/lint.rb
|
234
232
|
- lib/promiscuous/subscriber/mongoid.rb
|
233
|
+
- lib/promiscuous/subscriber/observer.rb
|
235
234
|
- lib/promiscuous/subscriber/worker.rb
|
235
|
+
- lib/promiscuous/subscriber/model.rb
|
236
236
|
- lib/promiscuous/error/subscriber.rb
|
237
|
-
- lib/promiscuous/error/connection.rb
|
238
237
|
- lib/promiscuous/error/publisher.rb
|
238
|
+
- lib/promiscuous/error/connection.rb
|
239
239
|
- lib/promiscuous/observer.rb
|
240
240
|
- lib/promiscuous/ephemeral.rb
|
241
241
|
- lib/promiscuous/autoload.rb
|
@@ -243,13 +243,13 @@ files:
|
|
243
243
|
- lib/promiscuous/error.rb
|
244
244
|
- lib/promiscuous/loader.rb
|
245
245
|
- lib/promiscuous/railtie.rb
|
246
|
-
- lib/promiscuous/cli.rb
|
247
246
|
- lib/promiscuous/common.rb
|
248
247
|
- lib/promiscuous/publisher.rb
|
249
248
|
- lib/promiscuous/worker.rb
|
250
|
-
- lib/promiscuous/amqp.rb
|
251
249
|
- lib/promiscuous/config.rb
|
250
|
+
- lib/promiscuous/amqp.rb
|
252
251
|
- lib/promiscuous/redis.rb
|
252
|
+
- lib/promiscuous/cli.rb
|
253
253
|
- lib/promiscuous/version.rb
|
254
254
|
- lib/promiscuous.rb
|
255
255
|
- bin/promiscuous
|