promiscuous 0.91.0 → 0.92.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/promiscuous.rb +1 -1
- data/lib/promiscuous/amqp.rb +1 -1
- data/lib/promiscuous/amqp/fake.rb +0 -3
- data/lib/promiscuous/amqp/file.rb +81 -0
- data/lib/promiscuous/amqp/null.rb +0 -3
- data/lib/promiscuous/cli.rb +35 -25
- data/lib/promiscuous/config.rb +8 -3
- data/lib/promiscuous/error.rb +1 -2
- data/lib/promiscuous/key.rb +1 -12
- data/lib/promiscuous/mongoid.rb +7 -0
- data/lib/promiscuous/publisher/context/base.rb +4 -4
- data/lib/promiscuous/publisher/context/middleware.rb +2 -23
- data/lib/promiscuous/publisher/model/ephemeral.rb +5 -1
- data/lib/promiscuous/publisher/model/mock.rb +9 -7
- data/lib/promiscuous/publisher/model/mongoid.rb +3 -1
- data/lib/promiscuous/publisher/operation.rb +1 -1
- data/lib/promiscuous/publisher/operation/atomic.rb +44 -32
- data/lib/promiscuous/publisher/operation/base.rb +14 -9
- data/lib/promiscuous/publisher/operation/ephemeral.rb +14 -0
- data/lib/promiscuous/publisher/operation/mongoid.rb +4 -12
- data/lib/promiscuous/subscriber/message_processor/base.rb +17 -1
- data/lib/promiscuous/subscriber/message_processor/regular.rb +94 -48
- data/lib/promiscuous/subscriber/model/active_record.rb +25 -0
- data/lib/promiscuous/subscriber/model/base.rb +17 -13
- data/lib/promiscuous/subscriber/model/mongoid.rb +20 -1
- data/lib/promiscuous/subscriber/model/observer.rb +4 -0
- data/lib/promiscuous/subscriber/operation/base.rb +14 -16
- data/lib/promiscuous/subscriber/operation/bootstrap.rb +7 -1
- data/lib/promiscuous/subscriber/operation/regular.rb +6 -0
- data/lib/promiscuous/subscriber/worker.rb +6 -2
- data/lib/promiscuous/subscriber/worker/eventual_destroyer.rb +85 -0
- data/lib/promiscuous/subscriber/worker/message.rb +9 -15
- data/lib/promiscuous/subscriber/worker/message_synchronizer.rb +24 -78
- data/lib/promiscuous/subscriber/worker/runner.rb +6 -2
- data/lib/promiscuous/subscriber/worker/stats.rb +11 -7
- data/lib/promiscuous/version.rb +1 -1
- metadata +66 -63
- data/lib/promiscuous/error/already_processed.rb +0 -5
@@ -32,33 +32,31 @@ class Promiscuous::Subscriber::Operation::Base
|
|
32
32
|
instance.save!
|
33
33
|
end
|
34
34
|
rescue Exception => e
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
dup_index_error = true if defined?(ActiveRecord) && e.is_a?(ActiveRecord::RecordNotUnique)
|
39
|
-
|
40
|
-
if dup_index_error
|
41
|
-
if options[:upsert]
|
42
|
-
update
|
43
|
-
else
|
44
|
-
warn "ignoring already created record"
|
45
|
-
end
|
35
|
+
if model.__promiscuous_duplicate_key_exception?(e)
|
36
|
+
options[:on_already_created] ||= proc { warn "ignoring already created record" }
|
37
|
+
options[:on_already_created].call
|
46
38
|
else
|
47
39
|
raise e
|
48
40
|
end
|
49
41
|
end
|
50
42
|
|
51
|
-
def update
|
43
|
+
def update(should_create_on_failure=true)
|
52
44
|
model.__promiscuous_fetch_existing(id).tap do |instance|
|
53
|
-
instance.
|
54
|
-
|
45
|
+
if instance.__promiscuous_eventual_consistency_update(self)
|
46
|
+
instance.__promiscuous_update(self)
|
47
|
+
instance.save!
|
48
|
+
end
|
55
49
|
end
|
56
50
|
rescue model.__promiscuous_missing_record_exception
|
57
|
-
warn "upserting
|
58
|
-
create
|
51
|
+
warn "upserting"
|
52
|
+
create :on_already_created => proc { update(false) if should_create_on_failure }
|
59
53
|
end
|
60
54
|
|
61
55
|
def destroy
|
56
|
+
if Promiscuous::Config.consistency == :eventual
|
57
|
+
Promiscuous::Subscriber::Worker::EventualDestroyer.postpone_destroy(model, id)
|
58
|
+
end
|
59
|
+
|
62
60
|
model.__promiscuous_fetch_existing(id).tap do |instance|
|
63
61
|
instance.destroy
|
64
62
|
end
|
@@ -14,7 +14,7 @@ class Promiscuous::Subscriber::Operation::Bootstrap < Promiscuous::Subscriber::O
|
|
14
14
|
end
|
15
15
|
|
16
16
|
def bootstrap_data
|
17
|
-
create
|
17
|
+
create :on_already_created => proc { update }
|
18
18
|
end
|
19
19
|
|
20
20
|
def on_bootstrap_operation(wanted_operation, options={})
|
@@ -46,6 +46,12 @@ class Promiscuous::Subscriber::Operation::Bootstrap < Promiscuous::Subscriber::O
|
|
46
46
|
else
|
47
47
|
raise "Invalid operation received: #{operation}"
|
48
48
|
end
|
49
|
+
rescue Exception => e
|
50
|
+
if Promiscuous::Config.ignore_exceptions
|
51
|
+
Promiscuous.warn "[receive] error while proceessing message but message still processed: #{e}\n#{e.backtrace.join("\n")}"
|
52
|
+
else
|
53
|
+
raise e
|
54
|
+
end
|
49
55
|
end
|
50
56
|
|
51
57
|
def message_processor
|
@@ -5,6 +5,12 @@ class Promiscuous::Subscriber::Operation::Regular < Promiscuous::Subscriber::Ope
|
|
5
5
|
when :update then update if model
|
6
6
|
when :destroy then destroy if model
|
7
7
|
end
|
8
|
+
rescue Exception => e
|
9
|
+
if Promiscuous::Config.ignore_exceptions && !e.is_a?(NameError)
|
10
|
+
Promiscuous.warn "[receive] error while proceessing message but message still processed: #{e}\n#{e.backtrace.join("\n")}"
|
11
|
+
else
|
12
|
+
raise e
|
13
|
+
end
|
8
14
|
end
|
9
15
|
|
10
16
|
def message_processor
|
@@ -1,14 +1,16 @@
|
|
1
1
|
class Promiscuous::Subscriber::Worker
|
2
2
|
extend Promiscuous::Autoload
|
3
|
-
autoload :Message, :Pump, :MessageSynchronizer, :Runner, :Stats, :Recorder
|
3
|
+
autoload :Message, :Pump, :MessageSynchronizer, :Runner, :Stats, :Recorder,
|
4
|
+
:EventualDestroyer
|
4
5
|
|
5
|
-
attr_accessor :message_synchronizer, :pump, :runner, :stats
|
6
|
+
attr_accessor :message_synchronizer, :pump, :runner, :stats, :eventual_destroyer
|
6
7
|
|
7
8
|
def initialize
|
8
9
|
@message_synchronizer = MessageSynchronizer.new(self)
|
9
10
|
@pump = Pump.new(self)
|
10
11
|
@runner = Runner.new(self)
|
11
12
|
@stats = Stats.new
|
13
|
+
@eventual_destroyer = EventualDestroyer.new if Promiscuous::Config.consistency == :eventual
|
12
14
|
end
|
13
15
|
|
14
16
|
def start
|
@@ -16,6 +18,7 @@ class Promiscuous::Subscriber::Worker
|
|
16
18
|
@pump.connect
|
17
19
|
@runner.start
|
18
20
|
@stats.connect
|
21
|
+
@eventual_destroyer.try(:start)
|
19
22
|
end
|
20
23
|
|
21
24
|
def stop
|
@@ -23,6 +26,7 @@ class Promiscuous::Subscriber::Worker
|
|
23
26
|
@runner.stop
|
24
27
|
@pump.disconnect
|
25
28
|
@message_synchronizer.disconnect
|
29
|
+
@eventual_destroyer.try(:stop)
|
26
30
|
end
|
27
31
|
|
28
32
|
def show_stop_status
|
@@ -0,0 +1,85 @@
|
|
1
|
+
class Promiscuous::Subscriber::Worker::EventualDestroyer
|
2
|
+
def start
|
3
|
+
@thread ||= Thread.new { main_loop }
|
4
|
+
end
|
5
|
+
|
6
|
+
def stop
|
7
|
+
@thread.try(:kill)
|
8
|
+
@thread = nil
|
9
|
+
end
|
10
|
+
|
11
|
+
def self.destroy_timeout
|
12
|
+
1.hour
|
13
|
+
end
|
14
|
+
|
15
|
+
def self.check_every
|
16
|
+
(10 + rand(10)).minutes
|
17
|
+
end
|
18
|
+
|
19
|
+
def main_loop
|
20
|
+
loop do
|
21
|
+
begin
|
22
|
+
PendingDestroy.next(self.class.destroy_timeout).each(&:perform)
|
23
|
+
rescue Exception => e
|
24
|
+
Promiscuous.warn "[eventual destroyer] #{e}\n#{e.backtrace.join("\n")}"
|
25
|
+
Promiscuous::Config.error_notifier.call(e)
|
26
|
+
end
|
27
|
+
|
28
|
+
sleep self.class.check_every.to_f
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
def self.postpone_destroy(model, id)
|
33
|
+
PendingDestroy.create(:class_name => model.to_s, :instance_id => id)
|
34
|
+
end
|
35
|
+
|
36
|
+
class PendingDestroy
|
37
|
+
attr_accessor :class_name, :instance_id
|
38
|
+
|
39
|
+
def perform
|
40
|
+
model = class_name.constantize
|
41
|
+
begin
|
42
|
+
model.__promiscuous_fetch_existing(instance_id).destroy
|
43
|
+
rescue model.__promiscuous_missing_record_exception
|
44
|
+
end
|
45
|
+
|
46
|
+
self.destroy
|
47
|
+
end
|
48
|
+
|
49
|
+
def destroy
|
50
|
+
self.class.redis.zrem(self.class.key, @raw)
|
51
|
+
end
|
52
|
+
|
53
|
+
def initialize(raw)
|
54
|
+
params = MultiJson.load(raw).with_indifferent_access
|
55
|
+
|
56
|
+
@class_name = params[:class_name]
|
57
|
+
@instance_id = params[:instance_id]
|
58
|
+
@raw = raw
|
59
|
+
end
|
60
|
+
|
61
|
+
def self.next(timeout)
|
62
|
+
redis.zrangebyscore(key, 0, timeout.ago.utc.to_i).map do |raw|
|
63
|
+
self.new(raw)
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
def self.create(options)
|
68
|
+
redis.zadd(key, Time.now.utc.to_i, options.to_json)
|
69
|
+
end
|
70
|
+
|
71
|
+
def self.count
|
72
|
+
redis.zcard(key)
|
73
|
+
end
|
74
|
+
|
75
|
+
private
|
76
|
+
|
77
|
+
def self.redis
|
78
|
+
Promiscuous::Redis.master.nodes.first
|
79
|
+
end
|
80
|
+
|
81
|
+
def self.key
|
82
|
+
Promiscuous::Key.new(:sub).join('eventualdestroyer:jobs')
|
83
|
+
end
|
84
|
+
end
|
85
|
+
end
|
@@ -23,6 +23,10 @@ class Promiscuous::Subscriber::Worker::Message
|
|
23
23
|
parsed_payload['timestamp'].to_i
|
24
24
|
end
|
25
25
|
|
26
|
+
def generation
|
27
|
+
parsed_payload['generation']
|
28
|
+
end
|
29
|
+
|
26
30
|
def dependencies
|
27
31
|
@dependencies ||= begin
|
28
32
|
dependencies = parsed_payload['dependencies'] || {}
|
@@ -53,10 +57,13 @@ class Promiscuous::Subscriber::Worker::Message
|
|
53
57
|
end
|
54
58
|
|
55
59
|
def has_dependencies?
|
56
|
-
return false if Promiscuous::Config.no_deps
|
57
60
|
dependencies.present?
|
58
61
|
end
|
59
62
|
|
63
|
+
def was_during_bootstrap?
|
64
|
+
!!parsed_payload['was_during_bootstrap']
|
65
|
+
end
|
66
|
+
|
60
67
|
def to_s
|
61
68
|
"#{app}/#{context} -> #{happens_before_dependencies.join(', ')}"
|
62
69
|
end
|
@@ -82,26 +89,13 @@ class Promiscuous::Subscriber::Worker::Message
|
|
82
89
|
end
|
83
90
|
|
84
91
|
def unit_of_work(type, &block)
|
85
|
-
|
86
|
-
# middleware?
|
87
|
-
if defined?(Mongoid)
|
88
|
-
Mongoid.unit_of_work { yield_and_catch_already_processsed(&block) }
|
89
|
-
else
|
90
|
-
yield_and_catch_already_processsed(&block)
|
91
|
-
end
|
92
|
+
Promiscuous.context { yield }
|
92
93
|
ensure
|
93
94
|
if defined?(ActiveRecord)
|
94
95
|
ActiveRecord::Base.clear_active_connections!
|
95
96
|
end
|
96
97
|
end
|
97
98
|
|
98
|
-
def yield_and_catch_already_processsed
|
99
|
-
Promiscuous.context { yield }
|
100
|
-
rescue Promiscuous::Error::AlreadyProcessed => orig_e
|
101
|
-
e = Promiscuous::Error::Subscriber.new(orig_e, :payload => payload)
|
102
|
-
Promiscuous.debug "[receive] #{payload} #{e}\n#{e.backtrace.join("\n")}"
|
103
|
-
end
|
104
|
-
|
105
99
|
def process
|
106
100
|
unit_of_work(context) do
|
107
101
|
if Promiscuous::Config.bootstrap
|
@@ -6,7 +6,7 @@ class Promiscuous::Subscriber::Worker::MessageSynchronizer
|
|
6
6
|
CLEANUP_INTERVAL = 100 # messages
|
7
7
|
QUEUE_MAX_AGE = 100 # messages
|
8
8
|
|
9
|
-
attr_accessor :redis, :node_synchronizers, :num_processed_messages
|
9
|
+
attr_accessor :redis, :node_synchronizers, :num_processed_messages
|
10
10
|
|
11
11
|
def initialize(root)
|
12
12
|
@root = root
|
@@ -24,10 +24,12 @@ class Promiscuous::Subscriber::Worker::MessageSynchronizer
|
|
24
24
|
return unless !connected?
|
25
25
|
|
26
26
|
@num_processed_messages = 0
|
27
|
-
@num_queued_messages = 0
|
28
27
|
redis = Promiscuous::Redis.new_blocking_connection
|
29
28
|
redis.nodes.each { |node| @node_synchronizers[node] = NodeSynchronizer.new(self, node) }
|
30
29
|
@redis = redis
|
30
|
+
|
31
|
+
@message_queue = Queue.new
|
32
|
+
@processor_thread = Thread.new { queue_process_main_loop }
|
31
33
|
end
|
32
34
|
# Do not recover messages while bootstrapping as there are a very large
|
33
35
|
# number of messages that remain un-acked. If bootstrap messages are missed
|
@@ -44,6 +46,7 @@ class Promiscuous::Subscriber::Worker::MessageSynchronizer
|
|
44
46
|
@node_synchronizers.values.each { |node_synchronizer| node_synchronizer.stop_main_loop }
|
45
47
|
@node_synchronizers.clear
|
46
48
|
redis.quit
|
49
|
+
@processor_thread.kill
|
47
50
|
end
|
48
51
|
rescue Exception
|
49
52
|
end
|
@@ -70,7 +73,7 @@ class Promiscuous::Subscriber::Worker::MessageSynchronizer
|
|
70
73
|
# If not we bail out and rely on the subscription to kick the processing.
|
71
74
|
# Because we subscribed in advanced, we will not miss the notification.
|
72
75
|
def process_when_ready(msg)
|
73
|
-
unless msg.has_dependencies?
|
76
|
+
unless Promiscuous::Config.consistency == :causal && msg.has_dependencies?
|
74
77
|
process_message!(msg)
|
75
78
|
return
|
76
79
|
end
|
@@ -78,18 +81,25 @@ class Promiscuous::Subscriber::Worker::MessageSynchronizer
|
|
78
81
|
# Dropped messages will be redelivered as we (re)connect
|
79
82
|
return unless self.redis
|
80
83
|
|
81
|
-
@
|
84
|
+
@message_queue.push([msg, msg.happens_before_dependencies.dup])
|
85
|
+
end
|
86
|
+
|
87
|
+
def queue_process_main_loop
|
88
|
+
loop do
|
89
|
+
msg, deps = @message_queue.pop
|
82
90
|
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
subscriber_redis = dep.redis_node(@redis)
|
91
|
+
if dep = deps.pop
|
92
|
+
get_redis = dep.redis_node
|
93
|
+
subscriber_redis = dep.redis_node(@redis)
|
87
94
|
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
95
|
+
key = dep.key(:sub).join('rw').to_s
|
96
|
+
version = dep.version
|
97
|
+
node_synchronizer = @node_synchronizers[subscriber_redis]
|
98
|
+
node_synchronizer.on_version(subscriber_redis, get_redis, key, version, msg) { @message_queue.push([msg, deps]) }
|
99
|
+
else
|
100
|
+
process_message!(msg)
|
101
|
+
end
|
102
|
+
end
|
93
103
|
end
|
94
104
|
|
95
105
|
def process_message!(msg)
|
@@ -99,72 +109,12 @@ class Promiscuous::Subscriber::Worker::MessageSynchronizer
|
|
99
109
|
|
100
110
|
cleanup = false
|
101
111
|
@lock.synchronize do
|
102
|
-
@num_queued_messages -= 1
|
103
112
|
@num_processed_messages += 1
|
104
113
|
cleanup = @num_processed_messages % CLEANUP_INTERVAL == 0
|
105
114
|
end
|
106
115
|
@node_synchronizers.values.each(&:cleanup_if_old) if @node_synchronizers && cleanup
|
107
116
|
end
|
108
117
|
|
109
|
-
def maybe_recover
|
110
|
-
if @num_queued_messages == Promiscuous::Config.prefetch
|
111
|
-
# We've reached the amount of messages the amqp queue is willing to give us.
|
112
|
-
# We also know that we are not processing messages (@num_queued_messages is
|
113
|
-
# decremented before we send the message to the runners), and we are called
|
114
|
-
# after adding a pending callback.
|
115
|
-
recover_dependencies_for(blocked_messages.first)
|
116
|
-
end
|
117
|
-
end
|
118
|
-
|
119
|
-
def recover_dependencies_for(msg)
|
120
|
-
# XXX This recovery mechanism only works with one worker.
|
121
|
-
# We are taking the earliest message to unblock, but in reality we should
|
122
|
-
# do the DAG of the happens before dependencies, take root nodes
|
123
|
-
# of the disconnected graphs, and sort by timestamps if needed.
|
124
|
-
|
125
|
-
incremented_deps = {}
|
126
|
-
|
127
|
-
msg.happens_before_dependencies.each do |dep|
|
128
|
-
key = dep.key(:sub).join('rw')
|
129
|
-
guard_key = key.join('guard') if dep.write?
|
130
|
-
version = dep.version
|
131
|
-
|
132
|
-
@@version_recovery_script ||= Promiscuous::Redis::Script.new <<-SCRIPT
|
133
|
-
local key = ARGV[1]
|
134
|
-
local wanted_version = tonumber(ARGV[2])
|
135
|
-
local guard_key = ARGV[3]
|
136
|
-
|
137
|
-
if redis.call('exists', guard_key) == 1 then
|
138
|
-
return
|
139
|
-
end
|
140
|
-
|
141
|
-
local current_version = tonumber(redis.call('get', key)) or 0
|
142
|
-
|
143
|
-
if wanted_version > current_version then
|
144
|
-
redis.call('set', guard_key, 1)
|
145
|
-
redis.call('expire', guard_key, 10)
|
146
|
-
|
147
|
-
redis.call('set', key, wanted_version)
|
148
|
-
redis.call('publish', key, wanted_version)
|
149
|
-
return wanted_version - current_version
|
150
|
-
end
|
151
|
-
SCRIPT
|
152
|
-
increment = @@version_recovery_script.eval(dep.redis_node, :argv => [key, version, guard_key].compact)
|
153
|
-
incremented_deps[dep] = increment if increment
|
154
|
-
end
|
155
|
-
|
156
|
-
if incremented_deps.present?
|
157
|
-
recovery_msg = "Incrementing "
|
158
|
-
recovery_msg += incremented_deps.map { |dep, increment| "#{dep} by #{increment}" }.join(", ")
|
159
|
-
|
160
|
-
e = Promiscuous::Error::Recovery.new(recovery_msg)
|
161
|
-
Promiscuous.error "[synchronization recovery] #{e}"
|
162
|
-
|
163
|
-
# TODO Should we report the error to the notifier, or the log file is enough?
|
164
|
-
# Promiscuous::Config.error_notifier.call(e)
|
165
|
-
end
|
166
|
-
end
|
167
|
-
|
168
118
|
def blocked_messages
|
169
119
|
@node_synchronizers.values
|
170
120
|
.map { |node_synchronizer| node_synchronizer.blocked_messages }
|
@@ -364,11 +314,7 @@ class Promiscuous::Subscriber::Worker::MessageSynchronizer
|
|
364
314
|
end
|
365
315
|
end
|
366
316
|
|
367
|
-
if can_perform_immediately
|
368
|
-
callback.perform
|
369
|
-
else
|
370
|
-
node_synchronizer.root_synchronizer.maybe_recover if Promiscuous::Config.recovery
|
371
|
-
end
|
317
|
+
callback.perform if can_perform_immediately
|
372
318
|
end
|
373
319
|
|
374
320
|
class Callback < Struct.new(:version, :callback, :message)
|
@@ -21,7 +21,7 @@ class Promiscuous::Subscriber::Worker::Runner
|
|
21
21
|
end
|
22
22
|
|
23
23
|
def show_stop_status(num_requests)
|
24
|
-
@runner_threads.each { |runner_thread| runner_thread.show_stop_status(num_requests) }
|
24
|
+
@runner_threads.to_a.each { |runner_thread| runner_thread.show_stop_status(num_requests) }
|
25
25
|
end
|
26
26
|
|
27
27
|
class RunnerThread
|
@@ -43,7 +43,11 @@ class Promiscuous::Subscriber::Worker::Runner
|
|
43
43
|
end
|
44
44
|
|
45
45
|
def stop
|
46
|
-
@kill_lock.
|
46
|
+
if @kill_lock.locked? && @thread.stop?
|
47
|
+
@thread.kill
|
48
|
+
else
|
49
|
+
@kill_lock.synchronize { @thread.kill }
|
50
|
+
end
|
47
51
|
end
|
48
52
|
|
49
53
|
def show_stop_status(num_requests)
|