promiscuous 0.25 → 0.31.0
Sign up to get free protection for your applications and to get access to all the features.
- data/lib/promiscuous/amqp/bunny.rb +1 -1
- data/lib/promiscuous/amqp/null.rb +1 -0
- data/lib/promiscuous/amqp/ruby_amqp.rb +17 -14
- data/lib/promiscuous/amqp.rb +4 -2
- data/lib/promiscuous/cli.rb +9 -16
- data/lib/promiscuous/common.rb +1 -1
- data/lib/promiscuous/config.rb +4 -2
- data/lib/promiscuous/error/connection.rb +8 -1
- data/lib/promiscuous/error/publisher.rb +7 -1
- data/lib/promiscuous/publisher/active_record.rb +1 -0
- data/lib/promiscuous/publisher/amqp.rb +2 -1
- data/lib/promiscuous/publisher/base.rb +4 -0
- data/lib/promiscuous/publisher/lint.rb +1 -1
- data/lib/promiscuous/publisher/model/active_record.rb +25 -0
- data/lib/promiscuous/publisher/model/mongoid.rb +108 -0
- data/lib/promiscuous/publisher/model.rb +71 -19
- data/lib/promiscuous/publisher/mongoid/embedded.rb +8 -4
- data/lib/promiscuous/publisher/mongoid.rb +12 -9
- data/lib/promiscuous/publisher.rb +1 -1
- data/lib/promiscuous/redis.rb +52 -0
- data/lib/promiscuous/subscriber/mongoid.rb +1 -2
- data/lib/promiscuous/subscriber/worker/message.rb +46 -0
- data/lib/promiscuous/subscriber/worker/message_synchronizer.rb +168 -0
- data/lib/promiscuous/subscriber/worker/pump.rb +47 -0
- data/lib/promiscuous/subscriber/worker/runner.rb +7 -0
- data/lib/promiscuous/subscriber/worker.rb +39 -47
- data/lib/promiscuous/version.rb +1 -1
- data/lib/promiscuous/worker.rb +1 -5
- data/lib/promiscuous.rb +1 -1
- metadata +86 -20
- data/lib/promiscuous/common/worker.rb +0 -52
- data/lib/promiscuous/publisher/mongoid/defer.rb +0 -75
- data/lib/promiscuous/publisher/mongoid/defer_embedded.rb +0 -25
- data/lib/promiscuous/publisher/worker.rb +0 -101
- data/lib/promiscuous/subscriber/mongoid/versioning.rb +0 -40
@@ -1,75 +0,0 @@
|
|
1
|
-
module Promiscuous::Publisher::Mongoid::Defer
|
2
|
-
extend ActiveSupport::Concern
|
3
|
-
include Promiscuous::Publisher::Envelope
|
4
|
-
|
5
|
-
mattr_accessor :klasses
|
6
|
-
mattr_accessor :collections
|
7
|
-
self.klasses = {}
|
8
|
-
self.collections = {}
|
9
|
-
|
10
|
-
def publish
|
11
|
-
super unless should_defer?
|
12
|
-
end
|
13
|
-
|
14
|
-
def payload
|
15
|
-
super.merge(:version => instance._psv)
|
16
|
-
end
|
17
|
-
|
18
|
-
def should_defer?
|
19
|
-
if options.has_key?(:defer)
|
20
|
-
options[:defer]
|
21
|
-
else
|
22
|
-
operation == :update
|
23
|
-
end
|
24
|
-
end
|
25
|
-
|
26
|
-
def self.hook_mongoid
|
27
|
-
return if @mongoid_hooked
|
28
|
-
@mongoid_hooked = true
|
29
|
-
|
30
|
-
Moped::Query.class_eval do
|
31
|
-
alias_method :update_orig, :update
|
32
|
-
def update(change, flags = nil)
|
33
|
-
change = promiscuous_seasoning(change)
|
34
|
-
update_orig(change, flags)
|
35
|
-
end
|
36
|
-
|
37
|
-
alias_method :modify_orig, :modify
|
38
|
-
def modify(change, options={})
|
39
|
-
change = promiscuous_seasoning(change) unless options[:bypass_promiscuous]
|
40
|
-
modify_orig(change, options)
|
41
|
-
end
|
42
|
-
|
43
|
-
def promiscuous_seasoning(change)
|
44
|
-
if Promiscuous::Publisher::Mongoid::Defer.collections[@collection.name]
|
45
|
-
change = change.dup
|
46
|
-
change['$set'] ||= {}
|
47
|
-
change['$inc'] ||= {}
|
48
|
-
change['$set'].merge!(:_psp => true)
|
49
|
-
change['$inc'].merge!(:_psv => 1)
|
50
|
-
end
|
51
|
-
change
|
52
|
-
end
|
53
|
-
end
|
54
|
-
end
|
55
|
-
|
56
|
-
module ClassMethods
|
57
|
-
def setup_class_binding
|
58
|
-
super
|
59
|
-
klass.class_eval do
|
60
|
-
cattr_accessor :publisher_defer_hooked
|
61
|
-
return if self.publisher_defer_hooked
|
62
|
-
self.publisher_defer_hooked = true
|
63
|
-
|
64
|
-
# TODO Make sure we are not overriding a field, although VERY unlikly
|
65
|
-
field :_psp, :type => Boolean
|
66
|
-
field :_psv, :type => Integer
|
67
|
-
index({:_psp => 1}, :background => true, :sparse => true)
|
68
|
-
|
69
|
-
Promiscuous::Publisher::Mongoid::Defer.hook_mongoid
|
70
|
-
Promiscuous::Publisher::Mongoid::Defer.klasses[self.to_s] = self
|
71
|
-
Promiscuous::Publisher::Mongoid::Defer.collections[collection.name] = true
|
72
|
-
end if klass
|
73
|
-
end
|
74
|
-
end
|
75
|
-
end
|
@@ -1,25 +0,0 @@
|
|
1
|
-
module Promiscuous::Publisher::Mongoid::DeferEmbedded
|
2
|
-
extend ActiveSupport::Concern
|
3
|
-
|
4
|
-
def payload
|
5
|
-
if instance.is_a?(Array)
|
6
|
-
Promiscuous::Publisher::Mongoid::EmbeddedMany.new(:instance => instance).payload
|
7
|
-
else
|
8
|
-
super.merge(:id => instance.id)
|
9
|
-
end
|
10
|
-
end
|
11
|
-
|
12
|
-
included do
|
13
|
-
klass.class_eval do
|
14
|
-
callback = proc do
|
15
|
-
if _parent.respond_to?(:promiscuous_publish_update)
|
16
|
-
_parent.promiscuous_publish_update
|
17
|
-
end
|
18
|
-
end
|
19
|
-
|
20
|
-
before_create callback
|
21
|
-
before_update callback
|
22
|
-
before_destroy callback
|
23
|
-
end
|
24
|
-
end
|
25
|
-
end
|
@@ -1,101 +0,0 @@
|
|
1
|
-
class Promiscuous::Publisher::Worker
|
2
|
-
include Promiscuous::Common::Worker
|
3
|
-
|
4
|
-
def self.poll_delay
|
5
|
-
# TODO Configurable globally
|
6
|
-
# TODO Configurable per publisher
|
7
|
-
1.second
|
8
|
-
end
|
9
|
-
|
10
|
-
def initialize(options={})
|
11
|
-
super
|
12
|
-
check_indexes
|
13
|
-
end
|
14
|
-
|
15
|
-
def check_indexes
|
16
|
-
Promiscuous::Publisher::Mongoid::Defer.klasses.values.each do |klass|
|
17
|
-
unless klass.collection.indexes.any? { |i| i['key'].keys.include? '_psp' }
|
18
|
-
raise 'Please run rake db:mongoid:create_indexes'
|
19
|
-
end
|
20
|
-
end
|
21
|
-
end
|
22
|
-
|
23
|
-
def resume
|
24
|
-
@timer ||= EM::PeriodicTimer.new(self.class.poll_delay) { self.replicate_once }
|
25
|
-
super
|
26
|
-
end
|
27
|
-
|
28
|
-
def stop
|
29
|
-
@timer.try(:cancel)
|
30
|
-
@timer = nil
|
31
|
-
super
|
32
|
-
end
|
33
|
-
|
34
|
-
def replicate_once
|
35
|
-
return if self.stopped?
|
36
|
-
|
37
|
-
self.unit_of_work('publisher') do
|
38
|
-
maybe_rescue_instance
|
39
|
-
replicate_all_collections
|
40
|
-
end
|
41
|
-
rescue Exception => e
|
42
|
-
unless e.is_a?(Promiscuous::Error::Publisher)
|
43
|
-
e = Promiscuous::Error::Publisher.new(e)
|
44
|
-
end
|
45
|
-
|
46
|
-
retry_msg = stop_for_a_while(e)
|
47
|
-
Promiscuous.warn "[publish] (#{retry_msg}) #{e} #{e.backtrace.join("\n")}"
|
48
|
-
|
49
|
-
if e.out_of_sync
|
50
|
-
Promiscuous.error "[publish] WARNING out of sync on #{e.instance.inspect}"
|
51
|
-
@out_of_sync_instance = e.instance
|
52
|
-
end
|
53
|
-
|
54
|
-
Promiscuous::Config.error_notifier.try(:call, e)
|
55
|
-
end
|
56
|
-
|
57
|
-
def replicate_all_collections
|
58
|
-
Promiscuous::Publisher::Mongoid::Defer.klasses.values.each do |klass|
|
59
|
-
replicate_collection(klass)
|
60
|
-
end
|
61
|
-
made_progress
|
62
|
-
end
|
63
|
-
|
64
|
-
def replicate_collection(klass)
|
65
|
-
loop do
|
66
|
-
break if self.stopped?
|
67
|
-
|
68
|
-
instance = klass.where(:_psp => true).
|
69
|
-
find_and_modify({'$unset' => {:_psp => 1}}, :bypass_promiscuous => true)
|
70
|
-
break unless instance
|
71
|
-
|
72
|
-
replicate_instance(instance)
|
73
|
-
end
|
74
|
-
end
|
75
|
-
|
76
|
-
def replicate_instance(instance)
|
77
|
-
instance.promiscuous_sync
|
78
|
-
made_progress
|
79
|
-
rescue Exception => e
|
80
|
-
out_of_sync = requeue_instance(instance)
|
81
|
-
raise Promiscuous::Error::Publisher.new(e, :instance => instance,
|
82
|
-
:out_of_sync => out_of_sync)
|
83
|
-
end
|
84
|
-
|
85
|
-
def requeue_instance(instance)
|
86
|
-
# The following update will set the _psp flag to true again,
|
87
|
-
# effectively requeuing the publish action.
|
88
|
-
instance.class.where(instance.atomic_selector).update({})
|
89
|
-
false
|
90
|
-
rescue Exception
|
91
|
-
# Swallow exception of a failed recovery.
|
92
|
-
# The user needs to manually resync.
|
93
|
-
true
|
94
|
-
end
|
95
|
-
|
96
|
-
def maybe_rescue_instance
|
97
|
-
return unless @out_of_sync_instance
|
98
|
-
replicate_instance(@out_of_sync_instance)
|
99
|
-
@out_of_sync_instance = nil
|
100
|
-
end
|
101
|
-
end
|
@@ -1,40 +0,0 @@
|
|
1
|
-
module Promiscuous::Subscriber::Mongoid::Versioning
|
2
|
-
extend ActiveSupport::Concern
|
3
|
-
|
4
|
-
module AtomicSelector
|
5
|
-
extend ActiveSupport::Concern
|
6
|
-
|
7
|
-
def atomic_selector
|
8
|
-
if use_atomic_promiscuous_selector
|
9
|
-
super.merge({ '$or' => [{'_psv' => { '$lte' => self._psv }},
|
10
|
-
{'_psv' => { '$exists' => false }}]})
|
11
|
-
else
|
12
|
-
super
|
13
|
-
end
|
14
|
-
end
|
15
|
-
|
16
|
-
included do
|
17
|
-
attr_accessor :use_atomic_promiscuous_selector
|
18
|
-
field :_psv, :type => Integer
|
19
|
-
end
|
20
|
-
end
|
21
|
-
|
22
|
-
def save_instance
|
23
|
-
if version
|
24
|
-
instance._psv = version
|
25
|
-
instance.use_atomic_promiscuous_selector = true
|
26
|
-
end
|
27
|
-
super
|
28
|
-
ensure
|
29
|
-
instance.use_atomic_promiscuous_selector = false
|
30
|
-
end
|
31
|
-
|
32
|
-
included { use_payload_attribute :version }
|
33
|
-
|
34
|
-
module ClassMethods
|
35
|
-
def setup_class_binding
|
36
|
-
super
|
37
|
-
klass.__send__(:include, AtomicSelector) if klass
|
38
|
-
end
|
39
|
-
end
|
40
|
-
end
|