promiscuous 0.25 → 0.31.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/lib/promiscuous/amqp/bunny.rb +1 -1
- data/lib/promiscuous/amqp/null.rb +1 -0
- data/lib/promiscuous/amqp/ruby_amqp.rb +17 -14
- data/lib/promiscuous/amqp.rb +4 -2
- data/lib/promiscuous/cli.rb +9 -16
- data/lib/promiscuous/common.rb +1 -1
- data/lib/promiscuous/config.rb +4 -2
- data/lib/promiscuous/error/connection.rb +8 -1
- data/lib/promiscuous/error/publisher.rb +7 -1
- data/lib/promiscuous/publisher/active_record.rb +1 -0
- data/lib/promiscuous/publisher/amqp.rb +2 -1
- data/lib/promiscuous/publisher/base.rb +4 -0
- data/lib/promiscuous/publisher/lint.rb +1 -1
- data/lib/promiscuous/publisher/model/active_record.rb +25 -0
- data/lib/promiscuous/publisher/model/mongoid.rb +108 -0
- data/lib/promiscuous/publisher/model.rb +71 -19
- data/lib/promiscuous/publisher/mongoid/embedded.rb +8 -4
- data/lib/promiscuous/publisher/mongoid.rb +12 -9
- data/lib/promiscuous/publisher.rb +1 -1
- data/lib/promiscuous/redis.rb +52 -0
- data/lib/promiscuous/subscriber/mongoid.rb +1 -2
- data/lib/promiscuous/subscriber/worker/message.rb +46 -0
- data/lib/promiscuous/subscriber/worker/message_synchronizer.rb +168 -0
- data/lib/promiscuous/subscriber/worker/pump.rb +47 -0
- data/lib/promiscuous/subscriber/worker/runner.rb +7 -0
- data/lib/promiscuous/subscriber/worker.rb +39 -47
- data/lib/promiscuous/version.rb +1 -1
- data/lib/promiscuous/worker.rb +1 -5
- data/lib/promiscuous.rb +1 -1
- metadata +86 -20
- data/lib/promiscuous/common/worker.rb +0 -52
- data/lib/promiscuous/publisher/mongoid/defer.rb +0 -75
- data/lib/promiscuous/publisher/mongoid/defer_embedded.rb +0 -25
- data/lib/promiscuous/publisher/worker.rb +0 -101
- data/lib/promiscuous/subscriber/mongoid/versioning.rb +0 -40
@@ -1,75 +0,0 @@
|
|
1
|
-
module Promiscuous::Publisher::Mongoid::Defer
|
2
|
-
extend ActiveSupport::Concern
|
3
|
-
include Promiscuous::Publisher::Envelope
|
4
|
-
|
5
|
-
mattr_accessor :klasses
|
6
|
-
mattr_accessor :collections
|
7
|
-
self.klasses = {}
|
8
|
-
self.collections = {}
|
9
|
-
|
10
|
-
def publish
|
11
|
-
super unless should_defer?
|
12
|
-
end
|
13
|
-
|
14
|
-
def payload
|
15
|
-
super.merge(:version => instance._psv)
|
16
|
-
end
|
17
|
-
|
18
|
-
def should_defer?
|
19
|
-
if options.has_key?(:defer)
|
20
|
-
options[:defer]
|
21
|
-
else
|
22
|
-
operation == :update
|
23
|
-
end
|
24
|
-
end
|
25
|
-
|
26
|
-
def self.hook_mongoid
|
27
|
-
return if @mongoid_hooked
|
28
|
-
@mongoid_hooked = true
|
29
|
-
|
30
|
-
Moped::Query.class_eval do
|
31
|
-
alias_method :update_orig, :update
|
32
|
-
def update(change, flags = nil)
|
33
|
-
change = promiscuous_seasoning(change)
|
34
|
-
update_orig(change, flags)
|
35
|
-
end
|
36
|
-
|
37
|
-
alias_method :modify_orig, :modify
|
38
|
-
def modify(change, options={})
|
39
|
-
change = promiscuous_seasoning(change) unless options[:bypass_promiscuous]
|
40
|
-
modify_orig(change, options)
|
41
|
-
end
|
42
|
-
|
43
|
-
def promiscuous_seasoning(change)
|
44
|
-
if Promiscuous::Publisher::Mongoid::Defer.collections[@collection.name]
|
45
|
-
change = change.dup
|
46
|
-
change['$set'] ||= {}
|
47
|
-
change['$inc'] ||= {}
|
48
|
-
change['$set'].merge!(:_psp => true)
|
49
|
-
change['$inc'].merge!(:_psv => 1)
|
50
|
-
end
|
51
|
-
change
|
52
|
-
end
|
53
|
-
end
|
54
|
-
end
|
55
|
-
|
56
|
-
module ClassMethods
|
57
|
-
def setup_class_binding
|
58
|
-
super
|
59
|
-
klass.class_eval do
|
60
|
-
cattr_accessor :publisher_defer_hooked
|
61
|
-
return if self.publisher_defer_hooked
|
62
|
-
self.publisher_defer_hooked = true
|
63
|
-
|
64
|
-
# TODO Make sure we are not overriding a field, although VERY unlikly
|
65
|
-
field :_psp, :type => Boolean
|
66
|
-
field :_psv, :type => Integer
|
67
|
-
index({:_psp => 1}, :background => true, :sparse => true)
|
68
|
-
|
69
|
-
Promiscuous::Publisher::Mongoid::Defer.hook_mongoid
|
70
|
-
Promiscuous::Publisher::Mongoid::Defer.klasses[self.to_s] = self
|
71
|
-
Promiscuous::Publisher::Mongoid::Defer.collections[collection.name] = true
|
72
|
-
end if klass
|
73
|
-
end
|
74
|
-
end
|
75
|
-
end
|
@@ -1,25 +0,0 @@
|
|
1
|
-
module Promiscuous::Publisher::Mongoid::DeferEmbedded
|
2
|
-
extend ActiveSupport::Concern
|
3
|
-
|
4
|
-
def payload
|
5
|
-
if instance.is_a?(Array)
|
6
|
-
Promiscuous::Publisher::Mongoid::EmbeddedMany.new(:instance => instance).payload
|
7
|
-
else
|
8
|
-
super.merge(:id => instance.id)
|
9
|
-
end
|
10
|
-
end
|
11
|
-
|
12
|
-
included do
|
13
|
-
klass.class_eval do
|
14
|
-
callback = proc do
|
15
|
-
if _parent.respond_to?(:promiscuous_publish_update)
|
16
|
-
_parent.promiscuous_publish_update
|
17
|
-
end
|
18
|
-
end
|
19
|
-
|
20
|
-
before_create callback
|
21
|
-
before_update callback
|
22
|
-
before_destroy callback
|
23
|
-
end
|
24
|
-
end
|
25
|
-
end
|
@@ -1,101 +0,0 @@
|
|
1
|
-
class Promiscuous::Publisher::Worker
|
2
|
-
include Promiscuous::Common::Worker
|
3
|
-
|
4
|
-
def self.poll_delay
|
5
|
-
# TODO Configurable globally
|
6
|
-
# TODO Configurable per publisher
|
7
|
-
1.second
|
8
|
-
end
|
9
|
-
|
10
|
-
def initialize(options={})
|
11
|
-
super
|
12
|
-
check_indexes
|
13
|
-
end
|
14
|
-
|
15
|
-
def check_indexes
|
16
|
-
Promiscuous::Publisher::Mongoid::Defer.klasses.values.each do |klass|
|
17
|
-
unless klass.collection.indexes.any? { |i| i['key'].keys.include? '_psp' }
|
18
|
-
raise 'Please run rake db:mongoid:create_indexes'
|
19
|
-
end
|
20
|
-
end
|
21
|
-
end
|
22
|
-
|
23
|
-
def resume
|
24
|
-
@timer ||= EM::PeriodicTimer.new(self.class.poll_delay) { self.replicate_once }
|
25
|
-
super
|
26
|
-
end
|
27
|
-
|
28
|
-
def stop
|
29
|
-
@timer.try(:cancel)
|
30
|
-
@timer = nil
|
31
|
-
super
|
32
|
-
end
|
33
|
-
|
34
|
-
def replicate_once
|
35
|
-
return if self.stopped?
|
36
|
-
|
37
|
-
self.unit_of_work('publisher') do
|
38
|
-
maybe_rescue_instance
|
39
|
-
replicate_all_collections
|
40
|
-
end
|
41
|
-
rescue Exception => e
|
42
|
-
unless e.is_a?(Promiscuous::Error::Publisher)
|
43
|
-
e = Promiscuous::Error::Publisher.new(e)
|
44
|
-
end
|
45
|
-
|
46
|
-
retry_msg = stop_for_a_while(e)
|
47
|
-
Promiscuous.warn "[publish] (#{retry_msg}) #{e} #{e.backtrace.join("\n")}"
|
48
|
-
|
49
|
-
if e.out_of_sync
|
50
|
-
Promiscuous.error "[publish] WARNING out of sync on #{e.instance.inspect}"
|
51
|
-
@out_of_sync_instance = e.instance
|
52
|
-
end
|
53
|
-
|
54
|
-
Promiscuous::Config.error_notifier.try(:call, e)
|
55
|
-
end
|
56
|
-
|
57
|
-
def replicate_all_collections
|
58
|
-
Promiscuous::Publisher::Mongoid::Defer.klasses.values.each do |klass|
|
59
|
-
replicate_collection(klass)
|
60
|
-
end
|
61
|
-
made_progress
|
62
|
-
end
|
63
|
-
|
64
|
-
def replicate_collection(klass)
|
65
|
-
loop do
|
66
|
-
break if self.stopped?
|
67
|
-
|
68
|
-
instance = klass.where(:_psp => true).
|
69
|
-
find_and_modify({'$unset' => {:_psp => 1}}, :bypass_promiscuous => true)
|
70
|
-
break unless instance
|
71
|
-
|
72
|
-
replicate_instance(instance)
|
73
|
-
end
|
74
|
-
end
|
75
|
-
|
76
|
-
def replicate_instance(instance)
|
77
|
-
instance.promiscuous_sync
|
78
|
-
made_progress
|
79
|
-
rescue Exception => e
|
80
|
-
out_of_sync = requeue_instance(instance)
|
81
|
-
raise Promiscuous::Error::Publisher.new(e, :instance => instance,
|
82
|
-
:out_of_sync => out_of_sync)
|
83
|
-
end
|
84
|
-
|
85
|
-
def requeue_instance(instance)
|
86
|
-
# The following update will set the _psp flag to true again,
|
87
|
-
# effectively requeuing the publish action.
|
88
|
-
instance.class.where(instance.atomic_selector).update({})
|
89
|
-
false
|
90
|
-
rescue Exception
|
91
|
-
# Swallow exception of a failed recovery.
|
92
|
-
# The user needs to manually resync.
|
93
|
-
true
|
94
|
-
end
|
95
|
-
|
96
|
-
def maybe_rescue_instance
|
97
|
-
return unless @out_of_sync_instance
|
98
|
-
replicate_instance(@out_of_sync_instance)
|
99
|
-
@out_of_sync_instance = nil
|
100
|
-
end
|
101
|
-
end
|
@@ -1,40 +0,0 @@
|
|
1
|
-
module Promiscuous::Subscriber::Mongoid::Versioning
|
2
|
-
extend ActiveSupport::Concern
|
3
|
-
|
4
|
-
module AtomicSelector
|
5
|
-
extend ActiveSupport::Concern
|
6
|
-
|
7
|
-
def atomic_selector
|
8
|
-
if use_atomic_promiscuous_selector
|
9
|
-
super.merge({ '$or' => [{'_psv' => { '$lte' => self._psv }},
|
10
|
-
{'_psv' => { '$exists' => false }}]})
|
11
|
-
else
|
12
|
-
super
|
13
|
-
end
|
14
|
-
end
|
15
|
-
|
16
|
-
included do
|
17
|
-
attr_accessor :use_atomic_promiscuous_selector
|
18
|
-
field :_psv, :type => Integer
|
19
|
-
end
|
20
|
-
end
|
21
|
-
|
22
|
-
def save_instance
|
23
|
-
if version
|
24
|
-
instance._psv = version
|
25
|
-
instance.use_atomic_promiscuous_selector = true
|
26
|
-
end
|
27
|
-
super
|
28
|
-
ensure
|
29
|
-
instance.use_atomic_promiscuous_selector = false
|
30
|
-
end
|
31
|
-
|
32
|
-
included { use_payload_attribute :version }
|
33
|
-
|
34
|
-
module ClassMethods
|
35
|
-
def setup_class_binding
|
36
|
-
super
|
37
|
-
klass.__send__(:include, AtomicSelector) if klass
|
38
|
-
end
|
39
|
-
end
|
40
|
-
end
|