promiscuous 0.25 → 0.31.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (35) hide show
  1. data/lib/promiscuous/amqp/bunny.rb +1 -1
  2. data/lib/promiscuous/amqp/null.rb +1 -0
  3. data/lib/promiscuous/amqp/ruby_amqp.rb +17 -14
  4. data/lib/promiscuous/amqp.rb +4 -2
  5. data/lib/promiscuous/cli.rb +9 -16
  6. data/lib/promiscuous/common.rb +1 -1
  7. data/lib/promiscuous/config.rb +4 -2
  8. data/lib/promiscuous/error/connection.rb +8 -1
  9. data/lib/promiscuous/error/publisher.rb +7 -1
  10. data/lib/promiscuous/publisher/active_record.rb +1 -0
  11. data/lib/promiscuous/publisher/amqp.rb +2 -1
  12. data/lib/promiscuous/publisher/base.rb +4 -0
  13. data/lib/promiscuous/publisher/lint.rb +1 -1
  14. data/lib/promiscuous/publisher/model/active_record.rb +25 -0
  15. data/lib/promiscuous/publisher/model/mongoid.rb +108 -0
  16. data/lib/promiscuous/publisher/model.rb +71 -19
  17. data/lib/promiscuous/publisher/mongoid/embedded.rb +8 -4
  18. data/lib/promiscuous/publisher/mongoid.rb +12 -9
  19. data/lib/promiscuous/publisher.rb +1 -1
  20. data/lib/promiscuous/redis.rb +52 -0
  21. data/lib/promiscuous/subscriber/mongoid.rb +1 -2
  22. data/lib/promiscuous/subscriber/worker/message.rb +46 -0
  23. data/lib/promiscuous/subscriber/worker/message_synchronizer.rb +168 -0
  24. data/lib/promiscuous/subscriber/worker/pump.rb +47 -0
  25. data/lib/promiscuous/subscriber/worker/runner.rb +7 -0
  26. data/lib/promiscuous/subscriber/worker.rb +39 -47
  27. data/lib/promiscuous/version.rb +1 -1
  28. data/lib/promiscuous/worker.rb +1 -5
  29. data/lib/promiscuous.rb +1 -1
  30. metadata +86 -20
  31. data/lib/promiscuous/common/worker.rb +0 -52
  32. data/lib/promiscuous/publisher/mongoid/defer.rb +0 -75
  33. data/lib/promiscuous/publisher/mongoid/defer_embedded.rb +0 -25
  34. data/lib/promiscuous/publisher/worker.rb +0 -101
  35. data/lib/promiscuous/subscriber/mongoid/versioning.rb +0 -40
@@ -1,75 +0,0 @@
1
- module Promiscuous::Publisher::Mongoid::Defer
2
- extend ActiveSupport::Concern
3
- include Promiscuous::Publisher::Envelope
4
-
5
- mattr_accessor :klasses
6
- mattr_accessor :collections
7
- self.klasses = {}
8
- self.collections = {}
9
-
10
- def publish
11
- super unless should_defer?
12
- end
13
-
14
- def payload
15
- super.merge(:version => instance._psv)
16
- end
17
-
18
- def should_defer?
19
- if options.has_key?(:defer)
20
- options[:defer]
21
- else
22
- operation == :update
23
- end
24
- end
25
-
26
- def self.hook_mongoid
27
- return if @mongoid_hooked
28
- @mongoid_hooked = true
29
-
30
- Moped::Query.class_eval do
31
- alias_method :update_orig, :update
32
- def update(change, flags = nil)
33
- change = promiscuous_seasoning(change)
34
- update_orig(change, flags)
35
- end
36
-
37
- alias_method :modify_orig, :modify
38
- def modify(change, options={})
39
- change = promiscuous_seasoning(change) unless options[:bypass_promiscuous]
40
- modify_orig(change, options)
41
- end
42
-
43
- def promiscuous_seasoning(change)
44
- if Promiscuous::Publisher::Mongoid::Defer.collections[@collection.name]
45
- change = change.dup
46
- change['$set'] ||= {}
47
- change['$inc'] ||= {}
48
- change['$set'].merge!(:_psp => true)
49
- change['$inc'].merge!(:_psv => 1)
50
- end
51
- change
52
- end
53
- end
54
- end
55
-
56
- module ClassMethods
57
- def setup_class_binding
58
- super
59
- klass.class_eval do
60
- cattr_accessor :publisher_defer_hooked
61
- return if self.publisher_defer_hooked
62
- self.publisher_defer_hooked = true
63
-
64
- # TODO Make sure we are not overriding a field, although VERY unlikly
65
- field :_psp, :type => Boolean
66
- field :_psv, :type => Integer
67
- index({:_psp => 1}, :background => true, :sparse => true)
68
-
69
- Promiscuous::Publisher::Mongoid::Defer.hook_mongoid
70
- Promiscuous::Publisher::Mongoid::Defer.klasses[self.to_s] = self
71
- Promiscuous::Publisher::Mongoid::Defer.collections[collection.name] = true
72
- end if klass
73
- end
74
- end
75
- end
@@ -1,25 +0,0 @@
1
- module Promiscuous::Publisher::Mongoid::DeferEmbedded
2
- extend ActiveSupport::Concern
3
-
4
- def payload
5
- if instance.is_a?(Array)
6
- Promiscuous::Publisher::Mongoid::EmbeddedMany.new(:instance => instance).payload
7
- else
8
- super.merge(:id => instance.id)
9
- end
10
- end
11
-
12
- included do
13
- klass.class_eval do
14
- callback = proc do
15
- if _parent.respond_to?(:promiscuous_publish_update)
16
- _parent.promiscuous_publish_update
17
- end
18
- end
19
-
20
- before_create callback
21
- before_update callback
22
- before_destroy callback
23
- end
24
- end
25
- end
@@ -1,101 +0,0 @@
1
- class Promiscuous::Publisher::Worker
2
- include Promiscuous::Common::Worker
3
-
4
- def self.poll_delay
5
- # TODO Configurable globally
6
- # TODO Configurable per publisher
7
- 1.second
8
- end
9
-
10
- def initialize(options={})
11
- super
12
- check_indexes
13
- end
14
-
15
- def check_indexes
16
- Promiscuous::Publisher::Mongoid::Defer.klasses.values.each do |klass|
17
- unless klass.collection.indexes.any? { |i| i['key'].keys.include? '_psp' }
18
- raise 'Please run rake db:mongoid:create_indexes'
19
- end
20
- end
21
- end
22
-
23
- def resume
24
- @timer ||= EM::PeriodicTimer.new(self.class.poll_delay) { self.replicate_once }
25
- super
26
- end
27
-
28
- def stop
29
- @timer.try(:cancel)
30
- @timer = nil
31
- super
32
- end
33
-
34
- def replicate_once
35
- return if self.stopped?
36
-
37
- self.unit_of_work('publisher') do
38
- maybe_rescue_instance
39
- replicate_all_collections
40
- end
41
- rescue Exception => e
42
- unless e.is_a?(Promiscuous::Error::Publisher)
43
- e = Promiscuous::Error::Publisher.new(e)
44
- end
45
-
46
- retry_msg = stop_for_a_while(e)
47
- Promiscuous.warn "[publish] (#{retry_msg}) #{e} #{e.backtrace.join("\n")}"
48
-
49
- if e.out_of_sync
50
- Promiscuous.error "[publish] WARNING out of sync on #{e.instance.inspect}"
51
- @out_of_sync_instance = e.instance
52
- end
53
-
54
- Promiscuous::Config.error_notifier.try(:call, e)
55
- end
56
-
57
- def replicate_all_collections
58
- Promiscuous::Publisher::Mongoid::Defer.klasses.values.each do |klass|
59
- replicate_collection(klass)
60
- end
61
- made_progress
62
- end
63
-
64
- def replicate_collection(klass)
65
- loop do
66
- break if self.stopped?
67
-
68
- instance = klass.where(:_psp => true).
69
- find_and_modify({'$unset' => {:_psp => 1}}, :bypass_promiscuous => true)
70
- break unless instance
71
-
72
- replicate_instance(instance)
73
- end
74
- end
75
-
76
- def replicate_instance(instance)
77
- instance.promiscuous_sync
78
- made_progress
79
- rescue Exception => e
80
- out_of_sync = requeue_instance(instance)
81
- raise Promiscuous::Error::Publisher.new(e, :instance => instance,
82
- :out_of_sync => out_of_sync)
83
- end
84
-
85
- def requeue_instance(instance)
86
- # The following update will set the _psp flag to true again,
87
- # effectively requeuing the publish action.
88
- instance.class.where(instance.atomic_selector).update({})
89
- false
90
- rescue Exception
91
- # Swallow exception of a failed recovery.
92
- # The user needs to manually resync.
93
- true
94
- end
95
-
96
- def maybe_rescue_instance
97
- return unless @out_of_sync_instance
98
- replicate_instance(@out_of_sync_instance)
99
- @out_of_sync_instance = nil
100
- end
101
- end
@@ -1,40 +0,0 @@
1
- module Promiscuous::Subscriber::Mongoid::Versioning
2
- extend ActiveSupport::Concern
3
-
4
- module AtomicSelector
5
- extend ActiveSupport::Concern
6
-
7
- def atomic_selector
8
- if use_atomic_promiscuous_selector
9
- super.merge({ '$or' => [{'_psv' => { '$lte' => self._psv }},
10
- {'_psv' => { '$exists' => false }}]})
11
- else
12
- super
13
- end
14
- end
15
-
16
- included do
17
- attr_accessor :use_atomic_promiscuous_selector
18
- field :_psv, :type => Integer
19
- end
20
- end
21
-
22
- def save_instance
23
- if version
24
- instance._psv = version
25
- instance.use_atomic_promiscuous_selector = true
26
- end
27
- super
28
- ensure
29
- instance.use_atomic_promiscuous_selector = false
30
- end
31
-
32
- included { use_payload_attribute :version }
33
-
34
- module ClassMethods
35
- def setup_class_binding
36
- super
37
- klass.__send__(:include, AtomicSelector) if klass
38
- end
39
- end
40
- end