promiscuous 0.53.1 → 0.90.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (106) hide show
  1. data/lib/promiscuous.rb +25 -28
  2. data/lib/promiscuous/amqp.rb +27 -8
  3. data/lib/promiscuous/amqp/bunny.rb +131 -16
  4. data/lib/promiscuous/amqp/fake.rb +52 -0
  5. data/lib/promiscuous/amqp/hot_bunnies.rb +56 -0
  6. data/lib/promiscuous/amqp/null.rb +6 -6
  7. data/lib/promiscuous/cli.rb +108 -24
  8. data/lib/promiscuous/config.rb +73 -12
  9. data/lib/promiscuous/convenience.rb +18 -0
  10. data/lib/promiscuous/dependency.rb +59 -0
  11. data/lib/promiscuous/dsl.rb +36 -0
  12. data/lib/promiscuous/error.rb +3 -1
  13. data/lib/promiscuous/error/already_processed.rb +5 -0
  14. data/lib/promiscuous/error/base.rb +1 -0
  15. data/lib/promiscuous/error/connection.rb +7 -5
  16. data/lib/promiscuous/error/dependency.rb +111 -0
  17. data/lib/promiscuous/error/lock_unavailable.rb +12 -0
  18. data/lib/promiscuous/error/lost_lock.rb +12 -0
  19. data/lib/promiscuous/error/missing_context.rb +29 -0
  20. data/lib/promiscuous/error/publisher.rb +5 -15
  21. data/lib/promiscuous/error/recovery.rb +7 -0
  22. data/lib/promiscuous/error/subscriber.rb +2 -4
  23. data/lib/promiscuous/key.rb +36 -0
  24. data/lib/promiscuous/loader.rb +12 -16
  25. data/lib/promiscuous/middleware.rb +112 -0
  26. data/lib/promiscuous/publisher.rb +7 -4
  27. data/lib/promiscuous/publisher/context.rb +92 -0
  28. data/lib/promiscuous/publisher/mock_generator.rb +72 -0
  29. data/lib/promiscuous/publisher/model.rb +3 -86
  30. data/lib/promiscuous/publisher/model/active_record.rb +8 -15
  31. data/lib/promiscuous/publisher/model/base.rb +136 -0
  32. data/lib/promiscuous/publisher/model/ephemeral.rb +69 -0
  33. data/lib/promiscuous/publisher/model/mock.rb +61 -0
  34. data/lib/promiscuous/publisher/model/mongoid.rb +57 -100
  35. data/lib/promiscuous/{common/lint.rb → publisher/operation.rb} +1 -1
  36. data/lib/promiscuous/publisher/operation/base.rb +707 -0
  37. data/lib/promiscuous/publisher/operation/mongoid.rb +370 -0
  38. data/lib/promiscuous/publisher/worker.rb +22 -0
  39. data/lib/promiscuous/railtie.rb +21 -3
  40. data/lib/promiscuous/redis.rb +132 -40
  41. data/lib/promiscuous/resque.rb +12 -0
  42. data/lib/promiscuous/sidekiq.rb +15 -0
  43. data/lib/promiscuous/subscriber.rb +9 -20
  44. data/lib/promiscuous/subscriber/model.rb +4 -104
  45. data/lib/promiscuous/subscriber/model/active_record.rb +10 -0
  46. data/lib/promiscuous/subscriber/model/base.rb +96 -0
  47. data/lib/promiscuous/subscriber/model/mongoid.rb +86 -0
  48. data/lib/promiscuous/subscriber/model/observer.rb +37 -0
  49. data/lib/promiscuous/subscriber/operation.rb +167 -0
  50. data/lib/promiscuous/subscriber/payload.rb +34 -0
  51. data/lib/promiscuous/subscriber/worker.rb +22 -18
  52. data/lib/promiscuous/subscriber/worker/message.rb +48 -25
  53. data/lib/promiscuous/subscriber/worker/message_synchronizer.rb +273 -181
  54. data/lib/promiscuous/subscriber/worker/pump.rb +17 -43
  55. data/lib/promiscuous/subscriber/worker/recorder.rb +24 -0
  56. data/lib/promiscuous/subscriber/worker/runner.rb +24 -3
  57. data/lib/promiscuous/subscriber/worker/stats.rb +62 -0
  58. data/lib/promiscuous/timer.rb +38 -0
  59. data/lib/promiscuous/version.rb +1 -1
  60. metadata +98 -143
  61. data/README.md +0 -33
  62. data/lib/promiscuous/amqp/ruby_amqp.rb +0 -140
  63. data/lib/promiscuous/common.rb +0 -4
  64. data/lib/promiscuous/common/class_helpers.rb +0 -12
  65. data/lib/promiscuous/common/lint/base.rb +0 -24
  66. data/lib/promiscuous/common/options.rb +0 -51
  67. data/lib/promiscuous/ephemeral.rb +0 -14
  68. data/lib/promiscuous/error/recover.rb +0 -1
  69. data/lib/promiscuous/observer.rb +0 -5
  70. data/lib/promiscuous/publisher/active_record.rb +0 -7
  71. data/lib/promiscuous/publisher/amqp.rb +0 -18
  72. data/lib/promiscuous/publisher/attributes.rb +0 -32
  73. data/lib/promiscuous/publisher/base.rb +0 -23
  74. data/lib/promiscuous/publisher/class.rb +0 -36
  75. data/lib/promiscuous/publisher/envelope.rb +0 -7
  76. data/lib/promiscuous/publisher/ephemeral.rb +0 -9
  77. data/lib/promiscuous/publisher/lint.rb +0 -35
  78. data/lib/promiscuous/publisher/lint/amqp.rb +0 -14
  79. data/lib/promiscuous/publisher/lint/attributes.rb +0 -12
  80. data/lib/promiscuous/publisher/lint/base.rb +0 -5
  81. data/lib/promiscuous/publisher/lint/class.rb +0 -15
  82. data/lib/promiscuous/publisher/lint/polymorphic.rb +0 -22
  83. data/lib/promiscuous/publisher/mock.rb +0 -79
  84. data/lib/promiscuous/publisher/mongoid.rb +0 -33
  85. data/lib/promiscuous/publisher/mongoid/embedded.rb +0 -27
  86. data/lib/promiscuous/publisher/mongoid/embedded_many.rb +0 -12
  87. data/lib/promiscuous/publisher/polymorphic.rb +0 -8
  88. data/lib/promiscuous/subscriber/active_record.rb +0 -11
  89. data/lib/promiscuous/subscriber/amqp.rb +0 -25
  90. data/lib/promiscuous/subscriber/attributes.rb +0 -35
  91. data/lib/promiscuous/subscriber/base.rb +0 -29
  92. data/lib/promiscuous/subscriber/class.rb +0 -29
  93. data/lib/promiscuous/subscriber/dummy.rb +0 -19
  94. data/lib/promiscuous/subscriber/envelope.rb +0 -18
  95. data/lib/promiscuous/subscriber/lint.rb +0 -30
  96. data/lib/promiscuous/subscriber/lint/amqp.rb +0 -21
  97. data/lib/promiscuous/subscriber/lint/attributes.rb +0 -21
  98. data/lib/promiscuous/subscriber/lint/base.rb +0 -14
  99. data/lib/promiscuous/subscriber/lint/class.rb +0 -13
  100. data/lib/promiscuous/subscriber/lint/polymorphic.rb +0 -39
  101. data/lib/promiscuous/subscriber/mongoid.rb +0 -27
  102. data/lib/promiscuous/subscriber/mongoid/embedded.rb +0 -17
  103. data/lib/promiscuous/subscriber/mongoid/embedded_many.rb +0 -44
  104. data/lib/promiscuous/subscriber/observer.rb +0 -26
  105. data/lib/promiscuous/subscriber/polymorphic.rb +0 -36
  106. data/lib/promiscuous/subscriber/upsert.rb +0 -12
@@ -0,0 +1,12 @@
1
+ require 'resque/job'
2
+
3
+ class Resque::Job
4
+ alias_method :perform_without_promiscuous, :perform
5
+
6
+ def perform
7
+ name = "resque/#{payload_class.name.underscore}"
8
+ Promiscuous::Middleware.with_context(name) do
9
+ perform_without_promiscuous
10
+ end
11
+ end
12
+ end
@@ -0,0 +1,15 @@
1
+ module Sidekiq
2
+ class Promiscuous
3
+ def call(worker_class, item, queue)
4
+ ::Promiscuous::Middleware.with_context "sidekiq/#{item['queue']}/#{worker_class.class.to_s.underscore}" do
5
+ yield
6
+ end
7
+ end
8
+ end
9
+ end
10
+
11
+ Sidekiq.configure_server do |config|
12
+ config.server_middleware do |chain|
13
+ chain.add Sidekiq::Promiscuous
14
+ end
15
+ end
@@ -1,27 +1,16 @@
1
1
  module Promiscuous::Subscriber
2
2
  extend Promiscuous::Autoload
3
- autoload :ActiveRecord, :AMQP, :Attributes, :Base, :Class, :Envelope, :Error,
4
- :Lint, :Model, :Mongoid, :Polymorphic, :Upsert, :Observer, :Worker, :Dummy
3
+ autoload :Worker, :Payload, :Model, :Operation
5
4
 
6
- def self.lint(*args)
7
- Lint.lint(*args)
8
- end
5
+ extend ActiveSupport::Concern
9
6
 
10
- def self.subscriber_class_for(payload)
11
- sub = AMQP.subscriber_from(payload)
12
- if sub && defined?(Polymorphic) && sub.include?(Polymorphic)
13
- sub = sub.polymorphic_subscriber_from(payload)
7
+ included do
8
+ if defined?(Mongoid::Document) && self < Mongoid::Document
9
+ include Promiscuous::Subscriber::Model::Mongoid
10
+ elsif defined?(ActiveRecord::Base) && self < ActiveRecord::Base
11
+ include Promiscuous::Subscriber::Model::ActiveRecord
12
+ else
13
+ raise "What kind of model is this? try including Promiscuous::Subscriber after all your includes"
14
14
  end
15
- sub || Base
16
- end
17
-
18
- def self.subscriber_for(payload, options={})
19
- self.subscriber_class_for(payload).new(options.merge(:payload => payload))
20
- end
21
-
22
- def self.process(payload, options={})
23
- sub = self.subscriber_for(payload, options)
24
- sub.process
25
- sub.instance
26
15
  end
27
16
  end
@@ -1,107 +1,7 @@
1
- require 'crowdtap_redis_lock'
2
-
3
1
  module Promiscuous::Subscriber::Model
4
- extend ActiveSupport::Concern
5
- include Promiscuous::Subscriber::Envelope
6
-
7
- def fetch_new
8
- if foreign_key
9
- klass.new(foreign_key => id)
10
- else
11
- klass.new.tap { |o| o.id = id }
12
- end
13
- end
14
-
15
- def fetch_existing
16
- if foreign_key
17
- if klass.respond_to?("find_by_#{foreign_key}!")
18
- klass.__send__("find_by_#{foreign_key}!", id)
19
- elsif klass.respond_to?("find_by")
20
- klass.find_by(foreign_key => id)
21
- else
22
- record = klass.where(foreign_key => id).first
23
- raise self.class.missing_record_exception.new(klass, id) if record.nil?
24
- record
25
- end
26
- else
27
- klass.find(id)
28
- end
29
- end
30
-
31
- def fetch
32
- case operation
33
- when :create then fetch_new
34
- when :update then fetch_existing
35
- when :destroy then fetch_existing
36
- when :dummy then fetch_new
37
- end
38
- end
39
-
40
- def process_attributes?
41
- !operation.in? [:destroy, :dummy]
42
- end
43
-
44
- def message
45
- options[:message]
46
- end
47
-
48
- def with_lock(&block)
49
- return yield if Promiscuous::Config.backend == :null
50
-
51
- key = Promiscuous::Redis.sub_key(instance.id)
52
- # We'll block for 60 seconds before raising an exception
53
- ::RedisLock.new(Promiscuous::Redis, key).retry(300).every(0.2).lock_for_update(&block)
54
- end
55
-
56
- def verify_dependencies
57
- @global_key = Promiscuous::Redis.sub_key('global')
58
- Promiscuous::Redis.get(@global_key).to_i + 1 == message.global_version
59
- end
60
-
61
- def update_dependencies
62
- Promiscuous::Redis.set(@global_key, message.global_version)
63
- @changed_global_key = true
64
- end
65
-
66
- def publish_dependencies
67
- Promiscuous::Redis.publish(@global_key, message.global_version) if @changed_global_key
68
- end
69
-
70
- def with_dependencies
71
- return yield unless message && message.has_dependencies?
72
-
73
- with_lock do
74
- if verify_dependencies
75
- yield
76
- update_dependencies
77
- else
78
- Promiscuous.info "[receive] (skipped, already processed) #{message.payload}"
79
- end
80
- end
81
-
82
- publish_dependencies
83
- end
84
-
85
- def process
86
- super
87
- commit
88
- end
89
-
90
- def commit
91
- with_dependencies do
92
- case operation
93
- when :create then instance.save!
94
- when :update then instance.save!
95
- when :destroy then instance.destroy
96
- when :dummy then nil
97
- end
98
- end
99
- end
100
-
101
- included do
102
- use_option :foreign_key
2
+ extend Promiscuous::Autoload
3
+ autoload :Base, :ActiveRecord, :Mongoid, :Observer
103
4
 
104
- use_payload_attribute :id
105
- use_payload_attribute :operation, :symbolize => true
106
- end
5
+ mattr_accessor :mapping
6
+ self.mapping = {}
107
7
  end
@@ -0,0 +1,10 @@
1
+ module Promiscuous::Subscriber::Model::ActiveRecord
2
+ extend ActiveSupport::Concern
3
+ include Promiscuous::Subscriber::Model::Base
4
+
5
+ module ClassMethods
6
+ def __promiscuous_missing_record_exception
7
+ ActiveRecord::RecordNotFound
8
+ end
9
+ end
10
+ end
@@ -0,0 +1,96 @@
1
+ module Promiscuous::Subscriber::Model::Base
2
+ extend ActiveSupport::Concern
3
+
4
+ def __promiscuous_update(payload, options={})
5
+ self.class.subscribed_attrs.map(&:to_s).each do |attr|
6
+ unless payload.attributes.has_key?(attr)
7
+ raise "Attribute '#{attr}' is missing from the payload"
8
+ end
9
+
10
+ value = payload.attributes[attr]
11
+ update = true
12
+
13
+ attr_payload = Promiscuous::Subscriber::Payload.new(value)
14
+ if model = attr_payload.model
15
+ # Nested subscriber
16
+ old_value = __send__(attr)
17
+ instance = old_value || model.__promiscuous_fetch_new(attr_payload.id)
18
+
19
+ if instance.class != model
20
+ # Because of the nasty trick with '__promiscuous__/embedded_many'
21
+ instance = model.__promiscuous_fetch_new(attr_payload.id)
22
+ end
23
+
24
+ nested_options = {:parent => self, :old_value => old_value}
25
+ update = instance.__promiscuous_update(attr_payload, nested_options)
26
+ value = instance
27
+ end
28
+
29
+ self.__send__("#{attr}=", value) if update
30
+ true
31
+ end
32
+ end
33
+
34
+ included do
35
+ class_attribute :promiscuous_root_class
36
+ class_attribute :subscribe_from, :subscribe_foreign_key, :subscribed_attrs
37
+ self.promiscuous_root_class = self
38
+ self.subscribe_foreign_key = :id
39
+ self.subscribed_attrs = []
40
+ end
41
+
42
+ module ClassMethods
43
+ def subscribe(*args)
44
+ options = args.extract_options!
45
+ attributes = args
46
+
47
+ # TODO reject invalid options
48
+
49
+ if attributes.present? && self.subscribe_from && options[:from] && self.subscribe_from != options[:from]
50
+ raise 'Subscribing from different publishers is not supported yet'
51
+ end
52
+
53
+ unless self.subscribe_from
54
+ self.subscribe_from = options[:from] || ".*/#{self.name.underscore}"
55
+ from_regexp = Regexp.new("^#{self.subscribe_from}$")
56
+ Promiscuous::Subscriber::Model.mapping[from_regexp] = self
57
+ end
58
+
59
+ self.subscribe_foreign_key = options[:foreign_key] if options[:foreign_key]
60
+ @subscribe_as = options[:as].to_s if options[:as]
61
+
62
+ ([self] + descendants).each { |klass| klass.subscribed_attrs |= attributes }
63
+ end
64
+
65
+ def subscribe_as
66
+ @subscribe_as || name
67
+ end
68
+
69
+ def inherited(subclass)
70
+ super
71
+ subclass.subscribed_attrs = self.subscribed_attrs.dup
72
+ end
73
+
74
+ class None; end
75
+ def __promiscuous_missing_record_exception
76
+ None
77
+ end
78
+
79
+ def __promiscuous_fetch_new(id)
80
+ new.tap { |m| m.__send__("#{subscribe_foreign_key}=", id) }
81
+ end
82
+
83
+ def __promiscuous_fetch_existing(id)
84
+ key = subscribe_foreign_key
85
+ if promiscuous_root_class.respond_to?("find_by_#{key}!")
86
+ promiscuous_root_class.__send__("find_by_#{key}!", id)
87
+ elsif respond_to?("find_by")
88
+ promiscuous_root_class.find_by(key => id)
89
+ else
90
+ instance = promiscuous_root_class.where(key => id).first
91
+ raise __promiscuous_missing_record_exception.new(promiscuous_root_class, id) if instance.nil?
92
+ instance
93
+ end
94
+ end
95
+ end
96
+ end
@@ -0,0 +1,86 @@
1
+ module Promiscuous::Subscriber::Model::Mongoid
2
+ extend ActiveSupport::Concern
3
+ include Promiscuous::Subscriber::Model::Base
4
+
5
+ def __promiscuous_update(payload, options={})
6
+ super
7
+ # The return value tells if the parent should assign the attribute
8
+ !self.embedded? || options[:old_value] != self
9
+ end
10
+
11
+ module ClassMethods
12
+ def subscribe(*args, &block)
13
+ super
14
+ return unless block
15
+
16
+ begin
17
+ @in_subscribe_block = true
18
+ block.call
19
+ ensure
20
+ @in_subscribe_block = false
21
+ end
22
+ end
23
+
24
+ def self.subscribe_on(method, options={})
25
+ define_method(method) do |name, *args, &block|
26
+ super(name, *args, &block)
27
+ if @in_subscribe_block
28
+ name = args.last[:as] if args.last.is_a?(Hash) && args.last[:as]
29
+ subscribe(name)
30
+ end
31
+ end
32
+ end
33
+
34
+ subscribe_on :field
35
+ subscribe_on :embeds_one
36
+ subscribe_on :embeds_many
37
+
38
+ def __promiscuous_missing_record_exception
39
+ Mongoid::Errors::DocumentNotFound
40
+ end
41
+ end
42
+
43
+ class EmbeddedMany
44
+ include Promiscuous::Subscriber::Model::Base
45
+
46
+ subscribe :from => '__promiscuous__/embedded_many'
47
+
48
+ def __promiscuous_update(payload, options={})
49
+ old_embeddeds = options[:old_value]
50
+ new_embeddeds = payload.attributes
51
+
52
+ # XXX Reordering is not supported
53
+
54
+ # find all updatable docs
55
+ new_embeddeds.each do |new_e|
56
+ old_e = old_embeddeds.select { |e| e.id.to_s == new_e['id'] }.first
57
+ if old_e
58
+ new_e['existed'] = true
59
+ old_e.instance_variable_set(:@keep, true)
60
+
61
+ payload = Promiscuous::Subscriber::Payload.new(new_e)
62
+ old_e.__promiscuous_update(payload, :old_value => old_e)
63
+ end
64
+ end
65
+
66
+ # delete all the old ones
67
+ old_embeddeds.each do |old_e|
68
+ old_e.destroy unless old_e.instance_variable_get(:@keep)
69
+ end
70
+
71
+ # create all the new ones
72
+ new_embeddeds.reject { |new_e| new_e['existed'] }.each do |new_e|
73
+ payload = Promiscuous::Subscriber::Payload.new(new_e)
74
+ new_e_instance = payload.model. __promiscuous_fetch_new(payload.id)
75
+ new_e_instance.__promiscuous_update(payload)
76
+ options[:parent].__send__(old_embeddeds.metadata[:name]) << new_e_instance
77
+ end
78
+
79
+ false
80
+ end
81
+
82
+ def self.__promiscuous_fetch_new(id)
83
+ new
84
+ end
85
+ end
86
+ end
@@ -0,0 +1,37 @@
1
+ module Promiscuous::Subscriber::Model::Observer
2
+ extend ActiveSupport::Concern
3
+ include Promiscuous::Subscriber::Model::Base
4
+
5
+ included do
6
+ extend ActiveModel::Callbacks
7
+ attr_accessor :id
8
+ define_model_callbacks :create, :update, :destroy, :only => :after
9
+ end
10
+
11
+ def __promiscuous_update(payload, options={})
12
+ super
13
+ run_callbacks payload.operation
14
+ end
15
+
16
+ def destroy
17
+ run_callbacks :destroy
18
+ end
19
+
20
+ def save!
21
+ end
22
+
23
+ module ClassMethods
24
+ def subscribe(*args)
25
+ super
26
+ subscribed_attrs.each do |attr|
27
+ # TODO do not overwrite existing methods
28
+ attr_accessor attr
29
+ end
30
+ end
31
+
32
+ def __promiscuous_fetch_new(id)
33
+ new.tap { |o| o.id = id }
34
+ end
35
+ alias __promiscuous_fetch_existing __promiscuous_fetch_new
36
+ end
37
+ end
@@ -0,0 +1,167 @@
1
+ class Promiscuous::Subscriber::Operation
2
+ attr_accessor :payload
3
+
4
+ delegate :model, :id, :operation, :message, :to => :payload
5
+
6
+ def initialize(payload)
7
+ self.payload = payload
8
+ end
9
+
10
+ # XXX TODO Code is not tolerent to losing a lock.
11
+
12
+ def update_dependencies_single(node_with_deps)
13
+ master_node = node_with_deps[0]
14
+ deps = node_with_deps[1]
15
+
16
+ @@update_script ||= Promiscuous::Redis::Script.new <<-SCRIPT
17
+ for i, key in ipairs(KEYS) do
18
+ local v = redis.call('incr', key .. ':rw')
19
+ redis.call('publish', key .. ':rw', v)
20
+ end
21
+ SCRIPT
22
+ keys = deps.map { |dep| dep.key(:sub).to_s }
23
+ @@update_script.eval(master_node, :keys => keys)
24
+ end
25
+
26
+ def update_dependencies_multi(nodes_with_deps, options={})
27
+ # With multi nodes, we have to do a 2pc for the lock recovery mechanism:
28
+ # 1) We do the secondaries first, with a recovery payload.
29
+ # 2) Then we do the master.
30
+ # 3) Then we cleanup the secondaries.
31
+ # We use a recovery_key unique to the operation to avoid any trouble of
32
+ # touching another operation.
33
+ secondary_nodes_with_deps = nodes_with_deps[1..-1]
34
+ recovery_key = @instance_dep.key(:sub).join(@instance_dep.version).to_s
35
+
36
+ secondary_nodes_with_deps.each do |node, deps|
37
+ @@update_script_secondary ||= Promiscuous::Redis::Script.new <<-SCRIPT
38
+ local recovery_key = ARGV[1]
39
+
40
+ if redis.call('get', recovery_key) == 'done' then
41
+ return
42
+ end
43
+
44
+ for i, key in ipairs(KEYS) do
45
+ local v = redis.call('incr', key .. ':rw')
46
+ redis.call('publish', key .. ':rw', v)
47
+ end
48
+
49
+ redis.call('set', recovery_key, 'done')
50
+ SCRIPT
51
+ keys = deps.map { |dep| dep.key(:sub).to_s }
52
+ @@update_script_secondary.eval(node, :keys => keys, :argv => [recovery_key])
53
+ after_secondary_update_hook
54
+ end
55
+
56
+ update_dependencies_single(nodes_with_deps.first) unless options[:skip_master]
57
+
58
+ secondary_nodes_with_deps.each do |node, deps|
59
+ node.del(recovery_key)
60
+ end
61
+ end
62
+
63
+ def after_secondary_update_hook; end # for tests
64
+
65
+ def update_dependencies
66
+ nodes_with_deps.size == 1 ? update_dependencies_single(nodes_with_deps.first) :
67
+ update_dependencies_multi(nodes_with_deps)
68
+ end
69
+
70
+ def verify_dependencies
71
+ key = @instance_dep.key(:sub).join('rw').to_s
72
+
73
+ if @instance_dep.redis_node.get(key).to_i + 1 > @instance_dep.version
74
+ if nodes_with_deps.size != 1
75
+ update_dependencies_multi(nodes_with_deps, :skip_master => true)
76
+ end
77
+
78
+ raise Promiscuous::Error::AlreadyProcessed
79
+ end
80
+ end
81
+
82
+ def nodes_with_deps
83
+ @nodes_with_deps ||= dependencies.group_by(&:redis_node).to_a
84
+ end
85
+
86
+ def dependencies
87
+ @dependencies ||= message.dependencies[:write] + message.dependencies[:read]
88
+ end
89
+
90
+ LOCK_OPTIONS = { :timeout => 1.5.minute, # after 1.5 minute , we give up
91
+ :sleep => 0.1, # polling every 100ms.
92
+ :expire => 1.minute } # after one minute, we are considered dead
93
+
94
+ def with_instance_dependencies
95
+ return yield unless message && message.has_dependencies?
96
+
97
+ @instance_dep = message.dependencies[:write].first
98
+ lock_options = LOCK_OPTIONS.merge(:node => @instance_dep.redis_node)
99
+ mutex = Promiscuous::Redis::Mutex.new(@instance_dep.key(:sub).to_s, lock_options)
100
+
101
+ unless mutex.lock
102
+ raise Promiscuous::Error::LockUnavailable.new(mutex.key)
103
+ end
104
+
105
+ begin
106
+ verify_dependencies
107
+ result = yield
108
+ update_dependencies
109
+ result
110
+ ensure
111
+ unless mutex.unlock
112
+ # TODO Be safe in case we have a duplicate message and lost the lock on it
113
+ raise "The subscriber lost the lock during its operation. It means that someone else\n"+
114
+ "received a duplicate message, and we got screwed.\n"
115
+ end
116
+ end
117
+ end
118
+
119
+ def create
120
+ model.__promiscuous_fetch_new(id).tap do |instance|
121
+ instance.__promiscuous_update(payload)
122
+ instance.save!
123
+ end
124
+ rescue Exception => e
125
+ # TODO Abstract the duplicated index error message
126
+ if e.message =~ /E11000 duplicate key error index: .*\.\$_id_ +dup key/
127
+ Promiscuous.warn "[receive] ignoring already created record #{message.payload}"
128
+ else
129
+ raise e
130
+ end
131
+ end
132
+
133
+ def update
134
+ model.__promiscuous_fetch_existing(id).tap do |instance|
135
+ instance.__promiscuous_update(payload)
136
+ instance.save!
137
+ end
138
+ rescue model.__promiscuous_missing_record_exception
139
+ Promiscuous.warn "[receive] upserting #{message.payload}"
140
+ create
141
+ end
142
+
143
+ def destroy
144
+ model.__promiscuous_fetch_existing(id).tap do |instance|
145
+ instance.destroy
146
+ end
147
+ rescue model.__promiscuous_missing_record_exception
148
+ Promiscuous.warn "[receive] ignoring missing record #{message.payload}"
149
+ end
150
+
151
+ def operation
152
+ # We must process messages with versions to stay in sync even if we
153
+ # don't have a subscriber.
154
+ payload.model.nil? ? :dummy : payload.operation
155
+ end
156
+
157
+ def commit
158
+ with_instance_dependencies do
159
+ case operation
160
+ when :create then create
161
+ when :update then update
162
+ when :destroy then destroy
163
+ when :dummy then nil
164
+ end
165
+ end
166
+ end
167
+ end