promiscuous 0.53.1 → 0.90.0
Sign up to get free protection for your applications and to get access to all the features.
- data/lib/promiscuous.rb +25 -28
- data/lib/promiscuous/amqp.rb +27 -8
- data/lib/promiscuous/amqp/bunny.rb +131 -16
- data/lib/promiscuous/amqp/fake.rb +52 -0
- data/lib/promiscuous/amqp/hot_bunnies.rb +56 -0
- data/lib/promiscuous/amqp/null.rb +6 -6
- data/lib/promiscuous/cli.rb +108 -24
- data/lib/promiscuous/config.rb +73 -12
- data/lib/promiscuous/convenience.rb +18 -0
- data/lib/promiscuous/dependency.rb +59 -0
- data/lib/promiscuous/dsl.rb +36 -0
- data/lib/promiscuous/error.rb +3 -1
- data/lib/promiscuous/error/already_processed.rb +5 -0
- data/lib/promiscuous/error/base.rb +1 -0
- data/lib/promiscuous/error/connection.rb +7 -5
- data/lib/promiscuous/error/dependency.rb +111 -0
- data/lib/promiscuous/error/lock_unavailable.rb +12 -0
- data/lib/promiscuous/error/lost_lock.rb +12 -0
- data/lib/promiscuous/error/missing_context.rb +29 -0
- data/lib/promiscuous/error/publisher.rb +5 -15
- data/lib/promiscuous/error/recovery.rb +7 -0
- data/lib/promiscuous/error/subscriber.rb +2 -4
- data/lib/promiscuous/key.rb +36 -0
- data/lib/promiscuous/loader.rb +12 -16
- data/lib/promiscuous/middleware.rb +112 -0
- data/lib/promiscuous/publisher.rb +7 -4
- data/lib/promiscuous/publisher/context.rb +92 -0
- data/lib/promiscuous/publisher/mock_generator.rb +72 -0
- data/lib/promiscuous/publisher/model.rb +3 -86
- data/lib/promiscuous/publisher/model/active_record.rb +8 -15
- data/lib/promiscuous/publisher/model/base.rb +136 -0
- data/lib/promiscuous/publisher/model/ephemeral.rb +69 -0
- data/lib/promiscuous/publisher/model/mock.rb +61 -0
- data/lib/promiscuous/publisher/model/mongoid.rb +57 -100
- data/lib/promiscuous/{common/lint.rb → publisher/operation.rb} +1 -1
- data/lib/promiscuous/publisher/operation/base.rb +707 -0
- data/lib/promiscuous/publisher/operation/mongoid.rb +370 -0
- data/lib/promiscuous/publisher/worker.rb +22 -0
- data/lib/promiscuous/railtie.rb +21 -3
- data/lib/promiscuous/redis.rb +132 -40
- data/lib/promiscuous/resque.rb +12 -0
- data/lib/promiscuous/sidekiq.rb +15 -0
- data/lib/promiscuous/subscriber.rb +9 -20
- data/lib/promiscuous/subscriber/model.rb +4 -104
- data/lib/promiscuous/subscriber/model/active_record.rb +10 -0
- data/lib/promiscuous/subscriber/model/base.rb +96 -0
- data/lib/promiscuous/subscriber/model/mongoid.rb +86 -0
- data/lib/promiscuous/subscriber/model/observer.rb +37 -0
- data/lib/promiscuous/subscriber/operation.rb +167 -0
- data/lib/promiscuous/subscriber/payload.rb +34 -0
- data/lib/promiscuous/subscriber/worker.rb +22 -18
- data/lib/promiscuous/subscriber/worker/message.rb +48 -25
- data/lib/promiscuous/subscriber/worker/message_synchronizer.rb +273 -181
- data/lib/promiscuous/subscriber/worker/pump.rb +17 -43
- data/lib/promiscuous/subscriber/worker/recorder.rb +24 -0
- data/lib/promiscuous/subscriber/worker/runner.rb +24 -3
- data/lib/promiscuous/subscriber/worker/stats.rb +62 -0
- data/lib/promiscuous/timer.rb +38 -0
- data/lib/promiscuous/version.rb +1 -1
- metadata +98 -143
- data/README.md +0 -33
- data/lib/promiscuous/amqp/ruby_amqp.rb +0 -140
- data/lib/promiscuous/common.rb +0 -4
- data/lib/promiscuous/common/class_helpers.rb +0 -12
- data/lib/promiscuous/common/lint/base.rb +0 -24
- data/lib/promiscuous/common/options.rb +0 -51
- data/lib/promiscuous/ephemeral.rb +0 -14
- data/lib/promiscuous/error/recover.rb +0 -1
- data/lib/promiscuous/observer.rb +0 -5
- data/lib/promiscuous/publisher/active_record.rb +0 -7
- data/lib/promiscuous/publisher/amqp.rb +0 -18
- data/lib/promiscuous/publisher/attributes.rb +0 -32
- data/lib/promiscuous/publisher/base.rb +0 -23
- data/lib/promiscuous/publisher/class.rb +0 -36
- data/lib/promiscuous/publisher/envelope.rb +0 -7
- data/lib/promiscuous/publisher/ephemeral.rb +0 -9
- data/lib/promiscuous/publisher/lint.rb +0 -35
- data/lib/promiscuous/publisher/lint/amqp.rb +0 -14
- data/lib/promiscuous/publisher/lint/attributes.rb +0 -12
- data/lib/promiscuous/publisher/lint/base.rb +0 -5
- data/lib/promiscuous/publisher/lint/class.rb +0 -15
- data/lib/promiscuous/publisher/lint/polymorphic.rb +0 -22
- data/lib/promiscuous/publisher/mock.rb +0 -79
- data/lib/promiscuous/publisher/mongoid.rb +0 -33
- data/lib/promiscuous/publisher/mongoid/embedded.rb +0 -27
- data/lib/promiscuous/publisher/mongoid/embedded_many.rb +0 -12
- data/lib/promiscuous/publisher/polymorphic.rb +0 -8
- data/lib/promiscuous/subscriber/active_record.rb +0 -11
- data/lib/promiscuous/subscriber/amqp.rb +0 -25
- data/lib/promiscuous/subscriber/attributes.rb +0 -35
- data/lib/promiscuous/subscriber/base.rb +0 -29
- data/lib/promiscuous/subscriber/class.rb +0 -29
- data/lib/promiscuous/subscriber/dummy.rb +0 -19
- data/lib/promiscuous/subscriber/envelope.rb +0 -18
- data/lib/promiscuous/subscriber/lint.rb +0 -30
- data/lib/promiscuous/subscriber/lint/amqp.rb +0 -21
- data/lib/promiscuous/subscriber/lint/attributes.rb +0 -21
- data/lib/promiscuous/subscriber/lint/base.rb +0 -14
- data/lib/promiscuous/subscriber/lint/class.rb +0 -13
- data/lib/promiscuous/subscriber/lint/polymorphic.rb +0 -39
- data/lib/promiscuous/subscriber/mongoid.rb +0 -27
- data/lib/promiscuous/subscriber/mongoid/embedded.rb +0 -17
- data/lib/promiscuous/subscriber/mongoid/embedded_many.rb +0 -44
- data/lib/promiscuous/subscriber/observer.rb +0 -26
- data/lib/promiscuous/subscriber/polymorphic.rb +0 -36
- data/lib/promiscuous/subscriber/upsert.rb +0 -12
@@ -0,0 +1,12 @@
|
|
1
|
+
require 'resque/job'
|
2
|
+
|
3
|
+
class Resque::Job
|
4
|
+
alias_method :perform_without_promiscuous, :perform
|
5
|
+
|
6
|
+
def perform
|
7
|
+
name = "resque/#{payload_class.name.underscore}"
|
8
|
+
Promiscuous::Middleware.with_context(name) do
|
9
|
+
perform_without_promiscuous
|
10
|
+
end
|
11
|
+
end
|
12
|
+
end
|
@@ -0,0 +1,15 @@
|
|
1
|
+
module Sidekiq
|
2
|
+
class Promiscuous
|
3
|
+
def call(worker_class, item, queue)
|
4
|
+
::Promiscuous::Middleware.with_context "sidekiq/#{item['queue']}/#{worker_class.class.to_s.underscore}" do
|
5
|
+
yield
|
6
|
+
end
|
7
|
+
end
|
8
|
+
end
|
9
|
+
end
|
10
|
+
|
11
|
+
Sidekiq.configure_server do |config|
|
12
|
+
config.server_middleware do |chain|
|
13
|
+
chain.add Sidekiq::Promiscuous
|
14
|
+
end
|
15
|
+
end
|
@@ -1,27 +1,16 @@
|
|
1
1
|
module Promiscuous::Subscriber
|
2
2
|
extend Promiscuous::Autoload
|
3
|
-
autoload :
|
4
|
-
:Lint, :Model, :Mongoid, :Polymorphic, :Upsert, :Observer, :Worker, :Dummy
|
3
|
+
autoload :Worker, :Payload, :Model, :Operation
|
5
4
|
|
6
|
-
|
7
|
-
Lint.lint(*args)
|
8
|
-
end
|
5
|
+
extend ActiveSupport::Concern
|
9
6
|
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
7
|
+
included do
|
8
|
+
if defined?(Mongoid::Document) && self < Mongoid::Document
|
9
|
+
include Promiscuous::Subscriber::Model::Mongoid
|
10
|
+
elsif defined?(ActiveRecord::Base) && self < ActiveRecord::Base
|
11
|
+
include Promiscuous::Subscriber::Model::ActiveRecord
|
12
|
+
else
|
13
|
+
raise "What kind of model is this? try including Promiscuous::Subscriber after all your includes"
|
14
14
|
end
|
15
|
-
sub || Base
|
16
|
-
end
|
17
|
-
|
18
|
-
def self.subscriber_for(payload, options={})
|
19
|
-
self.subscriber_class_for(payload).new(options.merge(:payload => payload))
|
20
|
-
end
|
21
|
-
|
22
|
-
def self.process(payload, options={})
|
23
|
-
sub = self.subscriber_for(payload, options)
|
24
|
-
sub.process
|
25
|
-
sub.instance
|
26
15
|
end
|
27
16
|
end
|
@@ -1,107 +1,7 @@
|
|
1
|
-
require 'crowdtap_redis_lock'
|
2
|
-
|
3
1
|
module Promiscuous::Subscriber::Model
|
4
|
-
extend
|
5
|
-
|
6
|
-
|
7
|
-
def fetch_new
|
8
|
-
if foreign_key
|
9
|
-
klass.new(foreign_key => id)
|
10
|
-
else
|
11
|
-
klass.new.tap { |o| o.id = id }
|
12
|
-
end
|
13
|
-
end
|
14
|
-
|
15
|
-
def fetch_existing
|
16
|
-
if foreign_key
|
17
|
-
if klass.respond_to?("find_by_#{foreign_key}!")
|
18
|
-
klass.__send__("find_by_#{foreign_key}!", id)
|
19
|
-
elsif klass.respond_to?("find_by")
|
20
|
-
klass.find_by(foreign_key => id)
|
21
|
-
else
|
22
|
-
record = klass.where(foreign_key => id).first
|
23
|
-
raise self.class.missing_record_exception.new(klass, id) if record.nil?
|
24
|
-
record
|
25
|
-
end
|
26
|
-
else
|
27
|
-
klass.find(id)
|
28
|
-
end
|
29
|
-
end
|
30
|
-
|
31
|
-
def fetch
|
32
|
-
case operation
|
33
|
-
when :create then fetch_new
|
34
|
-
when :update then fetch_existing
|
35
|
-
when :destroy then fetch_existing
|
36
|
-
when :dummy then fetch_new
|
37
|
-
end
|
38
|
-
end
|
39
|
-
|
40
|
-
def process_attributes?
|
41
|
-
!operation.in? [:destroy, :dummy]
|
42
|
-
end
|
43
|
-
|
44
|
-
def message
|
45
|
-
options[:message]
|
46
|
-
end
|
47
|
-
|
48
|
-
def with_lock(&block)
|
49
|
-
return yield if Promiscuous::Config.backend == :null
|
50
|
-
|
51
|
-
key = Promiscuous::Redis.sub_key(instance.id)
|
52
|
-
# We'll block for 60 seconds before raising an exception
|
53
|
-
::RedisLock.new(Promiscuous::Redis, key).retry(300).every(0.2).lock_for_update(&block)
|
54
|
-
end
|
55
|
-
|
56
|
-
def verify_dependencies
|
57
|
-
@global_key = Promiscuous::Redis.sub_key('global')
|
58
|
-
Promiscuous::Redis.get(@global_key).to_i + 1 == message.global_version
|
59
|
-
end
|
60
|
-
|
61
|
-
def update_dependencies
|
62
|
-
Promiscuous::Redis.set(@global_key, message.global_version)
|
63
|
-
@changed_global_key = true
|
64
|
-
end
|
65
|
-
|
66
|
-
def publish_dependencies
|
67
|
-
Promiscuous::Redis.publish(@global_key, message.global_version) if @changed_global_key
|
68
|
-
end
|
69
|
-
|
70
|
-
def with_dependencies
|
71
|
-
return yield unless message && message.has_dependencies?
|
72
|
-
|
73
|
-
with_lock do
|
74
|
-
if verify_dependencies
|
75
|
-
yield
|
76
|
-
update_dependencies
|
77
|
-
else
|
78
|
-
Promiscuous.info "[receive] (skipped, already processed) #{message.payload}"
|
79
|
-
end
|
80
|
-
end
|
81
|
-
|
82
|
-
publish_dependencies
|
83
|
-
end
|
84
|
-
|
85
|
-
def process
|
86
|
-
super
|
87
|
-
commit
|
88
|
-
end
|
89
|
-
|
90
|
-
def commit
|
91
|
-
with_dependencies do
|
92
|
-
case operation
|
93
|
-
when :create then instance.save!
|
94
|
-
when :update then instance.save!
|
95
|
-
when :destroy then instance.destroy
|
96
|
-
when :dummy then nil
|
97
|
-
end
|
98
|
-
end
|
99
|
-
end
|
100
|
-
|
101
|
-
included do
|
102
|
-
use_option :foreign_key
|
2
|
+
extend Promiscuous::Autoload
|
3
|
+
autoload :Base, :ActiveRecord, :Mongoid, :Observer
|
103
4
|
|
104
|
-
|
105
|
-
|
106
|
-
end
|
5
|
+
mattr_accessor :mapping
|
6
|
+
self.mapping = {}
|
107
7
|
end
|
@@ -0,0 +1,96 @@
|
|
1
|
+
module Promiscuous::Subscriber::Model::Base
|
2
|
+
extend ActiveSupport::Concern
|
3
|
+
|
4
|
+
def __promiscuous_update(payload, options={})
|
5
|
+
self.class.subscribed_attrs.map(&:to_s).each do |attr|
|
6
|
+
unless payload.attributes.has_key?(attr)
|
7
|
+
raise "Attribute '#{attr}' is missing from the payload"
|
8
|
+
end
|
9
|
+
|
10
|
+
value = payload.attributes[attr]
|
11
|
+
update = true
|
12
|
+
|
13
|
+
attr_payload = Promiscuous::Subscriber::Payload.new(value)
|
14
|
+
if model = attr_payload.model
|
15
|
+
# Nested subscriber
|
16
|
+
old_value = __send__(attr)
|
17
|
+
instance = old_value || model.__promiscuous_fetch_new(attr_payload.id)
|
18
|
+
|
19
|
+
if instance.class != model
|
20
|
+
# Because of the nasty trick with '__promiscuous__/embedded_many'
|
21
|
+
instance = model.__promiscuous_fetch_new(attr_payload.id)
|
22
|
+
end
|
23
|
+
|
24
|
+
nested_options = {:parent => self, :old_value => old_value}
|
25
|
+
update = instance.__promiscuous_update(attr_payload, nested_options)
|
26
|
+
value = instance
|
27
|
+
end
|
28
|
+
|
29
|
+
self.__send__("#{attr}=", value) if update
|
30
|
+
true
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
included do
|
35
|
+
class_attribute :promiscuous_root_class
|
36
|
+
class_attribute :subscribe_from, :subscribe_foreign_key, :subscribed_attrs
|
37
|
+
self.promiscuous_root_class = self
|
38
|
+
self.subscribe_foreign_key = :id
|
39
|
+
self.subscribed_attrs = []
|
40
|
+
end
|
41
|
+
|
42
|
+
module ClassMethods
|
43
|
+
def subscribe(*args)
|
44
|
+
options = args.extract_options!
|
45
|
+
attributes = args
|
46
|
+
|
47
|
+
# TODO reject invalid options
|
48
|
+
|
49
|
+
if attributes.present? && self.subscribe_from && options[:from] && self.subscribe_from != options[:from]
|
50
|
+
raise 'Subscribing from different publishers is not supported yet'
|
51
|
+
end
|
52
|
+
|
53
|
+
unless self.subscribe_from
|
54
|
+
self.subscribe_from = options[:from] || ".*/#{self.name.underscore}"
|
55
|
+
from_regexp = Regexp.new("^#{self.subscribe_from}$")
|
56
|
+
Promiscuous::Subscriber::Model.mapping[from_regexp] = self
|
57
|
+
end
|
58
|
+
|
59
|
+
self.subscribe_foreign_key = options[:foreign_key] if options[:foreign_key]
|
60
|
+
@subscribe_as = options[:as].to_s if options[:as]
|
61
|
+
|
62
|
+
([self] + descendants).each { |klass| klass.subscribed_attrs |= attributes }
|
63
|
+
end
|
64
|
+
|
65
|
+
def subscribe_as
|
66
|
+
@subscribe_as || name
|
67
|
+
end
|
68
|
+
|
69
|
+
def inherited(subclass)
|
70
|
+
super
|
71
|
+
subclass.subscribed_attrs = self.subscribed_attrs.dup
|
72
|
+
end
|
73
|
+
|
74
|
+
class None; end
|
75
|
+
def __promiscuous_missing_record_exception
|
76
|
+
None
|
77
|
+
end
|
78
|
+
|
79
|
+
def __promiscuous_fetch_new(id)
|
80
|
+
new.tap { |m| m.__send__("#{subscribe_foreign_key}=", id) }
|
81
|
+
end
|
82
|
+
|
83
|
+
def __promiscuous_fetch_existing(id)
|
84
|
+
key = subscribe_foreign_key
|
85
|
+
if promiscuous_root_class.respond_to?("find_by_#{key}!")
|
86
|
+
promiscuous_root_class.__send__("find_by_#{key}!", id)
|
87
|
+
elsif respond_to?("find_by")
|
88
|
+
promiscuous_root_class.find_by(key => id)
|
89
|
+
else
|
90
|
+
instance = promiscuous_root_class.where(key => id).first
|
91
|
+
raise __promiscuous_missing_record_exception.new(promiscuous_root_class, id) if instance.nil?
|
92
|
+
instance
|
93
|
+
end
|
94
|
+
end
|
95
|
+
end
|
96
|
+
end
|
@@ -0,0 +1,86 @@
|
|
1
|
+
module Promiscuous::Subscriber::Model::Mongoid
|
2
|
+
extend ActiveSupport::Concern
|
3
|
+
include Promiscuous::Subscriber::Model::Base
|
4
|
+
|
5
|
+
def __promiscuous_update(payload, options={})
|
6
|
+
super
|
7
|
+
# The return value tells if the parent should assign the attribute
|
8
|
+
!self.embedded? || options[:old_value] != self
|
9
|
+
end
|
10
|
+
|
11
|
+
module ClassMethods
|
12
|
+
def subscribe(*args, &block)
|
13
|
+
super
|
14
|
+
return unless block
|
15
|
+
|
16
|
+
begin
|
17
|
+
@in_subscribe_block = true
|
18
|
+
block.call
|
19
|
+
ensure
|
20
|
+
@in_subscribe_block = false
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
def self.subscribe_on(method, options={})
|
25
|
+
define_method(method) do |name, *args, &block|
|
26
|
+
super(name, *args, &block)
|
27
|
+
if @in_subscribe_block
|
28
|
+
name = args.last[:as] if args.last.is_a?(Hash) && args.last[:as]
|
29
|
+
subscribe(name)
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
subscribe_on :field
|
35
|
+
subscribe_on :embeds_one
|
36
|
+
subscribe_on :embeds_many
|
37
|
+
|
38
|
+
def __promiscuous_missing_record_exception
|
39
|
+
Mongoid::Errors::DocumentNotFound
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
class EmbeddedMany
|
44
|
+
include Promiscuous::Subscriber::Model::Base
|
45
|
+
|
46
|
+
subscribe :from => '__promiscuous__/embedded_many'
|
47
|
+
|
48
|
+
def __promiscuous_update(payload, options={})
|
49
|
+
old_embeddeds = options[:old_value]
|
50
|
+
new_embeddeds = payload.attributes
|
51
|
+
|
52
|
+
# XXX Reordering is not supported
|
53
|
+
|
54
|
+
# find all updatable docs
|
55
|
+
new_embeddeds.each do |new_e|
|
56
|
+
old_e = old_embeddeds.select { |e| e.id.to_s == new_e['id'] }.first
|
57
|
+
if old_e
|
58
|
+
new_e['existed'] = true
|
59
|
+
old_e.instance_variable_set(:@keep, true)
|
60
|
+
|
61
|
+
payload = Promiscuous::Subscriber::Payload.new(new_e)
|
62
|
+
old_e.__promiscuous_update(payload, :old_value => old_e)
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
# delete all the old ones
|
67
|
+
old_embeddeds.each do |old_e|
|
68
|
+
old_e.destroy unless old_e.instance_variable_get(:@keep)
|
69
|
+
end
|
70
|
+
|
71
|
+
# create all the new ones
|
72
|
+
new_embeddeds.reject { |new_e| new_e['existed'] }.each do |new_e|
|
73
|
+
payload = Promiscuous::Subscriber::Payload.new(new_e)
|
74
|
+
new_e_instance = payload.model. __promiscuous_fetch_new(payload.id)
|
75
|
+
new_e_instance.__promiscuous_update(payload)
|
76
|
+
options[:parent].__send__(old_embeddeds.metadata[:name]) << new_e_instance
|
77
|
+
end
|
78
|
+
|
79
|
+
false
|
80
|
+
end
|
81
|
+
|
82
|
+
def self.__promiscuous_fetch_new(id)
|
83
|
+
new
|
84
|
+
end
|
85
|
+
end
|
86
|
+
end
|
@@ -0,0 +1,37 @@
|
|
1
|
+
module Promiscuous::Subscriber::Model::Observer
|
2
|
+
extend ActiveSupport::Concern
|
3
|
+
include Promiscuous::Subscriber::Model::Base
|
4
|
+
|
5
|
+
included do
|
6
|
+
extend ActiveModel::Callbacks
|
7
|
+
attr_accessor :id
|
8
|
+
define_model_callbacks :create, :update, :destroy, :only => :after
|
9
|
+
end
|
10
|
+
|
11
|
+
def __promiscuous_update(payload, options={})
|
12
|
+
super
|
13
|
+
run_callbacks payload.operation
|
14
|
+
end
|
15
|
+
|
16
|
+
def destroy
|
17
|
+
run_callbacks :destroy
|
18
|
+
end
|
19
|
+
|
20
|
+
def save!
|
21
|
+
end
|
22
|
+
|
23
|
+
module ClassMethods
|
24
|
+
def subscribe(*args)
|
25
|
+
super
|
26
|
+
subscribed_attrs.each do |attr|
|
27
|
+
# TODO do not overwrite existing methods
|
28
|
+
attr_accessor attr
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
def __promiscuous_fetch_new(id)
|
33
|
+
new.tap { |o| o.id = id }
|
34
|
+
end
|
35
|
+
alias __promiscuous_fetch_existing __promiscuous_fetch_new
|
36
|
+
end
|
37
|
+
end
|
@@ -0,0 +1,167 @@
|
|
1
|
+
class Promiscuous::Subscriber::Operation
|
2
|
+
attr_accessor :payload
|
3
|
+
|
4
|
+
delegate :model, :id, :operation, :message, :to => :payload
|
5
|
+
|
6
|
+
def initialize(payload)
|
7
|
+
self.payload = payload
|
8
|
+
end
|
9
|
+
|
10
|
+
# XXX TODO Code is not tolerent to losing a lock.
|
11
|
+
|
12
|
+
def update_dependencies_single(node_with_deps)
|
13
|
+
master_node = node_with_deps[0]
|
14
|
+
deps = node_with_deps[1]
|
15
|
+
|
16
|
+
@@update_script ||= Promiscuous::Redis::Script.new <<-SCRIPT
|
17
|
+
for i, key in ipairs(KEYS) do
|
18
|
+
local v = redis.call('incr', key .. ':rw')
|
19
|
+
redis.call('publish', key .. ':rw', v)
|
20
|
+
end
|
21
|
+
SCRIPT
|
22
|
+
keys = deps.map { |dep| dep.key(:sub).to_s }
|
23
|
+
@@update_script.eval(master_node, :keys => keys)
|
24
|
+
end
|
25
|
+
|
26
|
+
def update_dependencies_multi(nodes_with_deps, options={})
|
27
|
+
# With multi nodes, we have to do a 2pc for the lock recovery mechanism:
|
28
|
+
# 1) We do the secondaries first, with a recovery payload.
|
29
|
+
# 2) Then we do the master.
|
30
|
+
# 3) Then we cleanup the secondaries.
|
31
|
+
# We use a recovery_key unique to the operation to avoid any trouble of
|
32
|
+
# touching another operation.
|
33
|
+
secondary_nodes_with_deps = nodes_with_deps[1..-1]
|
34
|
+
recovery_key = @instance_dep.key(:sub).join(@instance_dep.version).to_s
|
35
|
+
|
36
|
+
secondary_nodes_with_deps.each do |node, deps|
|
37
|
+
@@update_script_secondary ||= Promiscuous::Redis::Script.new <<-SCRIPT
|
38
|
+
local recovery_key = ARGV[1]
|
39
|
+
|
40
|
+
if redis.call('get', recovery_key) == 'done' then
|
41
|
+
return
|
42
|
+
end
|
43
|
+
|
44
|
+
for i, key in ipairs(KEYS) do
|
45
|
+
local v = redis.call('incr', key .. ':rw')
|
46
|
+
redis.call('publish', key .. ':rw', v)
|
47
|
+
end
|
48
|
+
|
49
|
+
redis.call('set', recovery_key, 'done')
|
50
|
+
SCRIPT
|
51
|
+
keys = deps.map { |dep| dep.key(:sub).to_s }
|
52
|
+
@@update_script_secondary.eval(node, :keys => keys, :argv => [recovery_key])
|
53
|
+
after_secondary_update_hook
|
54
|
+
end
|
55
|
+
|
56
|
+
update_dependencies_single(nodes_with_deps.first) unless options[:skip_master]
|
57
|
+
|
58
|
+
secondary_nodes_with_deps.each do |node, deps|
|
59
|
+
node.del(recovery_key)
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
def after_secondary_update_hook; end # for tests
|
64
|
+
|
65
|
+
def update_dependencies
|
66
|
+
nodes_with_deps.size == 1 ? update_dependencies_single(nodes_with_deps.first) :
|
67
|
+
update_dependencies_multi(nodes_with_deps)
|
68
|
+
end
|
69
|
+
|
70
|
+
def verify_dependencies
|
71
|
+
key = @instance_dep.key(:sub).join('rw').to_s
|
72
|
+
|
73
|
+
if @instance_dep.redis_node.get(key).to_i + 1 > @instance_dep.version
|
74
|
+
if nodes_with_deps.size != 1
|
75
|
+
update_dependencies_multi(nodes_with_deps, :skip_master => true)
|
76
|
+
end
|
77
|
+
|
78
|
+
raise Promiscuous::Error::AlreadyProcessed
|
79
|
+
end
|
80
|
+
end
|
81
|
+
|
82
|
+
def nodes_with_deps
|
83
|
+
@nodes_with_deps ||= dependencies.group_by(&:redis_node).to_a
|
84
|
+
end
|
85
|
+
|
86
|
+
def dependencies
|
87
|
+
@dependencies ||= message.dependencies[:write] + message.dependencies[:read]
|
88
|
+
end
|
89
|
+
|
90
|
+
LOCK_OPTIONS = { :timeout => 1.5.minute, # after 1.5 minute , we give up
|
91
|
+
:sleep => 0.1, # polling every 100ms.
|
92
|
+
:expire => 1.minute } # after one minute, we are considered dead
|
93
|
+
|
94
|
+
def with_instance_dependencies
|
95
|
+
return yield unless message && message.has_dependencies?
|
96
|
+
|
97
|
+
@instance_dep = message.dependencies[:write].first
|
98
|
+
lock_options = LOCK_OPTIONS.merge(:node => @instance_dep.redis_node)
|
99
|
+
mutex = Promiscuous::Redis::Mutex.new(@instance_dep.key(:sub).to_s, lock_options)
|
100
|
+
|
101
|
+
unless mutex.lock
|
102
|
+
raise Promiscuous::Error::LockUnavailable.new(mutex.key)
|
103
|
+
end
|
104
|
+
|
105
|
+
begin
|
106
|
+
verify_dependencies
|
107
|
+
result = yield
|
108
|
+
update_dependencies
|
109
|
+
result
|
110
|
+
ensure
|
111
|
+
unless mutex.unlock
|
112
|
+
# TODO Be safe in case we have a duplicate message and lost the lock on it
|
113
|
+
raise "The subscriber lost the lock during its operation. It means that someone else\n"+
|
114
|
+
"received a duplicate message, and we got screwed.\n"
|
115
|
+
end
|
116
|
+
end
|
117
|
+
end
|
118
|
+
|
119
|
+
def create
|
120
|
+
model.__promiscuous_fetch_new(id).tap do |instance|
|
121
|
+
instance.__promiscuous_update(payload)
|
122
|
+
instance.save!
|
123
|
+
end
|
124
|
+
rescue Exception => e
|
125
|
+
# TODO Abstract the duplicated index error message
|
126
|
+
if e.message =~ /E11000 duplicate key error index: .*\.\$_id_ +dup key/
|
127
|
+
Promiscuous.warn "[receive] ignoring already created record #{message.payload}"
|
128
|
+
else
|
129
|
+
raise e
|
130
|
+
end
|
131
|
+
end
|
132
|
+
|
133
|
+
def update
|
134
|
+
model.__promiscuous_fetch_existing(id).tap do |instance|
|
135
|
+
instance.__promiscuous_update(payload)
|
136
|
+
instance.save!
|
137
|
+
end
|
138
|
+
rescue model.__promiscuous_missing_record_exception
|
139
|
+
Promiscuous.warn "[receive] upserting #{message.payload}"
|
140
|
+
create
|
141
|
+
end
|
142
|
+
|
143
|
+
def destroy
|
144
|
+
model.__promiscuous_fetch_existing(id).tap do |instance|
|
145
|
+
instance.destroy
|
146
|
+
end
|
147
|
+
rescue model.__promiscuous_missing_record_exception
|
148
|
+
Promiscuous.warn "[receive] ignoring missing record #{message.payload}"
|
149
|
+
end
|
150
|
+
|
151
|
+
def operation
|
152
|
+
# We must process messages with versions to stay in sync even if we
|
153
|
+
# don't have a subscriber.
|
154
|
+
payload.model.nil? ? :dummy : payload.operation
|
155
|
+
end
|
156
|
+
|
157
|
+
def commit
|
158
|
+
with_instance_dependencies do
|
159
|
+
case operation
|
160
|
+
when :create then create
|
161
|
+
when :update then update
|
162
|
+
when :destroy then destroy
|
163
|
+
when :dummy then nil
|
164
|
+
end
|
165
|
+
end
|
166
|
+
end
|
167
|
+
end
|