promiscuous 0.53.1 → 0.90.0
Sign up to get free protection for your applications and to get access to all the features.
- data/lib/promiscuous.rb +25 -28
- data/lib/promiscuous/amqp.rb +27 -8
- data/lib/promiscuous/amqp/bunny.rb +131 -16
- data/lib/promiscuous/amqp/fake.rb +52 -0
- data/lib/promiscuous/amqp/hot_bunnies.rb +56 -0
- data/lib/promiscuous/amqp/null.rb +6 -6
- data/lib/promiscuous/cli.rb +108 -24
- data/lib/promiscuous/config.rb +73 -12
- data/lib/promiscuous/convenience.rb +18 -0
- data/lib/promiscuous/dependency.rb +59 -0
- data/lib/promiscuous/dsl.rb +36 -0
- data/lib/promiscuous/error.rb +3 -1
- data/lib/promiscuous/error/already_processed.rb +5 -0
- data/lib/promiscuous/error/base.rb +1 -0
- data/lib/promiscuous/error/connection.rb +7 -5
- data/lib/promiscuous/error/dependency.rb +111 -0
- data/lib/promiscuous/error/lock_unavailable.rb +12 -0
- data/lib/promiscuous/error/lost_lock.rb +12 -0
- data/lib/promiscuous/error/missing_context.rb +29 -0
- data/lib/promiscuous/error/publisher.rb +5 -15
- data/lib/promiscuous/error/recovery.rb +7 -0
- data/lib/promiscuous/error/subscriber.rb +2 -4
- data/lib/promiscuous/key.rb +36 -0
- data/lib/promiscuous/loader.rb +12 -16
- data/lib/promiscuous/middleware.rb +112 -0
- data/lib/promiscuous/publisher.rb +7 -4
- data/lib/promiscuous/publisher/context.rb +92 -0
- data/lib/promiscuous/publisher/mock_generator.rb +72 -0
- data/lib/promiscuous/publisher/model.rb +3 -86
- data/lib/promiscuous/publisher/model/active_record.rb +8 -15
- data/lib/promiscuous/publisher/model/base.rb +136 -0
- data/lib/promiscuous/publisher/model/ephemeral.rb +69 -0
- data/lib/promiscuous/publisher/model/mock.rb +61 -0
- data/lib/promiscuous/publisher/model/mongoid.rb +57 -100
- data/lib/promiscuous/{common/lint.rb → publisher/operation.rb} +1 -1
- data/lib/promiscuous/publisher/operation/base.rb +707 -0
- data/lib/promiscuous/publisher/operation/mongoid.rb +370 -0
- data/lib/promiscuous/publisher/worker.rb +22 -0
- data/lib/promiscuous/railtie.rb +21 -3
- data/lib/promiscuous/redis.rb +132 -40
- data/lib/promiscuous/resque.rb +12 -0
- data/lib/promiscuous/sidekiq.rb +15 -0
- data/lib/promiscuous/subscriber.rb +9 -20
- data/lib/promiscuous/subscriber/model.rb +4 -104
- data/lib/promiscuous/subscriber/model/active_record.rb +10 -0
- data/lib/promiscuous/subscriber/model/base.rb +96 -0
- data/lib/promiscuous/subscriber/model/mongoid.rb +86 -0
- data/lib/promiscuous/subscriber/model/observer.rb +37 -0
- data/lib/promiscuous/subscriber/operation.rb +167 -0
- data/lib/promiscuous/subscriber/payload.rb +34 -0
- data/lib/promiscuous/subscriber/worker.rb +22 -18
- data/lib/promiscuous/subscriber/worker/message.rb +48 -25
- data/lib/promiscuous/subscriber/worker/message_synchronizer.rb +273 -181
- data/lib/promiscuous/subscriber/worker/pump.rb +17 -43
- data/lib/promiscuous/subscriber/worker/recorder.rb +24 -0
- data/lib/promiscuous/subscriber/worker/runner.rb +24 -3
- data/lib/promiscuous/subscriber/worker/stats.rb +62 -0
- data/lib/promiscuous/timer.rb +38 -0
- data/lib/promiscuous/version.rb +1 -1
- metadata +98 -143
- data/README.md +0 -33
- data/lib/promiscuous/amqp/ruby_amqp.rb +0 -140
- data/lib/promiscuous/common.rb +0 -4
- data/lib/promiscuous/common/class_helpers.rb +0 -12
- data/lib/promiscuous/common/lint/base.rb +0 -24
- data/lib/promiscuous/common/options.rb +0 -51
- data/lib/promiscuous/ephemeral.rb +0 -14
- data/lib/promiscuous/error/recover.rb +0 -1
- data/lib/promiscuous/observer.rb +0 -5
- data/lib/promiscuous/publisher/active_record.rb +0 -7
- data/lib/promiscuous/publisher/amqp.rb +0 -18
- data/lib/promiscuous/publisher/attributes.rb +0 -32
- data/lib/promiscuous/publisher/base.rb +0 -23
- data/lib/promiscuous/publisher/class.rb +0 -36
- data/lib/promiscuous/publisher/envelope.rb +0 -7
- data/lib/promiscuous/publisher/ephemeral.rb +0 -9
- data/lib/promiscuous/publisher/lint.rb +0 -35
- data/lib/promiscuous/publisher/lint/amqp.rb +0 -14
- data/lib/promiscuous/publisher/lint/attributes.rb +0 -12
- data/lib/promiscuous/publisher/lint/base.rb +0 -5
- data/lib/promiscuous/publisher/lint/class.rb +0 -15
- data/lib/promiscuous/publisher/lint/polymorphic.rb +0 -22
- data/lib/promiscuous/publisher/mock.rb +0 -79
- data/lib/promiscuous/publisher/mongoid.rb +0 -33
- data/lib/promiscuous/publisher/mongoid/embedded.rb +0 -27
- data/lib/promiscuous/publisher/mongoid/embedded_many.rb +0 -12
- data/lib/promiscuous/publisher/polymorphic.rb +0 -8
- data/lib/promiscuous/subscriber/active_record.rb +0 -11
- data/lib/promiscuous/subscriber/amqp.rb +0 -25
- data/lib/promiscuous/subscriber/attributes.rb +0 -35
- data/lib/promiscuous/subscriber/base.rb +0 -29
- data/lib/promiscuous/subscriber/class.rb +0 -29
- data/lib/promiscuous/subscriber/dummy.rb +0 -19
- data/lib/promiscuous/subscriber/envelope.rb +0 -18
- data/lib/promiscuous/subscriber/lint.rb +0 -30
- data/lib/promiscuous/subscriber/lint/amqp.rb +0 -21
- data/lib/promiscuous/subscriber/lint/attributes.rb +0 -21
- data/lib/promiscuous/subscriber/lint/base.rb +0 -14
- data/lib/promiscuous/subscriber/lint/class.rb +0 -13
- data/lib/promiscuous/subscriber/lint/polymorphic.rb +0 -39
- data/lib/promiscuous/subscriber/mongoid.rb +0 -27
- data/lib/promiscuous/subscriber/mongoid/embedded.rb +0 -17
- data/lib/promiscuous/subscriber/mongoid/embedded_many.rb +0 -44
- data/lib/promiscuous/subscriber/observer.rb +0 -26
- data/lib/promiscuous/subscriber/polymorphic.rb +0 -36
- data/lib/promiscuous/subscriber/upsert.rb +0 -12
data/README.md
DELETED
@@ -1,33 +0,0 @@
|
|
1
|
-
Promiscuous
|
2
|
-
===========
|
3
|
-
|
4
|
-
[![Build Status](https://travis-ci.org/crowdtap/promiscuous.png?branch=travis)](https://travis-ci.org/crowdtap/promiscuous)
|
5
|
-
|
6
|
-
Promiscuous is designed to facilitate designing a
|
7
|
-
[service-oriented architecture](http://en.wikipedia.org/wiki/Service-oriented_architecture)
|
8
|
-
in Ruby.
|
9
|
-
|
10
|
-
Promiscuous offers an automatic way of propagating your data across one or more
|
11
|
-
applications. It supports Mongoid2, Mongoid3 and ActiveRecord.
|
12
|
-
It relies on [RabbitMQ](http://www.rabbitmq.com/) to push data around.
|
13
|
-
|
14
|
-
Philosophy
|
15
|
-
----------
|
16
|
-
|
17
|
-
In order for a service-oriented system to be successful, services *must* be
|
18
|
-
loosely coupled. The traditional Ruby way of tackling this problem is to
|
19
|
-
provide RESTful APIs.
|
20
|
-
Sadly, this come to a cost since one must write controllers, integration tests, etc.
|
21
|
-
Promiscuous to the rescue
|
22
|
-
|
23
|
-
Compatibility
|
24
|
-
-------------
|
25
|
-
|
26
|
-
Promiscuous is tested against MRI 1.9.3.
|
27
|
-
|
28
|
-
ActiveRecord, Mongoid 2.4.x and Mongoid 3.0.x are supported.
|
29
|
-
|
30
|
-
License
|
31
|
-
-------
|
32
|
-
|
33
|
-
Promiscuous is distributed under the MIT license.
|
@@ -1,140 +0,0 @@
|
|
1
|
-
require 'eventmachine'
|
2
|
-
require 'amqp'
|
3
|
-
|
4
|
-
module Promiscuous::AMQP::RubyAMQP
|
5
|
-
class Synchronizer
|
6
|
-
def initialize
|
7
|
-
@mutex = Mutex.new
|
8
|
-
@condition = ConditionVariable.new
|
9
|
-
@signaled = false
|
10
|
-
end
|
11
|
-
|
12
|
-
def wait
|
13
|
-
@mutex.synchronize do
|
14
|
-
loop do
|
15
|
-
return if @signaled
|
16
|
-
@condition.wait(@mutex)
|
17
|
-
end
|
18
|
-
end
|
19
|
-
end
|
20
|
-
|
21
|
-
def signal
|
22
|
-
@mutex.synchronize do
|
23
|
-
@signaled = true
|
24
|
-
@condition.signal
|
25
|
-
end
|
26
|
-
end
|
27
|
-
end
|
28
|
-
|
29
|
-
def self.maybe_start_event_machine
|
30
|
-
return if EM.reactor_running?
|
31
|
-
|
32
|
-
EM.error_handler { |e| Promiscuous::Config.error_notifier.try(:call, e) }
|
33
|
-
em_sync = Synchronizer.new
|
34
|
-
@event_machine_thread = Thread.new { EM.run { em_sync.signal } }
|
35
|
-
em_sync.wait
|
36
|
-
end
|
37
|
-
|
38
|
-
def self.connect
|
39
|
-
return if @connection
|
40
|
-
|
41
|
-
@channels = {}
|
42
|
-
@exchanges = {}
|
43
|
-
|
44
|
-
maybe_start_event_machine
|
45
|
-
|
46
|
-
amqp_options = if Promiscuous::Config.amqp_url
|
47
|
-
url = URI.parse(Promiscuous::Config.amqp_url)
|
48
|
-
raise "Please use amqp://user:password@host:port/vhost" if url.scheme != 'amqp'
|
49
|
-
|
50
|
-
{
|
51
|
-
:host => url.host,
|
52
|
-
:port => url.port,
|
53
|
-
:scheme => url.scheme,
|
54
|
-
:user => url.user,
|
55
|
-
:pass => url.password,
|
56
|
-
:vhost => url.path.empty? ? "/" : url.path,
|
57
|
-
:heartbeat => Promiscuous::Config.heartbeat
|
58
|
-
}
|
59
|
-
end
|
60
|
-
|
61
|
-
channel_sync = Synchronizer.new
|
62
|
-
::AMQP.connect(amqp_options) do |connection|
|
63
|
-
@connection = connection
|
64
|
-
@connection.on_tcp_connection_loss do |conn|
|
65
|
-
unless conn.reconnecting?
|
66
|
-
e = Promiscuous::AMQP.lost_connection_exception
|
67
|
-
Promiscuous.warn "[amqp] #{e}. Reconnecting..."
|
68
|
-
Promiscuous::Config.error_notifier.try(:call, e)
|
69
|
-
conn.periodically_reconnect(2.seconds)
|
70
|
-
end
|
71
|
-
end
|
72
|
-
|
73
|
-
@connection.on_recovery do |conn|
|
74
|
-
Promiscuous.warn "[amqp] Reconnected"
|
75
|
-
@channels.values.each(&:recover) if conn == @connection
|
76
|
-
end
|
77
|
-
|
78
|
-
@connection.on_error do |conn, conn_close|
|
79
|
-
# No need to handle CONNECTION_FORCED since on_tcp_connection_loss takes
|
80
|
-
# care of it.
|
81
|
-
Promiscuous.warn "[amqp] #{conn_close.reply_text}"
|
82
|
-
end
|
83
|
-
|
84
|
-
get_channel(:master) { channel_sync.signal }
|
85
|
-
end
|
86
|
-
channel_sync.wait
|
87
|
-
rescue Exception => e
|
88
|
-
self.disconnect
|
89
|
-
raise e
|
90
|
-
end
|
91
|
-
|
92
|
-
def self.get_channel(name, &block)
|
93
|
-
if @channels[name]
|
94
|
-
yield(@channels[name]) if block_given?
|
95
|
-
@channels[name]
|
96
|
-
else
|
97
|
-
options = {:auto_recovery => true, :prefetch => Promiscuous::Config.prefetch}
|
98
|
-
::AMQP::Channel.new(@connection, options) do |channel|
|
99
|
-
@channels[name] = channel
|
100
|
-
get_exchange(name)
|
101
|
-
yield(channel) if block_given?
|
102
|
-
end
|
103
|
-
end
|
104
|
-
end
|
105
|
-
|
106
|
-
def self.close_channel(name, &block)
|
107
|
-
EM.next_tick do
|
108
|
-
channel = @channels.try(:delete, name)
|
109
|
-
if channel
|
110
|
-
channel.close(&block)
|
111
|
-
else
|
112
|
-
block.call if block
|
113
|
-
end
|
114
|
-
end
|
115
|
-
end
|
116
|
-
|
117
|
-
def self.disconnect
|
118
|
-
@connection.close { EM.stop if @event_machine_thread } if @connection
|
119
|
-
@event_machine_thread.join if @event_machine_thread
|
120
|
-
@event_machine_thread = nil
|
121
|
-
@connection = nil
|
122
|
-
@channels = nil
|
123
|
-
@exchanges = nil
|
124
|
-
end
|
125
|
-
|
126
|
-
def self.connected?
|
127
|
-
@connection.connected? if @connection
|
128
|
-
end
|
129
|
-
|
130
|
-
def self.publish(options={})
|
131
|
-
EM.next_tick do
|
132
|
-
get_exchange(:master).publish(options[:payload], :routing_key => options[:key], :persistent => true) do
|
133
|
-
end
|
134
|
-
end
|
135
|
-
end
|
136
|
-
|
137
|
-
def self.get_exchange(name)
|
138
|
-
@exchanges[name] ||= get_channel(name).topic(Promiscuous::AMQP::EXCHANGE, :durable => true)
|
139
|
-
end
|
140
|
-
end
|
data/lib/promiscuous/common.rb
DELETED
@@ -1,12 +0,0 @@
|
|
1
|
-
module Promiscuous::Common::ClassHelpers
|
2
|
-
extend ActiveSupport::Concern
|
3
|
-
|
4
|
-
module ClassMethods
|
5
|
-
def guess_class_name(separator)
|
6
|
-
return nil if name.nil?
|
7
|
-
class_name = name.split("::").reverse.take_while { |name| name != separator }.reverse.join('::')
|
8
|
-
class_name = $1 if class_name =~ /^(.+)#{separator.singularize}$/
|
9
|
-
class_name
|
10
|
-
end
|
11
|
-
end
|
12
|
-
end
|
@@ -1,24 +0,0 @@
|
|
1
|
-
module Promiscuous::Common::Lint::Base
|
2
|
-
extend ActiveSupport::Concern
|
3
|
-
|
4
|
-
included do
|
5
|
-
attr_accessor :options
|
6
|
-
end
|
7
|
-
|
8
|
-
def initialize(options)
|
9
|
-
self.options = options
|
10
|
-
end
|
11
|
-
|
12
|
-
def lint
|
13
|
-
end
|
14
|
-
|
15
|
-
module ClassMethods
|
16
|
-
def use_option(*attrs)
|
17
|
-
attrs.each do |attr|
|
18
|
-
define_method(attr) do
|
19
|
-
self.options[attr]
|
20
|
-
end
|
21
|
-
end
|
22
|
-
end
|
23
|
-
end
|
24
|
-
end
|
@@ -1,51 +0,0 @@
|
|
1
|
-
module Promiscuous::Common::Options
|
2
|
-
extend ActiveSupport::Concern
|
3
|
-
|
4
|
-
included do
|
5
|
-
class_attribute :raw_options, :options, :options_mappings,
|
6
|
-
:instance_reader => false,
|
7
|
-
:instance_writer => false
|
8
|
-
self.raw_options = {}
|
9
|
-
self.options = {}
|
10
|
-
self.options_mappings = {}
|
11
|
-
end
|
12
|
-
|
13
|
-
module ClassMethods
|
14
|
-
def inherited(subclass)
|
15
|
-
super
|
16
|
-
subclass.options = self.options.dup
|
17
|
-
end
|
18
|
-
|
19
|
-
def use_option(attr, options={})
|
20
|
-
instance_reader = options.fetch(:instance_reader, true)
|
21
|
-
attr_alias = options.fetch(:as, attr)
|
22
|
-
|
23
|
-
self.options_mappings[attr] = attr_alias
|
24
|
-
|
25
|
-
base = self.ancestors[self.ancestors.index(Promiscuous::Common::Options) - 1]
|
26
|
-
|
27
|
-
# We need to let all the modules overload these methods, which is
|
28
|
-
# why we are injecting at the base level.
|
29
|
-
base.singleton_class.class_eval do
|
30
|
-
define_method("#{attr_alias}") { self.options[attr] }
|
31
|
-
define_method("#{attr_alias}=") { |value| self.options[attr] = value }
|
32
|
-
end
|
33
|
-
|
34
|
-
if instance_reader
|
35
|
-
define_method("#{attr_alias}") { self.class.__send__("#{attr_alias}") }
|
36
|
-
end
|
37
|
-
|
38
|
-
self.__send__("#{attr_alias}=", raw_options[attr]) if raw_options[attr]
|
39
|
-
end
|
40
|
-
|
41
|
-
def load_options(options)
|
42
|
-
self.raw_options = self.raw_options.dup
|
43
|
-
self.raw_options.merge!(options)
|
44
|
-
|
45
|
-
options.each do |attr, value|
|
46
|
-
attr_alias = self.options_mappings[attr]
|
47
|
-
self.__send__("#{attr_alias}=", value) if attr_alias
|
48
|
-
end
|
49
|
-
end
|
50
|
-
end
|
51
|
-
end
|
@@ -1,14 +0,0 @@
|
|
1
|
-
class Promiscuous::Ephemeral
|
2
|
-
def initialize(attributes)
|
3
|
-
attributes.each { |k, v| self.__send__("#{k}=", v) }
|
4
|
-
end
|
5
|
-
|
6
|
-
def self.create(attributes)
|
7
|
-
new(attributes).tap { |m| m.save }
|
8
|
-
end
|
9
|
-
|
10
|
-
def save
|
11
|
-
self.class.promiscuous_publisher.new(:instance => self).publish
|
12
|
-
end
|
13
|
-
alias save! save
|
14
|
-
end
|
@@ -1 +0,0 @@
|
|
1
|
-
class Promiscuous::Error::Recover < RuntimeError; end
|
data/lib/promiscuous/observer.rb
DELETED
@@ -1,7 +0,0 @@
|
|
1
|
-
class Promiscuous::Publisher::ActiveRecord < Promiscuous::Publisher::Base
|
2
|
-
include Promiscuous::Publisher::Class
|
3
|
-
include Promiscuous::Publisher::Attributes
|
4
|
-
include Promiscuous::Publisher::AMQP
|
5
|
-
include Promiscuous::Publisher::Model
|
6
|
-
include Promiscuous::Publisher::Model::ActiveRecord
|
7
|
-
end
|
@@ -1,18 +0,0 @@
|
|
1
|
-
module Promiscuous::Publisher::AMQP
|
2
|
-
extend ActiveSupport::Concern
|
3
|
-
include Promiscuous::Publisher::Envelope
|
4
|
-
|
5
|
-
def publish
|
6
|
-
exchange_name = Promiscuous::AMQP::EXCHANGE
|
7
|
-
exchange_name += ".#{options[:personality]}" if options[:personality]
|
8
|
-
Promiscuous::AMQP.publish(:exchange_name => exchange_name, :key => to, :payload => payload.to_json)
|
9
|
-
rescue Exception => e
|
10
|
-
raise_out_of_sync(e, payload.to_json)
|
11
|
-
end
|
12
|
-
|
13
|
-
def payload
|
14
|
-
super.merge(:__amqp__ => to)
|
15
|
-
end
|
16
|
-
|
17
|
-
included { use_option :to }
|
18
|
-
end
|
@@ -1,32 +0,0 @@
|
|
1
|
-
module Promiscuous::Publisher::Attributes
|
2
|
-
extend ActiveSupport::Concern
|
3
|
-
|
4
|
-
def payload
|
5
|
-
return nil unless include_attributes?
|
6
|
-
|
7
|
-
Hash[attributes.map { |field| [field, payload_for(field)] }]
|
8
|
-
end
|
9
|
-
|
10
|
-
def payload_for(field)
|
11
|
-
value = instance.__send__(field)
|
12
|
-
if value.respond_to?(:promiscuous_publisher)
|
13
|
-
value.promiscuous_publisher.new(options.merge(:instance => value)).payload
|
14
|
-
elsif value.class.respond_to?(:promiscuous_publisher)
|
15
|
-
value.class.promiscuous_publisher.new(options.merge(:instance => value)).payload
|
16
|
-
else
|
17
|
-
value
|
18
|
-
end
|
19
|
-
end
|
20
|
-
|
21
|
-
def include_attributes?
|
22
|
-
true
|
23
|
-
end
|
24
|
-
|
25
|
-
included { use_option :attributes }
|
26
|
-
|
27
|
-
module ClassMethods
|
28
|
-
def attributes=(value)
|
29
|
-
super(superclass.attributes.to_a + value)
|
30
|
-
end
|
31
|
-
end
|
32
|
-
end
|
@@ -1,23 +0,0 @@
|
|
1
|
-
class Promiscuous::Publisher::Base
|
2
|
-
include Promiscuous::Common::Options
|
3
|
-
|
4
|
-
cattr_accessor :published
|
5
|
-
attr_accessor :options
|
6
|
-
|
7
|
-
def initialize(options)
|
8
|
-
self.options = options
|
9
|
-
end
|
10
|
-
|
11
|
-
def instance
|
12
|
-
options[:instance]
|
13
|
-
end
|
14
|
-
|
15
|
-
def self.publish(options)
|
16
|
-
load_options(options)
|
17
|
-
self.published = true
|
18
|
-
end
|
19
|
-
|
20
|
-
def raise_out_of_sync(exception, payload)
|
21
|
-
raise Promiscuous::Error::Publisher.new(exception, :instance => instance, :out_of_sync => true, :payload => payload)
|
22
|
-
end
|
23
|
-
end
|
@@ -1,36 +0,0 @@
|
|
1
|
-
module Promiscuous::Publisher::Class
|
2
|
-
extend ActiveSupport::Concern
|
3
|
-
include Promiscuous::Common::ClassHelpers
|
4
|
-
|
5
|
-
included { use_option :class, :as => :klass }
|
6
|
-
|
7
|
-
module ClassMethods
|
8
|
-
def setup_class_binding
|
9
|
-
publisher_class = self
|
10
|
-
klass.class_eval do
|
11
|
-
class_attribute :promiscuous_publisher
|
12
|
-
self.promiscuous_publisher = publisher_class
|
13
|
-
end if klass
|
14
|
-
end
|
15
|
-
|
16
|
-
def self.publish(options)
|
17
|
-
super
|
18
|
-
setup_class_binding
|
19
|
-
end
|
20
|
-
|
21
|
-
def inherited(subclass)
|
22
|
-
super
|
23
|
-
subclass.setup_class_binding unless options[:class]
|
24
|
-
end
|
25
|
-
|
26
|
-
def klass=(value)
|
27
|
-
super
|
28
|
-
setup_class_binding
|
29
|
-
end
|
30
|
-
|
31
|
-
def klass
|
32
|
-
return nil if name.nil?
|
33
|
-
"::#{super ? super : guess_class_name('Publishers')}".constantize
|
34
|
-
end
|
35
|
-
end
|
36
|
-
end
|