promiscuous 0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README.md +132 -0
- data/lib/promiscuous.rb +5 -0
- data/lib/promiscuous/amqp.rb +34 -0
- data/lib/promiscuous/amqp/bunny.rb +23 -0
- data/lib/promiscuous/amqp/fake.rb +27 -0
- data/lib/promiscuous/amqp/null.rb +20 -0
- data/lib/promiscuous/amqp/ruby-amqp.rb +53 -0
- data/lib/promiscuous/publisher.rb +3 -0
- data/lib/promiscuous/publisher/amqp.rb +16 -0
- data/lib/promiscuous/publisher/attributes.rb +28 -0
- data/lib/promiscuous/publisher/base.rb +22 -0
- data/lib/promiscuous/publisher/class_bind.rb +15 -0
- data/lib/promiscuous/publisher/envelope.rb +7 -0
- data/lib/promiscuous/publisher/generic.rb +14 -0
- data/lib/promiscuous/publisher/mongoid.rb +17 -0
- data/lib/promiscuous/publisher/mongoid/embedded.rb +27 -0
- data/lib/promiscuous/publisher/mongoid/root.rb +31 -0
- data/lib/promiscuous/publisher/polymorphic.rb +10 -0
- data/lib/promiscuous/railtie.rb +28 -0
- data/lib/promiscuous/railtie/replicate.rake +12 -0
- data/lib/promiscuous/subscriber.rb +14 -0
- data/lib/promiscuous/subscriber/amqp.rb +32 -0
- data/lib/promiscuous/subscriber/attributes.rb +30 -0
- data/lib/promiscuous/subscriber/base.rb +31 -0
- data/lib/promiscuous/subscriber/custom_class.rb +17 -0
- data/lib/promiscuous/subscriber/envelope.rb +18 -0
- data/lib/promiscuous/subscriber/error.rb +18 -0
- data/lib/promiscuous/subscriber/generic.rb +14 -0
- data/lib/promiscuous/subscriber/mongoid.rb +25 -0
- data/lib/promiscuous/subscriber/mongoid/embedded.rb +13 -0
- data/lib/promiscuous/subscriber/mongoid/root.rb +35 -0
- data/lib/promiscuous/subscriber/mongoid/upsert.rb +12 -0
- data/lib/promiscuous/subscriber/polymorphic.rb +13 -0
- data/lib/promiscuous/version.rb +3 -0
- data/lib/promiscuous/worker.rb +25 -0
- metadata +163 -0
data/README.md
ADDED
@@ -0,0 +1,132 @@
|
|
1
|
+
Promiscuous
|
2
|
+
===========
|
3
|
+
|
4
|
+
[](https://secure.travis-ci.org/crowdtap/promiscuous)
|
5
|
+
|
6
|
+
Promiscuous offers an automatic way of propagating your model data across one or
|
7
|
+
more applications.
|
8
|
+
It uses [RabbitMQ](http://www.rabbitmq.com/).
|
9
|
+
|
10
|
+
Usage
|
11
|
+
------
|
12
|
+
|
13
|
+
From a publisher side (app that owns the data), create a Publisher per model.
|
14
|
+
as shown below.
|
15
|
+
|
16
|
+
From your subscribers side (apps that receive updates from the publisher),
|
17
|
+
create a Subscriber per model, as shown below.
|
18
|
+
|
19
|
+
Example
|
20
|
+
--------
|
21
|
+
|
22
|
+
### In your publisher app
|
23
|
+
|
24
|
+
```ruby
|
25
|
+
# initializer
|
26
|
+
Promiscuous::AMQP.configure(:backend => :bunny, :app => 'crowdtap', :logger => Rails.logger,
|
27
|
+
:server_uri => 'amqp://user:password@host:port/vhost')
|
28
|
+
|
29
|
+
# publisher
|
30
|
+
class ModelPublisher < Promiscuous::Publisher::Mongoid
|
31
|
+
publish :to => 'crowdtap/model',
|
32
|
+
:class => Model,
|
33
|
+
:attributes => [:field_1, :field_2, :field_3]
|
34
|
+
end
|
35
|
+
```
|
36
|
+
|
37
|
+
### In your subscriber app
|
38
|
+
|
39
|
+
```ruby
|
40
|
+
# initializer
|
41
|
+
Promiscuous::AMQP.configure(:backend => :rubyamqp, :app => 'sniper', :logger => Rails.logger,
|
42
|
+
:server_uri => 'amqp://user:password@host:port/vhost',
|
43
|
+
:queue_options => {:durable => true, :arguments => {'x-ha-policy' => 'all'}},
|
44
|
+
:error_handler => some_proc)
|
45
|
+
|
46
|
+
# subscriber
|
47
|
+
class ModelSubscriber < Promiscuous::Subscriber::Mongoid
|
48
|
+
subscribe :from => 'crowdtap/model',
|
49
|
+
:class => Model,
|
50
|
+
:attributes => [:field_1, :field_2, :field_3]
|
51
|
+
end
|
52
|
+
```
|
53
|
+
|
54
|
+
### Starting the subscriber worker
|
55
|
+
|
56
|
+
rake promiscuous:run[./path/to/promiscuous_initializer.rb]
|
57
|
+
|
58
|
+
How does it work ?
|
59
|
+
------------------
|
60
|
+
|
61
|
+
1. On the publisher side, Promiscuous hooks into the after_create/update/destroy callbacks.
|
62
|
+
2. When a model changes, Promiscuous sends a message to RabbitMQ, to the
|
63
|
+
'promiscuous' [topic exchange](http://www.rabbitmq.com/tutorials/tutorial-five-python.html).
|
64
|
+
3. RabbitMQ routes the messages to each application through queues.
|
65
|
+
We use one queue per application (TODO explain why we need one queue).
|
66
|
+
4. Subscribers apps are running the promiscuous worker, listening on their own queues,
|
67
|
+
executing the create/update/destroy on their databases.
|
68
|
+
|
69
|
+
Note that we use a single exchange to preserve the ordering of data updates
|
70
|
+
across application so that subscribers always see a consistant state of the
|
71
|
+
system.
|
72
|
+
|
73
|
+
WARNING/TODO
|
74
|
+
------------
|
75
|
+
|
76
|
+
Promiscuous does **not** handle:
|
77
|
+
- Any of the atomic operatiors, such as inc, or add_to_set.
|
78
|
+
- Association magic. Example:
|
79
|
+
```ruby
|
80
|
+
# This will NOT replicate particiation_ids:
|
81
|
+
m = Member.first
|
82
|
+
m.particiations = [Participation.first]
|
83
|
+
m.save
|
84
|
+
|
85
|
+
# On the other hand, this will:
|
86
|
+
m = Member.first
|
87
|
+
m.particiation_ids = [Participation.first.ids]
|
88
|
+
m.save
|
89
|
+
```
|
90
|
+
|
91
|
+
Furthermore, it can be racy. Consider this scenario with two interleaving
|
92
|
+
requests A and B:
|
93
|
+
|
94
|
+
1. (A) Update mongo doc X.value = 1
|
95
|
+
2. (B) Update mongo doc X.value = 2
|
96
|
+
3. (B) Publish 'X.value = 2' to Rabbit
|
97
|
+
4. (A) Publish 'X.value = 1' to Rabbit
|
98
|
+
|
99
|
+
At the end of the scenario, on the publisher side, the document X has value
|
100
|
+
equal to 2, while on the subscriber side, the document has a value of 1. This
|
101
|
+
will likely not occur in most scenarios BUT BEWARE. We have plans to fix this
|
102
|
+
issue by using version numbers and mongo's amazing findandmodify.
|
103
|
+
|
104
|
+
What's up with bunny vs ruby-amqp ?
|
105
|
+
-----------------------------------
|
106
|
+
|
107
|
+
Our publisher app does not run an eventmachine loop, which is required for
|
108
|
+
ruby-amqp. Bunny on the other hand allows a non-eventmachine based application
|
109
|
+
to publish messages to rabbitmq.
|
110
|
+
|
111
|
+
How to run the tests
|
112
|
+
--------------------
|
113
|
+
|
114
|
+
rake appraisal:install
|
115
|
+
rake
|
116
|
+
|
117
|
+
Compatibility
|
118
|
+
-------------
|
119
|
+
|
120
|
+
Promiscuous is tested against MRI 1.9.2 and 1.9.3.
|
121
|
+
|
122
|
+
Both Mongoid 2.4.x and Mongoid 3.0.x are supported.
|
123
|
+
|
124
|
+
Acknowledgments
|
125
|
+
----------------
|
126
|
+
|
127
|
+
Inspired by [Service-Oriented Design with Ruby and Rails](http://www.amazon.com/Service-Oriented-Design-Addison-Wesley-Professional-Series/dp/0321659368)
|
128
|
+
|
129
|
+
License
|
130
|
+
-------
|
131
|
+
|
132
|
+
Promiscuous is distributed under the MIT license.
|
data/lib/promiscuous.rb
ADDED
@@ -0,0 +1,34 @@
|
|
1
|
+
require 'promiscuous/amqp/bunny'
|
2
|
+
require 'promiscuous/amqp/fake'
|
3
|
+
require 'promiscuous/amqp/ruby-amqp'
|
4
|
+
require 'promiscuous/amqp/null'
|
5
|
+
|
6
|
+
module Promiscuous
|
7
|
+
module AMQP
|
8
|
+
mattr_accessor :backend, :app, :logger, :error_handler
|
9
|
+
|
10
|
+
def self.configure(options={}, &block)
|
11
|
+
options.symbolize_keys!
|
12
|
+
|
13
|
+
self.backend = "Promiscuous::AMQP::#{options[:backend].to_s.camelize.gsub(/amqp/, 'AMQP')}".constantize
|
14
|
+
self.backend.configure(options, &block)
|
15
|
+
self.app = options[:app]
|
16
|
+
self.logger = options[:logger] || Logger.new(STDOUT).tap { |l| l.level = Logger::WARN }
|
17
|
+
self.error_handler = options[:error_handler]
|
18
|
+
self
|
19
|
+
end
|
20
|
+
|
21
|
+
class << self
|
22
|
+
[:info, :error, :warn, :fatal].each do |level|
|
23
|
+
define_method(level) do |msg|
|
24
|
+
self.logger.__send__(level, "[AMQP] #{msg}")
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
# TODO Evaluate the performance hit of method_missing
|
30
|
+
def self.method_missing(method, *args, &block)
|
31
|
+
self.backend.__send__(method, *args, &block)
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
module Promiscuous
|
2
|
+
module AMQP
|
3
|
+
module Bunny
|
4
|
+
mattr_accessor :connection
|
5
|
+
|
6
|
+
def self.configure(options)
|
7
|
+
require 'bunny'
|
8
|
+
self.connection = ::Bunny.new(options[:server_uri])
|
9
|
+
self.connection.start
|
10
|
+
end
|
11
|
+
|
12
|
+
def self.publish(msg)
|
13
|
+
AMQP.info "[publish] #{msg[:key]} -> #{msg[:payload]}"
|
14
|
+
exchange = connection.exchange('promiscuous', :type => :topic, :durable => true)
|
15
|
+
exchange.publish(msg[:payload], :key => msg[:key], :persistent => true)
|
16
|
+
end
|
17
|
+
|
18
|
+
def self.close
|
19
|
+
self.connection.stop
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
module Promiscuous
|
2
|
+
module AMQP
|
3
|
+
module Fake
|
4
|
+
mattr_accessor :messages, :subscribe_options
|
5
|
+
self.messages = []
|
6
|
+
|
7
|
+
def self.configure(options)
|
8
|
+
end
|
9
|
+
|
10
|
+
def self.publish(msg)
|
11
|
+
self.messages << msg
|
12
|
+
end
|
13
|
+
|
14
|
+
def self.subscribe(options={}, &block)
|
15
|
+
self.subscribe_options = options
|
16
|
+
end
|
17
|
+
|
18
|
+
def self.clear
|
19
|
+
self.messages.clear
|
20
|
+
self.subscribe_options = nil
|
21
|
+
end
|
22
|
+
|
23
|
+
def self.close
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
@@ -0,0 +1,20 @@
|
|
1
|
+
module Promiscuous
|
2
|
+
module AMQP
|
3
|
+
module Null
|
4
|
+
def self.configure(options)
|
5
|
+
end
|
6
|
+
|
7
|
+
def self.publish(msg)
|
8
|
+
end
|
9
|
+
|
10
|
+
def self.subscribe(options={}, &block)
|
11
|
+
end
|
12
|
+
|
13
|
+
def self.clear
|
14
|
+
end
|
15
|
+
|
16
|
+
def self.close
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
@@ -0,0 +1,53 @@
|
|
1
|
+
module Promiscuous
|
2
|
+
module AMQP
|
3
|
+
module RubyAMQP
|
4
|
+
mattr_accessor :channel, :queue_options
|
5
|
+
|
6
|
+
def self.configure(options)
|
7
|
+
require 'amqp'
|
8
|
+
connection = ::AMQP.connect(build_connection_options(options))
|
9
|
+
self.channel = ::AMQP::Channel.new(connection)
|
10
|
+
self.queue_options = options[:queue_options] || {}
|
11
|
+
end
|
12
|
+
|
13
|
+
def self.build_connection_options(options)
|
14
|
+
if options[:server_uri]
|
15
|
+
uri = URI.parse(options[:server_uri])
|
16
|
+
raise "Please use amqp://user:password@host:port/vhost" if uri.scheme != 'amqp'
|
17
|
+
|
18
|
+
{
|
19
|
+
:host => uri.host,
|
20
|
+
:port => uri.port,
|
21
|
+
:scheme => uri.scheme,
|
22
|
+
:user => uri.user,
|
23
|
+
:pass => uri.password,
|
24
|
+
:vhost => uri.path.empty? ? "/" : uri.path,
|
25
|
+
}
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
def self.subscribe(options={}, &block)
|
30
|
+
queue_name = options[:queue_name]
|
31
|
+
bindings = options[:bindings]
|
32
|
+
|
33
|
+
queue = self.channel.queue(queue_name, self.queue_options)
|
34
|
+
exchange = channel.topic('promiscuous', :durable => true)
|
35
|
+
bindings.each do |binding|
|
36
|
+
queue.bind(exchange, :routing_key => binding)
|
37
|
+
AMQP.info "[bind] #{queue_name} -> #{binding}"
|
38
|
+
end
|
39
|
+
queue.subscribe(:ack => true, &block)
|
40
|
+
end
|
41
|
+
|
42
|
+
def self.publish(msg)
|
43
|
+
AMQP.info "[publish] #{msg[:key]} -> #{msg[:payload]}"
|
44
|
+
exchange = channel.topic('promiscuous', :durable => true)
|
45
|
+
exchange.publish(msg[:payload], :routing_key => msg[:key], :persistent => true)
|
46
|
+
end
|
47
|
+
|
48
|
+
def self.close
|
49
|
+
channel.close
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
@@ -0,0 +1,16 @@
|
|
1
|
+
require 'promiscuous/publisher/envelope'
|
2
|
+
|
3
|
+
module Promiscuous::Publisher::AMQP
|
4
|
+
extend ActiveSupport::Concern
|
5
|
+
include Promiscuous::Publisher::Envelope
|
6
|
+
|
7
|
+
def amqp_publish
|
8
|
+
Promiscuous::AMQP.publish(:key => to, :payload => payload.to_json)
|
9
|
+
end
|
10
|
+
|
11
|
+
def payload
|
12
|
+
super.merge(:__amqp__ => to)
|
13
|
+
end
|
14
|
+
|
15
|
+
included { use_option :to }
|
16
|
+
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
module Promiscuous::Publisher::Attributes
|
2
|
+
extend ActiveSupport::Concern
|
3
|
+
|
4
|
+
def payload
|
5
|
+
return nil unless include_attributes?
|
6
|
+
|
7
|
+
Hash[attributes.map do |field|
|
8
|
+
optional = field.to_s[-1] == '?'
|
9
|
+
field = field.to_s[0...-1].to_sym if optional
|
10
|
+
[field, payload_for(field)] if !optional || instance.respond_to?(field)
|
11
|
+
end]
|
12
|
+
end
|
13
|
+
|
14
|
+
def payload_for(field)
|
15
|
+
value = instance.__send__(field)
|
16
|
+
if value.class.respond_to?(:promiscuous_publisher)
|
17
|
+
value.class.promiscuous_publisher.new(options.merge(:instance => value)).payload
|
18
|
+
else
|
19
|
+
value
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
def include_attributes?
|
24
|
+
true
|
25
|
+
end
|
26
|
+
|
27
|
+
included { use_option :attributes }
|
28
|
+
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
class Promiscuous::Publisher::Base
|
2
|
+
attr_accessor :options
|
3
|
+
class_attribute :options
|
4
|
+
|
5
|
+
def initialize(options)
|
6
|
+
self.options = options
|
7
|
+
end
|
8
|
+
|
9
|
+
def instance
|
10
|
+
options[:instance]
|
11
|
+
end
|
12
|
+
|
13
|
+
def self.publish(options)
|
14
|
+
self.options = options
|
15
|
+
end
|
16
|
+
|
17
|
+
def self.use_option(attr)
|
18
|
+
define_method(attr) do
|
19
|
+
self.class.options[attr]
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
@@ -0,0 +1,15 @@
|
|
1
|
+
module Promiscuous::Publisher::ClassBind
|
2
|
+
extend ActiveSupport::Concern
|
3
|
+
|
4
|
+
module ClassMethods
|
5
|
+
def publish(options)
|
6
|
+
super
|
7
|
+
|
8
|
+
publisher_class = self
|
9
|
+
options[:class].class_eval do
|
10
|
+
class_attribute :promiscuous_publisher
|
11
|
+
self.promiscuous_publisher = publisher_class
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
@@ -0,0 +1,14 @@
|
|
1
|
+
require 'promiscuous/publisher/class_bind'
|
2
|
+
require 'promiscuous/publisher/base'
|
3
|
+
require 'promiscuous/publisher/attributes'
|
4
|
+
require 'promiscuous/publisher/polymorphic'
|
5
|
+
require 'promiscuous/publisher/amqp'
|
6
|
+
require 'promiscuous/publisher/envelope'
|
7
|
+
|
8
|
+
class Promiscuous::Publisher::Generic < Promiscuous::Publisher::Base
|
9
|
+
include Promiscuous::Publisher::ClassBind
|
10
|
+
include Promiscuous::Publisher::Attributes
|
11
|
+
include Promiscuous::Publisher::Polymorphic
|
12
|
+
include Promiscuous::Publisher::AMQP
|
13
|
+
include Promiscuous::Publisher::Envelope
|
14
|
+
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
require 'promiscuous/publisher/generic'
|
2
|
+
|
3
|
+
class Promiscuous::Publisher::Mongoid < Promiscuous::Publisher::Generic
|
4
|
+
def self.publish(options)
|
5
|
+
return super if options[:mongoid_loaded]
|
6
|
+
|
7
|
+
if options[:class].embedded?
|
8
|
+
require 'promiscuous/publisher/mongoid/embedded'
|
9
|
+
include Promiscuous::Publisher::Mongoid::Embedded
|
10
|
+
else
|
11
|
+
require 'promiscuous/publisher/mongoid/root'
|
12
|
+
include Promiscuous::Publisher::Mongoid::Root
|
13
|
+
end
|
14
|
+
|
15
|
+
self.publish(options.merge(:mongoid_loaded => true))
|
16
|
+
end
|
17
|
+
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
module Promiscuous::Publisher::Mongoid::Embedded
|
2
|
+
extend ActiveSupport::Concern
|
3
|
+
|
4
|
+
def payload
|
5
|
+
super.merge(:id => instance.id)
|
6
|
+
end
|
7
|
+
|
8
|
+
module ClassMethods
|
9
|
+
def publish(options)
|
10
|
+
super
|
11
|
+
|
12
|
+
options[:class].class_eval do
|
13
|
+
callback = proc do
|
14
|
+
if _parent.respond_to?(:promiscuous_publish_update)
|
15
|
+
_parent.save
|
16
|
+
_parent.reload # mongoid is not that smart, so we need to reload here.
|
17
|
+
_parent.promiscuous_publish_update
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
after_create callback
|
22
|
+
after_update callback
|
23
|
+
after_destroy callback
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
@@ -0,0 +1,31 @@
|
|
1
|
+
module Promiscuous::Publisher::Mongoid::Root
|
2
|
+
extend ActiveSupport::Concern
|
3
|
+
|
4
|
+
def operation
|
5
|
+
options[:operation]
|
6
|
+
end
|
7
|
+
|
8
|
+
def payload
|
9
|
+
super.merge(:id => instance.id, :operation => operation)
|
10
|
+
end
|
11
|
+
|
12
|
+
def include_attributes?
|
13
|
+
operation != :destroy
|
14
|
+
end
|
15
|
+
|
16
|
+
module ClassMethods
|
17
|
+
def publish(options)
|
18
|
+
super
|
19
|
+
|
20
|
+
options[:class].class_eval do
|
21
|
+
[:create, :update, :destroy].each do |operation|
|
22
|
+
__send__("after_#{operation}", "promiscuous_publish_#{operation}".to_sym)
|
23
|
+
|
24
|
+
define_method "promiscuous_publish_#{operation}" do
|
25
|
+
self.class.promiscuous_publisher.new(:instance => self, :operation => operation).amqp_publish
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
module Promiscuous
|
2
|
+
class Railtie < Rails::Railtie
|
3
|
+
rake_tasks { load 'promiscuous/railtie/replicate.rake' }
|
4
|
+
|
5
|
+
initializer 'load promiscuous' do
|
6
|
+
# TODO clean that up
|
7
|
+
config.after_initialize do
|
8
|
+
Dir[Rails.root.join('app', 'publishers', '**_publisher.rb')].map do |file|
|
9
|
+
file.split('/')[-1].split('.')[0].camelize.constantize
|
10
|
+
end
|
11
|
+
ActionDispatch::Reloader.to_prepare do
|
12
|
+
Dir[Rails.root.join('app', 'publishers', '**_publisher.rb')].map do |file|
|
13
|
+
file.split('/')[-1].split('.')[0].camelize.constantize
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
Dir[Rails.root.join('app', 'subscribers', '**_subscriber.rb')].map do |file|
|
18
|
+
file.split('/')[-1].split('.')[0].camelize.constantize
|
19
|
+
end
|
20
|
+
ActionDispatch::Reloader.to_prepare do
|
21
|
+
Dir[Rails.root.join('app', 'subscribers', '**_subscriber.rb')].map do |file|
|
22
|
+
file.split('/')[-1].split('.')[0].camelize.constantize
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
@@ -0,0 +1,12 @@
|
|
1
|
+
namespace :replicable do
|
2
|
+
desc 'Run the subscribers worker'
|
3
|
+
task :run, [:initializer] => :environment do |t, args|
|
4
|
+
require 'replicable/worker'
|
5
|
+
require 'eventmachine'
|
6
|
+
require 'em-synchrony'
|
7
|
+
EM.synchrony do
|
8
|
+
load args.initializer
|
9
|
+
Replicable::Worker.run
|
10
|
+
end
|
11
|
+
end
|
12
|
+
end
|
@@ -0,0 +1,14 @@
|
|
1
|
+
module Promiscuous::Subscriber
|
2
|
+
require 'promiscuous/subscriber/error'
|
3
|
+
require 'promiscuous/subscriber/mongoid'
|
4
|
+
require 'promiscuous/subscriber/amqp'
|
5
|
+
|
6
|
+
def self.process(payload, options={})
|
7
|
+
subscriber = Promiscuous::Subscriber::AMQP.subscriber_for(payload)
|
8
|
+
return payload if subscriber.nil?
|
9
|
+
|
10
|
+
sub = subscriber.new(options.merge(:payload => payload))
|
11
|
+
sub.process if sub.respond_to?(:process)
|
12
|
+
sub.instance
|
13
|
+
end
|
14
|
+
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
require 'promiscuous/subscriber/envelope'
|
2
|
+
|
3
|
+
module Promiscuous::Subscriber::AMQP
|
4
|
+
extend ActiveSupport::Concern
|
5
|
+
|
6
|
+
mattr_accessor :subscribers
|
7
|
+
self.subscribers = {}
|
8
|
+
|
9
|
+
def self.subscriber_for(payload)
|
10
|
+
origin = payload.is_a?(Hash) ? payload['__amqp__'] : nil
|
11
|
+
if origin
|
12
|
+
unless subscribers.has_key?(origin)
|
13
|
+
raise "FATAL: Unknown binding: '#{origin}'"
|
14
|
+
end
|
15
|
+
subscribers[origin]
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
module ClassMethods
|
20
|
+
def subscribe(options)
|
21
|
+
super
|
22
|
+
|
23
|
+
subscribers = Promiscuous::Subscriber::AMQP.subscribers
|
24
|
+
from = options[:from]
|
25
|
+
|
26
|
+
if subscribers.has_key?(from)
|
27
|
+
raise "The subscriber '#{subscribers[from]}' already listen on '#{from}'"
|
28
|
+
end
|
29
|
+
subscribers[from] = self
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
@@ -0,0 +1,30 @@
|
|
1
|
+
module Promiscuous::Subscriber::Attributes
|
2
|
+
extend ActiveSupport::Concern
|
3
|
+
|
4
|
+
def process
|
5
|
+
super
|
6
|
+
return unless process_attributes?
|
7
|
+
|
8
|
+
attributes.each do |attr|
|
9
|
+
attr = attr.to_s
|
10
|
+
optional = attr[-1] == '?'
|
11
|
+
attr = attr[0...-1] if optional
|
12
|
+
setter = "#{attr}="
|
13
|
+
|
14
|
+
if payload.has_key?(attr)
|
15
|
+
value = payload[attr]
|
16
|
+
old_value = instance.__send__(attr)
|
17
|
+
new_value = Promiscuous::Subscriber.process(payload[attr], :old_value => old_value)
|
18
|
+
instance.__send__(setter, new_value) if old_value != new_value
|
19
|
+
else
|
20
|
+
raise "Unknown attribute '#{attr}'" unless optional
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
def process_attributes?
|
26
|
+
true
|
27
|
+
end
|
28
|
+
|
29
|
+
included { use_option :attributes }
|
30
|
+
end
|
@@ -0,0 +1,31 @@
|
|
1
|
+
class Promiscuous::Subscriber::Base
|
2
|
+
attr_accessor :options
|
3
|
+
class_attribute :options
|
4
|
+
|
5
|
+
def initialize(options)
|
6
|
+
self.options = options
|
7
|
+
end
|
8
|
+
|
9
|
+
def payload
|
10
|
+
options[:payload]
|
11
|
+
end
|
12
|
+
alias :instance :payload
|
13
|
+
|
14
|
+
def process
|
15
|
+
end
|
16
|
+
|
17
|
+
def subscribe_options
|
18
|
+
self.class.options
|
19
|
+
end
|
20
|
+
|
21
|
+
def self.subscribe(options)
|
22
|
+
self.options = options
|
23
|
+
end
|
24
|
+
|
25
|
+
def self.use_option(attr, options={})
|
26
|
+
as = options[:as].nil? ? attr : options[:as]
|
27
|
+
define_method(as) do
|
28
|
+
self.class.options[attr]
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
require 'promiscuous/subscriber/envelope'
|
2
|
+
|
3
|
+
module Promiscuous::Subscriber::CustomClass
|
4
|
+
extend ActiveSupport::Concern
|
5
|
+
|
6
|
+
def klass
|
7
|
+
unless subscribe_options[:class]
|
8
|
+
raise "I don't want to be rude or anything, "
|
9
|
+
"but have you defined the class to deserialize?"
|
10
|
+
end
|
11
|
+
subscribe_options[:class]
|
12
|
+
end
|
13
|
+
|
14
|
+
def instance
|
15
|
+
@instance ||= fetch
|
16
|
+
end
|
17
|
+
end
|
@@ -0,0 +1,18 @@
|
|
1
|
+
module Promiscuous::Subscriber::Envelope
|
2
|
+
extend ActiveSupport::Concern
|
3
|
+
|
4
|
+
module ClassMethods
|
5
|
+
def use_payload_attribute(attr, options={})
|
6
|
+
define_method(attr) do
|
7
|
+
value = payload_with_envelope[attr.to_s]
|
8
|
+
value = value.to_sym if options[:symbolize]
|
9
|
+
value
|
10
|
+
end
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
included do
|
15
|
+
alias_method :payload_with_envelope, :payload
|
16
|
+
use_payload_attribute :payload
|
17
|
+
end
|
18
|
+
end
|
@@ -0,0 +1,18 @@
|
|
1
|
+
class Promiscuous::Subscriber::Error < RuntimeError
|
2
|
+
attr_accessor :inner, :payload
|
3
|
+
|
4
|
+
def initialize(inner, payload)
|
5
|
+
super(inner)
|
6
|
+
set_backtrace(inner.backtrace)
|
7
|
+
self.inner = inner
|
8
|
+
self.payload = payload
|
9
|
+
end
|
10
|
+
|
11
|
+
def message
|
12
|
+
"#{inner.message} while processing #{payload}"
|
13
|
+
end
|
14
|
+
|
15
|
+
def to_s
|
16
|
+
message
|
17
|
+
end
|
18
|
+
end
|
@@ -0,0 +1,14 @@
|
|
1
|
+
require 'promiscuous/subscriber/custom_class'
|
2
|
+
require 'promiscuous/subscriber/base'
|
3
|
+
require 'promiscuous/subscriber/attributes'
|
4
|
+
require 'promiscuous/subscriber/polymorphic'
|
5
|
+
require 'promiscuous/subscriber/amqp'
|
6
|
+
require 'promiscuous/subscriber/envelope'
|
7
|
+
|
8
|
+
class Promiscuous::Subscriber::Generic < Promiscuous::Subscriber::Base
|
9
|
+
include Promiscuous::Subscriber::CustomClass
|
10
|
+
include Promiscuous::Subscriber::Attributes
|
11
|
+
include Promiscuous::Subscriber::Polymorphic
|
12
|
+
include Promiscuous::Subscriber::AMQP
|
13
|
+
include Promiscuous::Subscriber::Envelope
|
14
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
require 'promiscuous/subscriber/generic'
|
2
|
+
|
3
|
+
class Promiscuous::Subscriber::Mongoid < Promiscuous::Subscriber::Generic
|
4
|
+
def self.subscribe(options)
|
5
|
+
return super if options[:mongoid_loaded]
|
6
|
+
|
7
|
+
klass = options[:class]
|
8
|
+
klass = options[:classes].values.first if klass.nil?
|
9
|
+
|
10
|
+
if klass.embedded?
|
11
|
+
require 'promiscuous/subscriber/mongoid/embedded'
|
12
|
+
include Promiscuous::Subscriber::Mongoid::Embedded
|
13
|
+
else
|
14
|
+
require 'promiscuous/subscriber/mongoid/root'
|
15
|
+
include Promiscuous::Subscriber::Mongoid::Root
|
16
|
+
|
17
|
+
if options[:upsert]
|
18
|
+
require 'promiscuous/subscriber/mongoid/upsert'
|
19
|
+
include Promiscuous::Subscriber::Mongoid::Upsert
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
self.subscribe(options.merge(:mongoid_loaded => true))
|
24
|
+
end
|
25
|
+
end
|
@@ -0,0 +1,13 @@
|
|
1
|
+
module Promiscuous::Subscriber::Mongoid::Embedded
|
2
|
+
extend ActiveSupport::Concern
|
3
|
+
|
4
|
+
def fetch
|
5
|
+
old_value.nil? ? klass.new.tap { |m| m.id = id } : old_value
|
6
|
+
end
|
7
|
+
|
8
|
+
def old_value
|
9
|
+
options[:old_value]
|
10
|
+
end
|
11
|
+
|
12
|
+
included { use_payload_attribute :id }
|
13
|
+
end
|
@@ -0,0 +1,35 @@
|
|
1
|
+
module Promiscuous::Subscriber::Mongoid::Root
|
2
|
+
extend ActiveSupport::Concern
|
3
|
+
|
4
|
+
def fetch
|
5
|
+
case operation
|
6
|
+
when :create
|
7
|
+
klass.new.tap { |o| o.id = id }
|
8
|
+
when :update
|
9
|
+
klass.find(id)
|
10
|
+
when :destroy
|
11
|
+
klass.find(id)
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
def process_attributes?
|
16
|
+
operation != :destroy
|
17
|
+
end
|
18
|
+
|
19
|
+
def process
|
20
|
+
super
|
21
|
+
case operation
|
22
|
+
when :create
|
23
|
+
instance.save!
|
24
|
+
when :update
|
25
|
+
instance.save!
|
26
|
+
when :destroy
|
27
|
+
instance.destroy
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
included do
|
32
|
+
use_payload_attribute :id
|
33
|
+
use_payload_attribute :operation, :symbolize => true
|
34
|
+
end
|
35
|
+
end
|
@@ -0,0 +1,12 @@
|
|
1
|
+
module Promiscuous::Subscriber::Mongoid::Upsert
|
2
|
+
extend ActiveSupport::Concern
|
3
|
+
|
4
|
+
def fetch
|
5
|
+
begin
|
6
|
+
super
|
7
|
+
rescue Mongoid::Errors::DocumentNotFound
|
8
|
+
Promiscuous::AMQP.warn "[receive] upserting #{payload}"
|
9
|
+
klass.new.tap { |o| o.id = id }
|
10
|
+
end
|
11
|
+
end
|
12
|
+
end
|
@@ -0,0 +1,13 @@
|
|
1
|
+
require 'promiscuous/subscriber/envelope'
|
2
|
+
|
3
|
+
module Promiscuous::Subscriber::Polymorphic
|
4
|
+
extend ActiveSupport::Concern
|
5
|
+
include Promiscuous::Subscriber::Envelope
|
6
|
+
|
7
|
+
def klass
|
8
|
+
klass = (subscribe_options[:classes] || {})[type]
|
9
|
+
klass.nil? ? super : klass
|
10
|
+
end
|
11
|
+
|
12
|
+
included { use_payload_attribute :type }
|
13
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
module Promiscuous
|
2
|
+
module Worker
|
3
|
+
def self.run
|
4
|
+
queue_name = "#{Promiscuous::AMQP.app}.promiscuous"
|
5
|
+
|
6
|
+
stop = false
|
7
|
+
Promiscuous::AMQP.subscribe(:queue_name => queue_name,
|
8
|
+
:bindings => Promiscuous::Subscriber::AMQP.subscribers.keys) do |metadata, payload|
|
9
|
+
begin
|
10
|
+
unless stop
|
11
|
+
Promiscuous::AMQP.info "[receive] #{payload}"
|
12
|
+
Promiscuous::Subscriber.process(JSON.parse(payload))
|
13
|
+
metadata.ack
|
14
|
+
end
|
15
|
+
rescue Exception => e
|
16
|
+
e = Promiscuous::Subscriber::Error.new(e, payload)
|
17
|
+
stop = true
|
18
|
+
Promiscuous::AMQP.close
|
19
|
+
Promiscuous::AMQP.error "[receive] FATAL #{e}"
|
20
|
+
Promiscuous::AMQP.error_handler.call(e) if Promiscuous::AMQP.error_handler
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
metadata
ADDED
@@ -0,0 +1,163 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: promiscuous
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: '0.1'
|
5
|
+
prerelease:
|
6
|
+
platform: ruby
|
7
|
+
authors:
|
8
|
+
- Nicolas Viennot
|
9
|
+
- Kareem Kouddous
|
10
|
+
autorequire:
|
11
|
+
bindir: bin
|
12
|
+
cert_chain: []
|
13
|
+
date: 2012-08-11 00:00:00.000000000 Z
|
14
|
+
dependencies:
|
15
|
+
- !ruby/object:Gem::Dependency
|
16
|
+
name: mongoid
|
17
|
+
requirement: !ruby/object:Gem::Requirement
|
18
|
+
none: false
|
19
|
+
requirements:
|
20
|
+
- - ! '>='
|
21
|
+
- !ruby/object:Gem::Version
|
22
|
+
version: '2.4'
|
23
|
+
type: :runtime
|
24
|
+
prerelease: false
|
25
|
+
version_requirements: !ruby/object:Gem::Requirement
|
26
|
+
none: false
|
27
|
+
requirements:
|
28
|
+
- - ! '>='
|
29
|
+
- !ruby/object:Gem::Version
|
30
|
+
version: '2.4'
|
31
|
+
- !ruby/object:Gem::Dependency
|
32
|
+
name: activesupport
|
33
|
+
requirement: !ruby/object:Gem::Requirement
|
34
|
+
none: false
|
35
|
+
requirements:
|
36
|
+
- - ! '>='
|
37
|
+
- !ruby/object:Gem::Version
|
38
|
+
version: '0'
|
39
|
+
type: :runtime
|
40
|
+
prerelease: false
|
41
|
+
version_requirements: !ruby/object:Gem::Requirement
|
42
|
+
none: false
|
43
|
+
requirements:
|
44
|
+
- - ! '>='
|
45
|
+
- !ruby/object:Gem::Version
|
46
|
+
version: '0'
|
47
|
+
- !ruby/object:Gem::Dependency
|
48
|
+
name: bunny
|
49
|
+
requirement: !ruby/object:Gem::Requirement
|
50
|
+
none: false
|
51
|
+
requirements:
|
52
|
+
- - ! '>='
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
55
|
+
type: :runtime
|
56
|
+
prerelease: false
|
57
|
+
version_requirements: !ruby/object:Gem::Requirement
|
58
|
+
none: false
|
59
|
+
requirements:
|
60
|
+
- - ! '>='
|
61
|
+
- !ruby/object:Gem::Version
|
62
|
+
version: '0'
|
63
|
+
- !ruby/object:Gem::Dependency
|
64
|
+
name: amqp
|
65
|
+
requirement: !ruby/object:Gem::Requirement
|
66
|
+
none: false
|
67
|
+
requirements:
|
68
|
+
- - ! '>='
|
69
|
+
- !ruby/object:Gem::Version
|
70
|
+
version: '0'
|
71
|
+
type: :runtime
|
72
|
+
prerelease: false
|
73
|
+
version_requirements: !ruby/object:Gem::Requirement
|
74
|
+
none: false
|
75
|
+
requirements:
|
76
|
+
- - ! '>='
|
77
|
+
- !ruby/object:Gem::Version
|
78
|
+
version: '0'
|
79
|
+
- !ruby/object:Gem::Dependency
|
80
|
+
name: em-synchrony
|
81
|
+
requirement: !ruby/object:Gem::Requirement
|
82
|
+
none: false
|
83
|
+
requirements:
|
84
|
+
- - ! '>='
|
85
|
+
- !ruby/object:Gem::Version
|
86
|
+
version: '0'
|
87
|
+
type: :runtime
|
88
|
+
prerelease: false
|
89
|
+
version_requirements: !ruby/object:Gem::Requirement
|
90
|
+
none: false
|
91
|
+
requirements:
|
92
|
+
- - ! '>='
|
93
|
+
- !ruby/object:Gem::Version
|
94
|
+
version: '0'
|
95
|
+
description: Replicate data across your applications
|
96
|
+
email:
|
97
|
+
- nicolas@viennot.biz
|
98
|
+
- kareem@doubleonemedia.com
|
99
|
+
executables: []
|
100
|
+
extensions: []
|
101
|
+
extra_rdoc_files: []
|
102
|
+
files:
|
103
|
+
- lib/promiscuous/amqp/fake.rb
|
104
|
+
- lib/promiscuous/amqp/null.rb
|
105
|
+
- lib/promiscuous/amqp/bunny.rb
|
106
|
+
- lib/promiscuous/amqp/ruby-amqp.rb
|
107
|
+
- lib/promiscuous/railtie/replicate.rake
|
108
|
+
- lib/promiscuous/publisher/mongoid/embedded.rb
|
109
|
+
- lib/promiscuous/publisher/mongoid/root.rb
|
110
|
+
- lib/promiscuous/publisher/base.rb
|
111
|
+
- lib/promiscuous/publisher/class_bind.rb
|
112
|
+
- lib/promiscuous/publisher/envelope.rb
|
113
|
+
- lib/promiscuous/publisher/mongoid.rb
|
114
|
+
- lib/promiscuous/publisher/polymorphic.rb
|
115
|
+
- lib/promiscuous/publisher/attributes.rb
|
116
|
+
- lib/promiscuous/publisher/generic.rb
|
117
|
+
- lib/promiscuous/publisher/amqp.rb
|
118
|
+
- lib/promiscuous/subscriber/mongoid/embedded.rb
|
119
|
+
- lib/promiscuous/subscriber/mongoid/upsert.rb
|
120
|
+
- lib/promiscuous/subscriber/mongoid/root.rb
|
121
|
+
- lib/promiscuous/subscriber/attributes.rb
|
122
|
+
- lib/promiscuous/subscriber/base.rb
|
123
|
+
- lib/promiscuous/subscriber/custom_class.rb
|
124
|
+
- lib/promiscuous/subscriber/envelope.rb
|
125
|
+
- lib/promiscuous/subscriber/generic.rb
|
126
|
+
- lib/promiscuous/subscriber/polymorphic.rb
|
127
|
+
- lib/promiscuous/subscriber/mongoid.rb
|
128
|
+
- lib/promiscuous/subscriber/amqp.rb
|
129
|
+
- lib/promiscuous/subscriber/error.rb
|
130
|
+
- lib/promiscuous/version.rb
|
131
|
+
- lib/promiscuous/publisher.rb
|
132
|
+
- lib/promiscuous/railtie.rb
|
133
|
+
- lib/promiscuous/amqp.rb
|
134
|
+
- lib/promiscuous/subscriber.rb
|
135
|
+
- lib/promiscuous/worker.rb
|
136
|
+
- lib/promiscuous.rb
|
137
|
+
- README.md
|
138
|
+
homepage: http://github.com/crowdtap/promiscuous
|
139
|
+
licenses: []
|
140
|
+
post_install_message:
|
141
|
+
rdoc_options: []
|
142
|
+
require_paths:
|
143
|
+
- lib
|
144
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
145
|
+
none: false
|
146
|
+
requirements:
|
147
|
+
- - ! '>='
|
148
|
+
- !ruby/object:Gem::Version
|
149
|
+
version: '0'
|
150
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
151
|
+
none: false
|
152
|
+
requirements:
|
153
|
+
- - ! '>='
|
154
|
+
- !ruby/object:Gem::Version
|
155
|
+
version: '0'
|
156
|
+
requirements: []
|
157
|
+
rubyforge_project:
|
158
|
+
rubygems_version: 1.8.24
|
159
|
+
signing_key:
|
160
|
+
specification_version: 3
|
161
|
+
summary: Model replication over RabbitMQ
|
162
|
+
test_files: []
|
163
|
+
has_rdoc: false
|