promiscuous 0.1
Sign up to get free protection for your applications and to get access to all the features.
- data/README.md +132 -0
- data/lib/promiscuous.rb +5 -0
- data/lib/promiscuous/amqp.rb +34 -0
- data/lib/promiscuous/amqp/bunny.rb +23 -0
- data/lib/promiscuous/amqp/fake.rb +27 -0
- data/lib/promiscuous/amqp/null.rb +20 -0
- data/lib/promiscuous/amqp/ruby-amqp.rb +53 -0
- data/lib/promiscuous/publisher.rb +3 -0
- data/lib/promiscuous/publisher/amqp.rb +16 -0
- data/lib/promiscuous/publisher/attributes.rb +28 -0
- data/lib/promiscuous/publisher/base.rb +22 -0
- data/lib/promiscuous/publisher/class_bind.rb +15 -0
- data/lib/promiscuous/publisher/envelope.rb +7 -0
- data/lib/promiscuous/publisher/generic.rb +14 -0
- data/lib/promiscuous/publisher/mongoid.rb +17 -0
- data/lib/promiscuous/publisher/mongoid/embedded.rb +27 -0
- data/lib/promiscuous/publisher/mongoid/root.rb +31 -0
- data/lib/promiscuous/publisher/polymorphic.rb +10 -0
- data/lib/promiscuous/railtie.rb +28 -0
- data/lib/promiscuous/railtie/replicate.rake +12 -0
- data/lib/promiscuous/subscriber.rb +14 -0
- data/lib/promiscuous/subscriber/amqp.rb +32 -0
- data/lib/promiscuous/subscriber/attributes.rb +30 -0
- data/lib/promiscuous/subscriber/base.rb +31 -0
- data/lib/promiscuous/subscriber/custom_class.rb +17 -0
- data/lib/promiscuous/subscriber/envelope.rb +18 -0
- data/lib/promiscuous/subscriber/error.rb +18 -0
- data/lib/promiscuous/subscriber/generic.rb +14 -0
- data/lib/promiscuous/subscriber/mongoid.rb +25 -0
- data/lib/promiscuous/subscriber/mongoid/embedded.rb +13 -0
- data/lib/promiscuous/subscriber/mongoid/root.rb +35 -0
- data/lib/promiscuous/subscriber/mongoid/upsert.rb +12 -0
- data/lib/promiscuous/subscriber/polymorphic.rb +13 -0
- data/lib/promiscuous/version.rb +3 -0
- data/lib/promiscuous/worker.rb +25 -0
- metadata +163 -0
data/README.md
ADDED
@@ -0,0 +1,132 @@
|
|
1
|
+
Promiscuous
|
2
|
+
===========
|
3
|
+
|
4
|
+
[![Build Status](https://secure.travis-ci.org/crowdtap/promiscuous.png?branch=master)](https://secure.travis-ci.org/crowdtap/promiscuous)
|
5
|
+
|
6
|
+
Promiscuous offers an automatic way of propagating your model data across one or
|
7
|
+
more applications.
|
8
|
+
It uses [RabbitMQ](http://www.rabbitmq.com/).
|
9
|
+
|
10
|
+
Usage
|
11
|
+
------
|
12
|
+
|
13
|
+
From a publisher side (app that owns the data), create a Publisher per model.
|
14
|
+
as shown below.
|
15
|
+
|
16
|
+
From your subscribers side (apps that receive updates from the publisher),
|
17
|
+
create a Subscriber per model, as shown below.
|
18
|
+
|
19
|
+
Example
|
20
|
+
--------
|
21
|
+
|
22
|
+
### In your publisher app
|
23
|
+
|
24
|
+
```ruby
|
25
|
+
# initializer
|
26
|
+
Promiscuous::AMQP.configure(:backend => :bunny, :app => 'crowdtap', :logger => Rails.logger,
|
27
|
+
:server_uri => 'amqp://user:password@host:port/vhost')
|
28
|
+
|
29
|
+
# publisher
|
30
|
+
class ModelPublisher < Promiscuous::Publisher::Mongoid
|
31
|
+
publish :to => 'crowdtap/model',
|
32
|
+
:class => Model,
|
33
|
+
:attributes => [:field_1, :field_2, :field_3]
|
34
|
+
end
|
35
|
+
```
|
36
|
+
|
37
|
+
### In your subscriber app
|
38
|
+
|
39
|
+
```ruby
|
40
|
+
# initializer
|
41
|
+
Promiscuous::AMQP.configure(:backend => :rubyamqp, :app => 'sniper', :logger => Rails.logger,
|
42
|
+
:server_uri => 'amqp://user:password@host:port/vhost',
|
43
|
+
:queue_options => {:durable => true, :arguments => {'x-ha-policy' => 'all'}},
|
44
|
+
:error_handler => some_proc)
|
45
|
+
|
46
|
+
# subscriber
|
47
|
+
class ModelSubscriber < Promiscuous::Subscriber::Mongoid
|
48
|
+
subscribe :from => 'crowdtap/model',
|
49
|
+
:class => Model,
|
50
|
+
:attributes => [:field_1, :field_2, :field_3]
|
51
|
+
end
|
52
|
+
```
|
53
|
+
|
54
|
+
### Starting the subscriber worker
|
55
|
+
|
56
|
+
rake promiscuous:run[./path/to/promiscuous_initializer.rb]
|
57
|
+
|
58
|
+
How does it work ?
|
59
|
+
------------------
|
60
|
+
|
61
|
+
1. On the publisher side, Promiscuous hooks into the after_create/update/destroy callbacks.
|
62
|
+
2. When a model changes, Promiscuous sends a message to RabbitMQ, to the
|
63
|
+
'promiscuous' [topic exchange](http://www.rabbitmq.com/tutorials/tutorial-five-python.html).
|
64
|
+
3. RabbitMQ routes the messages to each application through queues.
|
65
|
+
We use one queue per application (TODO explain why we need one queue).
|
66
|
+
4. Subscribers apps are running the promiscuous worker, listening on their own queues,
|
67
|
+
executing the create/update/destroy on their databases.
|
68
|
+
|
69
|
+
Note that we use a single exchange to preserve the ordering of data updates
|
70
|
+
across application so that subscribers always see a consistant state of the
|
71
|
+
system.
|
72
|
+
|
73
|
+
WARNING/TODO
|
74
|
+
------------
|
75
|
+
|
76
|
+
Promiscuous does **not** handle:
|
77
|
+
- Any of the atomic operatiors, such as inc, or add_to_set.
|
78
|
+
- Association magic. Example:
|
79
|
+
```ruby
|
80
|
+
# This will NOT replicate particiation_ids:
|
81
|
+
m = Member.first
|
82
|
+
m.particiations = [Participation.first]
|
83
|
+
m.save
|
84
|
+
|
85
|
+
# On the other hand, this will:
|
86
|
+
m = Member.first
|
87
|
+
m.particiation_ids = [Participation.first.ids]
|
88
|
+
m.save
|
89
|
+
```
|
90
|
+
|
91
|
+
Furthermore, it can be racy. Consider this scenario with two interleaving
|
92
|
+
requests A and B:
|
93
|
+
|
94
|
+
1. (A) Update mongo doc X.value = 1
|
95
|
+
2. (B) Update mongo doc X.value = 2
|
96
|
+
3. (B) Publish 'X.value = 2' to Rabbit
|
97
|
+
4. (A) Publish 'X.value = 1' to Rabbit
|
98
|
+
|
99
|
+
At the end of the scenario, on the publisher side, the document X has value
|
100
|
+
equal to 2, while on the subscriber side, the document has a value of 1. This
|
101
|
+
will likely not occur in most scenarios BUT BEWARE. We have plans to fix this
|
102
|
+
issue by using version numbers and mongo's amazing findandmodify.
|
103
|
+
|
104
|
+
What's up with bunny vs ruby-amqp ?
|
105
|
+
-----------------------------------
|
106
|
+
|
107
|
+
Our publisher app does not run an eventmachine loop, which is required for
|
108
|
+
ruby-amqp. Bunny on the other hand allows a non-eventmachine based application
|
109
|
+
to publish messages to rabbitmq.
|
110
|
+
|
111
|
+
How to run the tests
|
112
|
+
--------------------
|
113
|
+
|
114
|
+
rake appraisal:install
|
115
|
+
rake
|
116
|
+
|
117
|
+
Compatibility
|
118
|
+
-------------
|
119
|
+
|
120
|
+
Promiscuous is tested against MRI 1.9.2 and 1.9.3.
|
121
|
+
|
122
|
+
Both Mongoid 2.4.x and Mongoid 3.0.x are supported.
|
123
|
+
|
124
|
+
Acknowledgments
|
125
|
+
----------------
|
126
|
+
|
127
|
+
Inspired by [Service-Oriented Design with Ruby and Rails](http://www.amazon.com/Service-Oriented-Design-Addison-Wesley-Professional-Series/dp/0321659368)
|
128
|
+
|
129
|
+
License
|
130
|
+
-------
|
131
|
+
|
132
|
+
Promiscuous is distributed under the MIT license.
|
data/lib/promiscuous.rb
ADDED
@@ -0,0 +1,34 @@
|
|
1
|
+
require 'promiscuous/amqp/bunny'
|
2
|
+
require 'promiscuous/amqp/fake'
|
3
|
+
require 'promiscuous/amqp/ruby-amqp'
|
4
|
+
require 'promiscuous/amqp/null'
|
5
|
+
|
6
|
+
module Promiscuous
|
7
|
+
module AMQP
|
8
|
+
mattr_accessor :backend, :app, :logger, :error_handler
|
9
|
+
|
10
|
+
def self.configure(options={}, &block)
|
11
|
+
options.symbolize_keys!
|
12
|
+
|
13
|
+
self.backend = "Promiscuous::AMQP::#{options[:backend].to_s.camelize.gsub(/amqp/, 'AMQP')}".constantize
|
14
|
+
self.backend.configure(options, &block)
|
15
|
+
self.app = options[:app]
|
16
|
+
self.logger = options[:logger] || Logger.new(STDOUT).tap { |l| l.level = Logger::WARN }
|
17
|
+
self.error_handler = options[:error_handler]
|
18
|
+
self
|
19
|
+
end
|
20
|
+
|
21
|
+
class << self
|
22
|
+
[:info, :error, :warn, :fatal].each do |level|
|
23
|
+
define_method(level) do |msg|
|
24
|
+
self.logger.__send__(level, "[AMQP] #{msg}")
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
# TODO Evaluate the performance hit of method_missing
|
30
|
+
def self.method_missing(method, *args, &block)
|
31
|
+
self.backend.__send__(method, *args, &block)
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
module Promiscuous
|
2
|
+
module AMQP
|
3
|
+
module Bunny
|
4
|
+
mattr_accessor :connection
|
5
|
+
|
6
|
+
def self.configure(options)
|
7
|
+
require 'bunny'
|
8
|
+
self.connection = ::Bunny.new(options[:server_uri])
|
9
|
+
self.connection.start
|
10
|
+
end
|
11
|
+
|
12
|
+
def self.publish(msg)
|
13
|
+
AMQP.info "[publish] #{msg[:key]} -> #{msg[:payload]}"
|
14
|
+
exchange = connection.exchange('promiscuous', :type => :topic, :durable => true)
|
15
|
+
exchange.publish(msg[:payload], :key => msg[:key], :persistent => true)
|
16
|
+
end
|
17
|
+
|
18
|
+
def self.close
|
19
|
+
self.connection.stop
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
module Promiscuous
|
2
|
+
module AMQP
|
3
|
+
module Fake
|
4
|
+
mattr_accessor :messages, :subscribe_options
|
5
|
+
self.messages = []
|
6
|
+
|
7
|
+
def self.configure(options)
|
8
|
+
end
|
9
|
+
|
10
|
+
def self.publish(msg)
|
11
|
+
self.messages << msg
|
12
|
+
end
|
13
|
+
|
14
|
+
def self.subscribe(options={}, &block)
|
15
|
+
self.subscribe_options = options
|
16
|
+
end
|
17
|
+
|
18
|
+
def self.clear
|
19
|
+
self.messages.clear
|
20
|
+
self.subscribe_options = nil
|
21
|
+
end
|
22
|
+
|
23
|
+
def self.close
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
@@ -0,0 +1,20 @@
|
|
1
|
+
module Promiscuous
|
2
|
+
module AMQP
|
3
|
+
module Null
|
4
|
+
def self.configure(options)
|
5
|
+
end
|
6
|
+
|
7
|
+
def self.publish(msg)
|
8
|
+
end
|
9
|
+
|
10
|
+
def self.subscribe(options={}, &block)
|
11
|
+
end
|
12
|
+
|
13
|
+
def self.clear
|
14
|
+
end
|
15
|
+
|
16
|
+
def self.close
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
@@ -0,0 +1,53 @@
|
|
1
|
+
module Promiscuous
|
2
|
+
module AMQP
|
3
|
+
module RubyAMQP
|
4
|
+
mattr_accessor :channel, :queue_options
|
5
|
+
|
6
|
+
def self.configure(options)
|
7
|
+
require 'amqp'
|
8
|
+
connection = ::AMQP.connect(build_connection_options(options))
|
9
|
+
self.channel = ::AMQP::Channel.new(connection)
|
10
|
+
self.queue_options = options[:queue_options] || {}
|
11
|
+
end
|
12
|
+
|
13
|
+
def self.build_connection_options(options)
|
14
|
+
if options[:server_uri]
|
15
|
+
uri = URI.parse(options[:server_uri])
|
16
|
+
raise "Please use amqp://user:password@host:port/vhost" if uri.scheme != 'amqp'
|
17
|
+
|
18
|
+
{
|
19
|
+
:host => uri.host,
|
20
|
+
:port => uri.port,
|
21
|
+
:scheme => uri.scheme,
|
22
|
+
:user => uri.user,
|
23
|
+
:pass => uri.password,
|
24
|
+
:vhost => uri.path.empty? ? "/" : uri.path,
|
25
|
+
}
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
def self.subscribe(options={}, &block)
|
30
|
+
queue_name = options[:queue_name]
|
31
|
+
bindings = options[:bindings]
|
32
|
+
|
33
|
+
queue = self.channel.queue(queue_name, self.queue_options)
|
34
|
+
exchange = channel.topic('promiscuous', :durable => true)
|
35
|
+
bindings.each do |binding|
|
36
|
+
queue.bind(exchange, :routing_key => binding)
|
37
|
+
AMQP.info "[bind] #{queue_name} -> #{binding}"
|
38
|
+
end
|
39
|
+
queue.subscribe(:ack => true, &block)
|
40
|
+
end
|
41
|
+
|
42
|
+
def self.publish(msg)
|
43
|
+
AMQP.info "[publish] #{msg[:key]} -> #{msg[:payload]}"
|
44
|
+
exchange = channel.topic('promiscuous', :durable => true)
|
45
|
+
exchange.publish(msg[:payload], :routing_key => msg[:key], :persistent => true)
|
46
|
+
end
|
47
|
+
|
48
|
+
def self.close
|
49
|
+
channel.close
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
@@ -0,0 +1,16 @@
|
|
1
|
+
require 'promiscuous/publisher/envelope'
|
2
|
+
|
3
|
+
module Promiscuous::Publisher::AMQP
|
4
|
+
extend ActiveSupport::Concern
|
5
|
+
include Promiscuous::Publisher::Envelope
|
6
|
+
|
7
|
+
def amqp_publish
|
8
|
+
Promiscuous::AMQP.publish(:key => to, :payload => payload.to_json)
|
9
|
+
end
|
10
|
+
|
11
|
+
def payload
|
12
|
+
super.merge(:__amqp__ => to)
|
13
|
+
end
|
14
|
+
|
15
|
+
included { use_option :to }
|
16
|
+
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
module Promiscuous::Publisher::Attributes
|
2
|
+
extend ActiveSupport::Concern
|
3
|
+
|
4
|
+
def payload
|
5
|
+
return nil unless include_attributes?
|
6
|
+
|
7
|
+
Hash[attributes.map do |field|
|
8
|
+
optional = field.to_s[-1] == '?'
|
9
|
+
field = field.to_s[0...-1].to_sym if optional
|
10
|
+
[field, payload_for(field)] if !optional || instance.respond_to?(field)
|
11
|
+
end]
|
12
|
+
end
|
13
|
+
|
14
|
+
def payload_for(field)
|
15
|
+
value = instance.__send__(field)
|
16
|
+
if value.class.respond_to?(:promiscuous_publisher)
|
17
|
+
value.class.promiscuous_publisher.new(options.merge(:instance => value)).payload
|
18
|
+
else
|
19
|
+
value
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
def include_attributes?
|
24
|
+
true
|
25
|
+
end
|
26
|
+
|
27
|
+
included { use_option :attributes }
|
28
|
+
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
class Promiscuous::Publisher::Base
|
2
|
+
attr_accessor :options
|
3
|
+
class_attribute :options
|
4
|
+
|
5
|
+
def initialize(options)
|
6
|
+
self.options = options
|
7
|
+
end
|
8
|
+
|
9
|
+
def instance
|
10
|
+
options[:instance]
|
11
|
+
end
|
12
|
+
|
13
|
+
def self.publish(options)
|
14
|
+
self.options = options
|
15
|
+
end
|
16
|
+
|
17
|
+
def self.use_option(attr)
|
18
|
+
define_method(attr) do
|
19
|
+
self.class.options[attr]
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
@@ -0,0 +1,15 @@
|
|
1
|
+
module Promiscuous::Publisher::ClassBind
|
2
|
+
extend ActiveSupport::Concern
|
3
|
+
|
4
|
+
module ClassMethods
|
5
|
+
def publish(options)
|
6
|
+
super
|
7
|
+
|
8
|
+
publisher_class = self
|
9
|
+
options[:class].class_eval do
|
10
|
+
class_attribute :promiscuous_publisher
|
11
|
+
self.promiscuous_publisher = publisher_class
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
@@ -0,0 +1,14 @@
|
|
1
|
+
require 'promiscuous/publisher/class_bind'
|
2
|
+
require 'promiscuous/publisher/base'
|
3
|
+
require 'promiscuous/publisher/attributes'
|
4
|
+
require 'promiscuous/publisher/polymorphic'
|
5
|
+
require 'promiscuous/publisher/amqp'
|
6
|
+
require 'promiscuous/publisher/envelope'
|
7
|
+
|
8
|
+
class Promiscuous::Publisher::Generic < Promiscuous::Publisher::Base
|
9
|
+
include Promiscuous::Publisher::ClassBind
|
10
|
+
include Promiscuous::Publisher::Attributes
|
11
|
+
include Promiscuous::Publisher::Polymorphic
|
12
|
+
include Promiscuous::Publisher::AMQP
|
13
|
+
include Promiscuous::Publisher::Envelope
|
14
|
+
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
require 'promiscuous/publisher/generic'
|
2
|
+
|
3
|
+
class Promiscuous::Publisher::Mongoid < Promiscuous::Publisher::Generic
|
4
|
+
def self.publish(options)
|
5
|
+
return super if options[:mongoid_loaded]
|
6
|
+
|
7
|
+
if options[:class].embedded?
|
8
|
+
require 'promiscuous/publisher/mongoid/embedded'
|
9
|
+
include Promiscuous::Publisher::Mongoid::Embedded
|
10
|
+
else
|
11
|
+
require 'promiscuous/publisher/mongoid/root'
|
12
|
+
include Promiscuous::Publisher::Mongoid::Root
|
13
|
+
end
|
14
|
+
|
15
|
+
self.publish(options.merge(:mongoid_loaded => true))
|
16
|
+
end
|
17
|
+
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
module Promiscuous::Publisher::Mongoid::Embedded
|
2
|
+
extend ActiveSupport::Concern
|
3
|
+
|
4
|
+
def payload
|
5
|
+
super.merge(:id => instance.id)
|
6
|
+
end
|
7
|
+
|
8
|
+
module ClassMethods
|
9
|
+
def publish(options)
|
10
|
+
super
|
11
|
+
|
12
|
+
options[:class].class_eval do
|
13
|
+
callback = proc do
|
14
|
+
if _parent.respond_to?(:promiscuous_publish_update)
|
15
|
+
_parent.save
|
16
|
+
_parent.reload # mongoid is not that smart, so we need to reload here.
|
17
|
+
_parent.promiscuous_publish_update
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
after_create callback
|
22
|
+
after_update callback
|
23
|
+
after_destroy callback
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
@@ -0,0 +1,31 @@
|
|
1
|
+
module Promiscuous::Publisher::Mongoid::Root
|
2
|
+
extend ActiveSupport::Concern
|
3
|
+
|
4
|
+
def operation
|
5
|
+
options[:operation]
|
6
|
+
end
|
7
|
+
|
8
|
+
def payload
|
9
|
+
super.merge(:id => instance.id, :operation => operation)
|
10
|
+
end
|
11
|
+
|
12
|
+
def include_attributes?
|
13
|
+
operation != :destroy
|
14
|
+
end
|
15
|
+
|
16
|
+
module ClassMethods
|
17
|
+
def publish(options)
|
18
|
+
super
|
19
|
+
|
20
|
+
options[:class].class_eval do
|
21
|
+
[:create, :update, :destroy].each do |operation|
|
22
|
+
__send__("after_#{operation}", "promiscuous_publish_#{operation}".to_sym)
|
23
|
+
|
24
|
+
define_method "promiscuous_publish_#{operation}" do
|
25
|
+
self.class.promiscuous_publisher.new(:instance => self, :operation => operation).amqp_publish
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
module Promiscuous
|
2
|
+
class Railtie < Rails::Railtie
|
3
|
+
rake_tasks { load 'promiscuous/railtie/replicate.rake' }
|
4
|
+
|
5
|
+
initializer 'load promiscuous' do
|
6
|
+
# TODO clean that up
|
7
|
+
config.after_initialize do
|
8
|
+
Dir[Rails.root.join('app', 'publishers', '**_publisher.rb')].map do |file|
|
9
|
+
file.split('/')[-1].split('.')[0].camelize.constantize
|
10
|
+
end
|
11
|
+
ActionDispatch::Reloader.to_prepare do
|
12
|
+
Dir[Rails.root.join('app', 'publishers', '**_publisher.rb')].map do |file|
|
13
|
+
file.split('/')[-1].split('.')[0].camelize.constantize
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
Dir[Rails.root.join('app', 'subscribers', '**_subscriber.rb')].map do |file|
|
18
|
+
file.split('/')[-1].split('.')[0].camelize.constantize
|
19
|
+
end
|
20
|
+
ActionDispatch::Reloader.to_prepare do
|
21
|
+
Dir[Rails.root.join('app', 'subscribers', '**_subscriber.rb')].map do |file|
|
22
|
+
file.split('/')[-1].split('.')[0].camelize.constantize
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
@@ -0,0 +1,12 @@
|
|
1
|
+
namespace :replicable do
|
2
|
+
desc 'Run the subscribers worker'
|
3
|
+
task :run, [:initializer] => :environment do |t, args|
|
4
|
+
require 'replicable/worker'
|
5
|
+
require 'eventmachine'
|
6
|
+
require 'em-synchrony'
|
7
|
+
EM.synchrony do
|
8
|
+
load args.initializer
|
9
|
+
Replicable::Worker.run
|
10
|
+
end
|
11
|
+
end
|
12
|
+
end
|
@@ -0,0 +1,14 @@
|
|
1
|
+
module Promiscuous::Subscriber
|
2
|
+
require 'promiscuous/subscriber/error'
|
3
|
+
require 'promiscuous/subscriber/mongoid'
|
4
|
+
require 'promiscuous/subscriber/amqp'
|
5
|
+
|
6
|
+
def self.process(payload, options={})
|
7
|
+
subscriber = Promiscuous::Subscriber::AMQP.subscriber_for(payload)
|
8
|
+
return payload if subscriber.nil?
|
9
|
+
|
10
|
+
sub = subscriber.new(options.merge(:payload => payload))
|
11
|
+
sub.process if sub.respond_to?(:process)
|
12
|
+
sub.instance
|
13
|
+
end
|
14
|
+
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
require 'promiscuous/subscriber/envelope'
|
2
|
+
|
3
|
+
module Promiscuous::Subscriber::AMQP
|
4
|
+
extend ActiveSupport::Concern
|
5
|
+
|
6
|
+
mattr_accessor :subscribers
|
7
|
+
self.subscribers = {}
|
8
|
+
|
9
|
+
def self.subscriber_for(payload)
|
10
|
+
origin = payload.is_a?(Hash) ? payload['__amqp__'] : nil
|
11
|
+
if origin
|
12
|
+
unless subscribers.has_key?(origin)
|
13
|
+
raise "FATAL: Unknown binding: '#{origin}'"
|
14
|
+
end
|
15
|
+
subscribers[origin]
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
module ClassMethods
|
20
|
+
def subscribe(options)
|
21
|
+
super
|
22
|
+
|
23
|
+
subscribers = Promiscuous::Subscriber::AMQP.subscribers
|
24
|
+
from = options[:from]
|
25
|
+
|
26
|
+
if subscribers.has_key?(from)
|
27
|
+
raise "The subscriber '#{subscribers[from]}' already listen on '#{from}'"
|
28
|
+
end
|
29
|
+
subscribers[from] = self
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
@@ -0,0 +1,30 @@
|
|
1
|
+
module Promiscuous::Subscriber::Attributes
|
2
|
+
extend ActiveSupport::Concern
|
3
|
+
|
4
|
+
def process
|
5
|
+
super
|
6
|
+
return unless process_attributes?
|
7
|
+
|
8
|
+
attributes.each do |attr|
|
9
|
+
attr = attr.to_s
|
10
|
+
optional = attr[-1] == '?'
|
11
|
+
attr = attr[0...-1] if optional
|
12
|
+
setter = "#{attr}="
|
13
|
+
|
14
|
+
if payload.has_key?(attr)
|
15
|
+
value = payload[attr]
|
16
|
+
old_value = instance.__send__(attr)
|
17
|
+
new_value = Promiscuous::Subscriber.process(payload[attr], :old_value => old_value)
|
18
|
+
instance.__send__(setter, new_value) if old_value != new_value
|
19
|
+
else
|
20
|
+
raise "Unknown attribute '#{attr}'" unless optional
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
def process_attributes?
|
26
|
+
true
|
27
|
+
end
|
28
|
+
|
29
|
+
included { use_option :attributes }
|
30
|
+
end
|
@@ -0,0 +1,31 @@
|
|
1
|
+
class Promiscuous::Subscriber::Base
|
2
|
+
attr_accessor :options
|
3
|
+
class_attribute :options
|
4
|
+
|
5
|
+
def initialize(options)
|
6
|
+
self.options = options
|
7
|
+
end
|
8
|
+
|
9
|
+
def payload
|
10
|
+
options[:payload]
|
11
|
+
end
|
12
|
+
alias :instance :payload
|
13
|
+
|
14
|
+
def process
|
15
|
+
end
|
16
|
+
|
17
|
+
def subscribe_options
|
18
|
+
self.class.options
|
19
|
+
end
|
20
|
+
|
21
|
+
def self.subscribe(options)
|
22
|
+
self.options = options
|
23
|
+
end
|
24
|
+
|
25
|
+
def self.use_option(attr, options={})
|
26
|
+
as = options[:as].nil? ? attr : options[:as]
|
27
|
+
define_method(as) do
|
28
|
+
self.class.options[attr]
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
require 'promiscuous/subscriber/envelope'
|
2
|
+
|
3
|
+
module Promiscuous::Subscriber::CustomClass
|
4
|
+
extend ActiveSupport::Concern
|
5
|
+
|
6
|
+
def klass
|
7
|
+
unless subscribe_options[:class]
|
8
|
+
raise "I don't want to be rude or anything, "
|
9
|
+
"but have you defined the class to deserialize?"
|
10
|
+
end
|
11
|
+
subscribe_options[:class]
|
12
|
+
end
|
13
|
+
|
14
|
+
def instance
|
15
|
+
@instance ||= fetch
|
16
|
+
end
|
17
|
+
end
|
@@ -0,0 +1,18 @@
|
|
1
|
+
module Promiscuous::Subscriber::Envelope
|
2
|
+
extend ActiveSupport::Concern
|
3
|
+
|
4
|
+
module ClassMethods
|
5
|
+
def use_payload_attribute(attr, options={})
|
6
|
+
define_method(attr) do
|
7
|
+
value = payload_with_envelope[attr.to_s]
|
8
|
+
value = value.to_sym if options[:symbolize]
|
9
|
+
value
|
10
|
+
end
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
included do
|
15
|
+
alias_method :payload_with_envelope, :payload
|
16
|
+
use_payload_attribute :payload
|
17
|
+
end
|
18
|
+
end
|
@@ -0,0 +1,18 @@
|
|
1
|
+
class Promiscuous::Subscriber::Error < RuntimeError
|
2
|
+
attr_accessor :inner, :payload
|
3
|
+
|
4
|
+
def initialize(inner, payload)
|
5
|
+
super(inner)
|
6
|
+
set_backtrace(inner.backtrace)
|
7
|
+
self.inner = inner
|
8
|
+
self.payload = payload
|
9
|
+
end
|
10
|
+
|
11
|
+
def message
|
12
|
+
"#{inner.message} while processing #{payload}"
|
13
|
+
end
|
14
|
+
|
15
|
+
def to_s
|
16
|
+
message
|
17
|
+
end
|
18
|
+
end
|
@@ -0,0 +1,14 @@
|
|
1
|
+
require 'promiscuous/subscriber/custom_class'
|
2
|
+
require 'promiscuous/subscriber/base'
|
3
|
+
require 'promiscuous/subscriber/attributes'
|
4
|
+
require 'promiscuous/subscriber/polymorphic'
|
5
|
+
require 'promiscuous/subscriber/amqp'
|
6
|
+
require 'promiscuous/subscriber/envelope'
|
7
|
+
|
8
|
+
class Promiscuous::Subscriber::Generic < Promiscuous::Subscriber::Base
|
9
|
+
include Promiscuous::Subscriber::CustomClass
|
10
|
+
include Promiscuous::Subscriber::Attributes
|
11
|
+
include Promiscuous::Subscriber::Polymorphic
|
12
|
+
include Promiscuous::Subscriber::AMQP
|
13
|
+
include Promiscuous::Subscriber::Envelope
|
14
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
require 'promiscuous/subscriber/generic'
|
2
|
+
|
3
|
+
class Promiscuous::Subscriber::Mongoid < Promiscuous::Subscriber::Generic
|
4
|
+
def self.subscribe(options)
|
5
|
+
return super if options[:mongoid_loaded]
|
6
|
+
|
7
|
+
klass = options[:class]
|
8
|
+
klass = options[:classes].values.first if klass.nil?
|
9
|
+
|
10
|
+
if klass.embedded?
|
11
|
+
require 'promiscuous/subscriber/mongoid/embedded'
|
12
|
+
include Promiscuous::Subscriber::Mongoid::Embedded
|
13
|
+
else
|
14
|
+
require 'promiscuous/subscriber/mongoid/root'
|
15
|
+
include Promiscuous::Subscriber::Mongoid::Root
|
16
|
+
|
17
|
+
if options[:upsert]
|
18
|
+
require 'promiscuous/subscriber/mongoid/upsert'
|
19
|
+
include Promiscuous::Subscriber::Mongoid::Upsert
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
self.subscribe(options.merge(:mongoid_loaded => true))
|
24
|
+
end
|
25
|
+
end
|
@@ -0,0 +1,13 @@
|
|
1
|
+
module Promiscuous::Subscriber::Mongoid::Embedded
|
2
|
+
extend ActiveSupport::Concern
|
3
|
+
|
4
|
+
def fetch
|
5
|
+
old_value.nil? ? klass.new.tap { |m| m.id = id } : old_value
|
6
|
+
end
|
7
|
+
|
8
|
+
def old_value
|
9
|
+
options[:old_value]
|
10
|
+
end
|
11
|
+
|
12
|
+
included { use_payload_attribute :id }
|
13
|
+
end
|
@@ -0,0 +1,35 @@
|
|
1
|
+
module Promiscuous::Subscriber::Mongoid::Root
|
2
|
+
extend ActiveSupport::Concern
|
3
|
+
|
4
|
+
def fetch
|
5
|
+
case operation
|
6
|
+
when :create
|
7
|
+
klass.new.tap { |o| o.id = id }
|
8
|
+
when :update
|
9
|
+
klass.find(id)
|
10
|
+
when :destroy
|
11
|
+
klass.find(id)
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
def process_attributes?
|
16
|
+
operation != :destroy
|
17
|
+
end
|
18
|
+
|
19
|
+
def process
|
20
|
+
super
|
21
|
+
case operation
|
22
|
+
when :create
|
23
|
+
instance.save!
|
24
|
+
when :update
|
25
|
+
instance.save!
|
26
|
+
when :destroy
|
27
|
+
instance.destroy
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
included do
|
32
|
+
use_payload_attribute :id
|
33
|
+
use_payload_attribute :operation, :symbolize => true
|
34
|
+
end
|
35
|
+
end
|
@@ -0,0 +1,12 @@
|
|
1
|
+
module Promiscuous::Subscriber::Mongoid::Upsert
|
2
|
+
extend ActiveSupport::Concern
|
3
|
+
|
4
|
+
def fetch
|
5
|
+
begin
|
6
|
+
super
|
7
|
+
rescue Mongoid::Errors::DocumentNotFound
|
8
|
+
Promiscuous::AMQP.warn "[receive] upserting #{payload}"
|
9
|
+
klass.new.tap { |o| o.id = id }
|
10
|
+
end
|
11
|
+
end
|
12
|
+
end
|
@@ -0,0 +1,13 @@
|
|
1
|
+
require 'promiscuous/subscriber/envelope'
|
2
|
+
|
3
|
+
module Promiscuous::Subscriber::Polymorphic
|
4
|
+
extend ActiveSupport::Concern
|
5
|
+
include Promiscuous::Subscriber::Envelope
|
6
|
+
|
7
|
+
def klass
|
8
|
+
klass = (subscribe_options[:classes] || {})[type]
|
9
|
+
klass.nil? ? super : klass
|
10
|
+
end
|
11
|
+
|
12
|
+
included { use_payload_attribute :type }
|
13
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
module Promiscuous
|
2
|
+
module Worker
|
3
|
+
def self.run
|
4
|
+
queue_name = "#{Promiscuous::AMQP.app}.promiscuous"
|
5
|
+
|
6
|
+
stop = false
|
7
|
+
Promiscuous::AMQP.subscribe(:queue_name => queue_name,
|
8
|
+
:bindings => Promiscuous::Subscriber::AMQP.subscribers.keys) do |metadata, payload|
|
9
|
+
begin
|
10
|
+
unless stop
|
11
|
+
Promiscuous::AMQP.info "[receive] #{payload}"
|
12
|
+
Promiscuous::Subscriber.process(JSON.parse(payload))
|
13
|
+
metadata.ack
|
14
|
+
end
|
15
|
+
rescue Exception => e
|
16
|
+
e = Promiscuous::Subscriber::Error.new(e, payload)
|
17
|
+
stop = true
|
18
|
+
Promiscuous::AMQP.close
|
19
|
+
Promiscuous::AMQP.error "[receive] FATAL #{e}"
|
20
|
+
Promiscuous::AMQP.error_handler.call(e) if Promiscuous::AMQP.error_handler
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
metadata
ADDED
@@ -0,0 +1,163 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: promiscuous
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: '0.1'
|
5
|
+
prerelease:
|
6
|
+
platform: ruby
|
7
|
+
authors:
|
8
|
+
- Nicolas Viennot
|
9
|
+
- Kareem Kouddous
|
10
|
+
autorequire:
|
11
|
+
bindir: bin
|
12
|
+
cert_chain: []
|
13
|
+
date: 2012-08-11 00:00:00.000000000 Z
|
14
|
+
dependencies:
|
15
|
+
- !ruby/object:Gem::Dependency
|
16
|
+
name: mongoid
|
17
|
+
requirement: !ruby/object:Gem::Requirement
|
18
|
+
none: false
|
19
|
+
requirements:
|
20
|
+
- - ! '>='
|
21
|
+
- !ruby/object:Gem::Version
|
22
|
+
version: '2.4'
|
23
|
+
type: :runtime
|
24
|
+
prerelease: false
|
25
|
+
version_requirements: !ruby/object:Gem::Requirement
|
26
|
+
none: false
|
27
|
+
requirements:
|
28
|
+
- - ! '>='
|
29
|
+
- !ruby/object:Gem::Version
|
30
|
+
version: '2.4'
|
31
|
+
- !ruby/object:Gem::Dependency
|
32
|
+
name: activesupport
|
33
|
+
requirement: !ruby/object:Gem::Requirement
|
34
|
+
none: false
|
35
|
+
requirements:
|
36
|
+
- - ! '>='
|
37
|
+
- !ruby/object:Gem::Version
|
38
|
+
version: '0'
|
39
|
+
type: :runtime
|
40
|
+
prerelease: false
|
41
|
+
version_requirements: !ruby/object:Gem::Requirement
|
42
|
+
none: false
|
43
|
+
requirements:
|
44
|
+
- - ! '>='
|
45
|
+
- !ruby/object:Gem::Version
|
46
|
+
version: '0'
|
47
|
+
- !ruby/object:Gem::Dependency
|
48
|
+
name: bunny
|
49
|
+
requirement: !ruby/object:Gem::Requirement
|
50
|
+
none: false
|
51
|
+
requirements:
|
52
|
+
- - ! '>='
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
55
|
+
type: :runtime
|
56
|
+
prerelease: false
|
57
|
+
version_requirements: !ruby/object:Gem::Requirement
|
58
|
+
none: false
|
59
|
+
requirements:
|
60
|
+
- - ! '>='
|
61
|
+
- !ruby/object:Gem::Version
|
62
|
+
version: '0'
|
63
|
+
- !ruby/object:Gem::Dependency
|
64
|
+
name: amqp
|
65
|
+
requirement: !ruby/object:Gem::Requirement
|
66
|
+
none: false
|
67
|
+
requirements:
|
68
|
+
- - ! '>='
|
69
|
+
- !ruby/object:Gem::Version
|
70
|
+
version: '0'
|
71
|
+
type: :runtime
|
72
|
+
prerelease: false
|
73
|
+
version_requirements: !ruby/object:Gem::Requirement
|
74
|
+
none: false
|
75
|
+
requirements:
|
76
|
+
- - ! '>='
|
77
|
+
- !ruby/object:Gem::Version
|
78
|
+
version: '0'
|
79
|
+
- !ruby/object:Gem::Dependency
|
80
|
+
name: em-synchrony
|
81
|
+
requirement: !ruby/object:Gem::Requirement
|
82
|
+
none: false
|
83
|
+
requirements:
|
84
|
+
- - ! '>='
|
85
|
+
- !ruby/object:Gem::Version
|
86
|
+
version: '0'
|
87
|
+
type: :runtime
|
88
|
+
prerelease: false
|
89
|
+
version_requirements: !ruby/object:Gem::Requirement
|
90
|
+
none: false
|
91
|
+
requirements:
|
92
|
+
- - ! '>='
|
93
|
+
- !ruby/object:Gem::Version
|
94
|
+
version: '0'
|
95
|
+
description: Replicate data across your applications
|
96
|
+
email:
|
97
|
+
- nicolas@viennot.biz
|
98
|
+
- kareem@doubleonemedia.com
|
99
|
+
executables: []
|
100
|
+
extensions: []
|
101
|
+
extra_rdoc_files: []
|
102
|
+
files:
|
103
|
+
- lib/promiscuous/amqp/fake.rb
|
104
|
+
- lib/promiscuous/amqp/null.rb
|
105
|
+
- lib/promiscuous/amqp/bunny.rb
|
106
|
+
- lib/promiscuous/amqp/ruby-amqp.rb
|
107
|
+
- lib/promiscuous/railtie/replicate.rake
|
108
|
+
- lib/promiscuous/publisher/mongoid/embedded.rb
|
109
|
+
- lib/promiscuous/publisher/mongoid/root.rb
|
110
|
+
- lib/promiscuous/publisher/base.rb
|
111
|
+
- lib/promiscuous/publisher/class_bind.rb
|
112
|
+
- lib/promiscuous/publisher/envelope.rb
|
113
|
+
- lib/promiscuous/publisher/mongoid.rb
|
114
|
+
- lib/promiscuous/publisher/polymorphic.rb
|
115
|
+
- lib/promiscuous/publisher/attributes.rb
|
116
|
+
- lib/promiscuous/publisher/generic.rb
|
117
|
+
- lib/promiscuous/publisher/amqp.rb
|
118
|
+
- lib/promiscuous/subscriber/mongoid/embedded.rb
|
119
|
+
- lib/promiscuous/subscriber/mongoid/upsert.rb
|
120
|
+
- lib/promiscuous/subscriber/mongoid/root.rb
|
121
|
+
- lib/promiscuous/subscriber/attributes.rb
|
122
|
+
- lib/promiscuous/subscriber/base.rb
|
123
|
+
- lib/promiscuous/subscriber/custom_class.rb
|
124
|
+
- lib/promiscuous/subscriber/envelope.rb
|
125
|
+
- lib/promiscuous/subscriber/generic.rb
|
126
|
+
- lib/promiscuous/subscriber/polymorphic.rb
|
127
|
+
- lib/promiscuous/subscriber/mongoid.rb
|
128
|
+
- lib/promiscuous/subscriber/amqp.rb
|
129
|
+
- lib/promiscuous/subscriber/error.rb
|
130
|
+
- lib/promiscuous/version.rb
|
131
|
+
- lib/promiscuous/publisher.rb
|
132
|
+
- lib/promiscuous/railtie.rb
|
133
|
+
- lib/promiscuous/amqp.rb
|
134
|
+
- lib/promiscuous/subscriber.rb
|
135
|
+
- lib/promiscuous/worker.rb
|
136
|
+
- lib/promiscuous.rb
|
137
|
+
- README.md
|
138
|
+
homepage: http://github.com/crowdtap/promiscuous
|
139
|
+
licenses: []
|
140
|
+
post_install_message:
|
141
|
+
rdoc_options: []
|
142
|
+
require_paths:
|
143
|
+
- lib
|
144
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
145
|
+
none: false
|
146
|
+
requirements:
|
147
|
+
- - ! '>='
|
148
|
+
- !ruby/object:Gem::Version
|
149
|
+
version: '0'
|
150
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
151
|
+
none: false
|
152
|
+
requirements:
|
153
|
+
- - ! '>='
|
154
|
+
- !ruby/object:Gem::Version
|
155
|
+
version: '0'
|
156
|
+
requirements: []
|
157
|
+
rubyforge_project:
|
158
|
+
rubygems_version: 1.8.24
|
159
|
+
signing_key:
|
160
|
+
specification_version: 3
|
161
|
+
summary: Model replication over RabbitMQ
|
162
|
+
test_files: []
|
163
|
+
has_rdoc: false
|