promiscuous 0.53.1 → 0.90.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (106) hide show
  1. data/lib/promiscuous.rb +25 -28
  2. data/lib/promiscuous/amqp.rb +27 -8
  3. data/lib/promiscuous/amqp/bunny.rb +131 -16
  4. data/lib/promiscuous/amqp/fake.rb +52 -0
  5. data/lib/promiscuous/amqp/hot_bunnies.rb +56 -0
  6. data/lib/promiscuous/amqp/null.rb +6 -6
  7. data/lib/promiscuous/cli.rb +108 -24
  8. data/lib/promiscuous/config.rb +73 -12
  9. data/lib/promiscuous/convenience.rb +18 -0
  10. data/lib/promiscuous/dependency.rb +59 -0
  11. data/lib/promiscuous/dsl.rb +36 -0
  12. data/lib/promiscuous/error.rb +3 -1
  13. data/lib/promiscuous/error/already_processed.rb +5 -0
  14. data/lib/promiscuous/error/base.rb +1 -0
  15. data/lib/promiscuous/error/connection.rb +7 -5
  16. data/lib/promiscuous/error/dependency.rb +111 -0
  17. data/lib/promiscuous/error/lock_unavailable.rb +12 -0
  18. data/lib/promiscuous/error/lost_lock.rb +12 -0
  19. data/lib/promiscuous/error/missing_context.rb +29 -0
  20. data/lib/promiscuous/error/publisher.rb +5 -15
  21. data/lib/promiscuous/error/recovery.rb +7 -0
  22. data/lib/promiscuous/error/subscriber.rb +2 -4
  23. data/lib/promiscuous/key.rb +36 -0
  24. data/lib/promiscuous/loader.rb +12 -16
  25. data/lib/promiscuous/middleware.rb +112 -0
  26. data/lib/promiscuous/publisher.rb +7 -4
  27. data/lib/promiscuous/publisher/context.rb +92 -0
  28. data/lib/promiscuous/publisher/mock_generator.rb +72 -0
  29. data/lib/promiscuous/publisher/model.rb +3 -86
  30. data/lib/promiscuous/publisher/model/active_record.rb +8 -15
  31. data/lib/promiscuous/publisher/model/base.rb +136 -0
  32. data/lib/promiscuous/publisher/model/ephemeral.rb +69 -0
  33. data/lib/promiscuous/publisher/model/mock.rb +61 -0
  34. data/lib/promiscuous/publisher/model/mongoid.rb +57 -100
  35. data/lib/promiscuous/{common/lint.rb → publisher/operation.rb} +1 -1
  36. data/lib/promiscuous/publisher/operation/base.rb +707 -0
  37. data/lib/promiscuous/publisher/operation/mongoid.rb +370 -0
  38. data/lib/promiscuous/publisher/worker.rb +22 -0
  39. data/lib/promiscuous/railtie.rb +21 -3
  40. data/lib/promiscuous/redis.rb +132 -40
  41. data/lib/promiscuous/resque.rb +12 -0
  42. data/lib/promiscuous/sidekiq.rb +15 -0
  43. data/lib/promiscuous/subscriber.rb +9 -20
  44. data/lib/promiscuous/subscriber/model.rb +4 -104
  45. data/lib/promiscuous/subscriber/model/active_record.rb +10 -0
  46. data/lib/promiscuous/subscriber/model/base.rb +96 -0
  47. data/lib/promiscuous/subscriber/model/mongoid.rb +86 -0
  48. data/lib/promiscuous/subscriber/model/observer.rb +37 -0
  49. data/lib/promiscuous/subscriber/operation.rb +167 -0
  50. data/lib/promiscuous/subscriber/payload.rb +34 -0
  51. data/lib/promiscuous/subscriber/worker.rb +22 -18
  52. data/lib/promiscuous/subscriber/worker/message.rb +48 -25
  53. data/lib/promiscuous/subscriber/worker/message_synchronizer.rb +273 -181
  54. data/lib/promiscuous/subscriber/worker/pump.rb +17 -43
  55. data/lib/promiscuous/subscriber/worker/recorder.rb +24 -0
  56. data/lib/promiscuous/subscriber/worker/runner.rb +24 -3
  57. data/lib/promiscuous/subscriber/worker/stats.rb +62 -0
  58. data/lib/promiscuous/timer.rb +38 -0
  59. data/lib/promiscuous/version.rb +1 -1
  60. metadata +98 -143
  61. data/README.md +0 -33
  62. data/lib/promiscuous/amqp/ruby_amqp.rb +0 -140
  63. data/lib/promiscuous/common.rb +0 -4
  64. data/lib/promiscuous/common/class_helpers.rb +0 -12
  65. data/lib/promiscuous/common/lint/base.rb +0 -24
  66. data/lib/promiscuous/common/options.rb +0 -51
  67. data/lib/promiscuous/ephemeral.rb +0 -14
  68. data/lib/promiscuous/error/recover.rb +0 -1
  69. data/lib/promiscuous/observer.rb +0 -5
  70. data/lib/promiscuous/publisher/active_record.rb +0 -7
  71. data/lib/promiscuous/publisher/amqp.rb +0 -18
  72. data/lib/promiscuous/publisher/attributes.rb +0 -32
  73. data/lib/promiscuous/publisher/base.rb +0 -23
  74. data/lib/promiscuous/publisher/class.rb +0 -36
  75. data/lib/promiscuous/publisher/envelope.rb +0 -7
  76. data/lib/promiscuous/publisher/ephemeral.rb +0 -9
  77. data/lib/promiscuous/publisher/lint.rb +0 -35
  78. data/lib/promiscuous/publisher/lint/amqp.rb +0 -14
  79. data/lib/promiscuous/publisher/lint/attributes.rb +0 -12
  80. data/lib/promiscuous/publisher/lint/base.rb +0 -5
  81. data/lib/promiscuous/publisher/lint/class.rb +0 -15
  82. data/lib/promiscuous/publisher/lint/polymorphic.rb +0 -22
  83. data/lib/promiscuous/publisher/mock.rb +0 -79
  84. data/lib/promiscuous/publisher/mongoid.rb +0 -33
  85. data/lib/promiscuous/publisher/mongoid/embedded.rb +0 -27
  86. data/lib/promiscuous/publisher/mongoid/embedded_many.rb +0 -12
  87. data/lib/promiscuous/publisher/polymorphic.rb +0 -8
  88. data/lib/promiscuous/subscriber/active_record.rb +0 -11
  89. data/lib/promiscuous/subscriber/amqp.rb +0 -25
  90. data/lib/promiscuous/subscriber/attributes.rb +0 -35
  91. data/lib/promiscuous/subscriber/base.rb +0 -29
  92. data/lib/promiscuous/subscriber/class.rb +0 -29
  93. data/lib/promiscuous/subscriber/dummy.rb +0 -19
  94. data/lib/promiscuous/subscriber/envelope.rb +0 -18
  95. data/lib/promiscuous/subscriber/lint.rb +0 -30
  96. data/lib/promiscuous/subscriber/lint/amqp.rb +0 -21
  97. data/lib/promiscuous/subscriber/lint/attributes.rb +0 -21
  98. data/lib/promiscuous/subscriber/lint/base.rb +0 -14
  99. data/lib/promiscuous/subscriber/lint/class.rb +0 -13
  100. data/lib/promiscuous/subscriber/lint/polymorphic.rb +0 -39
  101. data/lib/promiscuous/subscriber/mongoid.rb +0 -27
  102. data/lib/promiscuous/subscriber/mongoid/embedded.rb +0 -17
  103. data/lib/promiscuous/subscriber/mongoid/embedded_many.rb +0 -44
  104. data/lib/promiscuous/subscriber/observer.rb +0 -26
  105. data/lib/promiscuous/subscriber/polymorphic.rb +0 -36
  106. data/lib/promiscuous/subscriber/upsert.rb +0 -12
data/README.md DELETED
@@ -1,33 +0,0 @@
1
- Promiscuous
2
- ===========
3
-
4
- [![Build Status](https://travis-ci.org/crowdtap/promiscuous.png?branch=travis)](https://travis-ci.org/crowdtap/promiscuous)
5
-
6
- Promiscuous is designed to facilitate designing a
7
- [service-oriented architecture](http://en.wikipedia.org/wiki/Service-oriented_architecture)
8
- in Ruby.
9
-
10
- Promiscuous offers an automatic way of propagating your data across one or more
11
- applications. It supports Mongoid2, Mongoid3 and ActiveRecord.
12
- It relies on [RabbitMQ](http://www.rabbitmq.com/) to push data around.
13
-
14
- Philosophy
15
- ----------
16
-
17
- In order for a service-oriented system to be successful, services *must* be
18
- loosely coupled. The traditional Ruby way of tackling this problem is to
19
- provide RESTful APIs.
20
- Sadly, this come to a cost since one must write controllers, integration tests, etc.
21
- Promiscuous to the rescue
22
-
23
- Compatibility
24
- -------------
25
-
26
- Promiscuous is tested against MRI 1.9.3.
27
-
28
- ActiveRecord, Mongoid 2.4.x and Mongoid 3.0.x are supported.
29
-
30
- License
31
- -------
32
-
33
- Promiscuous is distributed under the MIT license.
@@ -1,140 +0,0 @@
1
- require 'eventmachine'
2
- require 'amqp'
3
-
4
- module Promiscuous::AMQP::RubyAMQP
5
- class Synchronizer
6
- def initialize
7
- @mutex = Mutex.new
8
- @condition = ConditionVariable.new
9
- @signaled = false
10
- end
11
-
12
- def wait
13
- @mutex.synchronize do
14
- loop do
15
- return if @signaled
16
- @condition.wait(@mutex)
17
- end
18
- end
19
- end
20
-
21
- def signal
22
- @mutex.synchronize do
23
- @signaled = true
24
- @condition.signal
25
- end
26
- end
27
- end
28
-
29
- def self.maybe_start_event_machine
30
- return if EM.reactor_running?
31
-
32
- EM.error_handler { |e| Promiscuous::Config.error_notifier.try(:call, e) }
33
- em_sync = Synchronizer.new
34
- @event_machine_thread = Thread.new { EM.run { em_sync.signal } }
35
- em_sync.wait
36
- end
37
-
38
- def self.connect
39
- return if @connection
40
-
41
- @channels = {}
42
- @exchanges = {}
43
-
44
- maybe_start_event_machine
45
-
46
- amqp_options = if Promiscuous::Config.amqp_url
47
- url = URI.parse(Promiscuous::Config.amqp_url)
48
- raise "Please use amqp://user:password@host:port/vhost" if url.scheme != 'amqp'
49
-
50
- {
51
- :host => url.host,
52
- :port => url.port,
53
- :scheme => url.scheme,
54
- :user => url.user,
55
- :pass => url.password,
56
- :vhost => url.path.empty? ? "/" : url.path,
57
- :heartbeat => Promiscuous::Config.heartbeat
58
- }
59
- end
60
-
61
- channel_sync = Synchronizer.new
62
- ::AMQP.connect(amqp_options) do |connection|
63
- @connection = connection
64
- @connection.on_tcp_connection_loss do |conn|
65
- unless conn.reconnecting?
66
- e = Promiscuous::AMQP.lost_connection_exception
67
- Promiscuous.warn "[amqp] #{e}. Reconnecting..."
68
- Promiscuous::Config.error_notifier.try(:call, e)
69
- conn.periodically_reconnect(2.seconds)
70
- end
71
- end
72
-
73
- @connection.on_recovery do |conn|
74
- Promiscuous.warn "[amqp] Reconnected"
75
- @channels.values.each(&:recover) if conn == @connection
76
- end
77
-
78
- @connection.on_error do |conn, conn_close|
79
- # No need to handle CONNECTION_FORCED since on_tcp_connection_loss takes
80
- # care of it.
81
- Promiscuous.warn "[amqp] #{conn_close.reply_text}"
82
- end
83
-
84
- get_channel(:master) { channel_sync.signal }
85
- end
86
- channel_sync.wait
87
- rescue Exception => e
88
- self.disconnect
89
- raise e
90
- end
91
-
92
- def self.get_channel(name, &block)
93
- if @channels[name]
94
- yield(@channels[name]) if block_given?
95
- @channels[name]
96
- else
97
- options = {:auto_recovery => true, :prefetch => Promiscuous::Config.prefetch}
98
- ::AMQP::Channel.new(@connection, options) do |channel|
99
- @channels[name] = channel
100
- get_exchange(name)
101
- yield(channel) if block_given?
102
- end
103
- end
104
- end
105
-
106
- def self.close_channel(name, &block)
107
- EM.next_tick do
108
- channel = @channels.try(:delete, name)
109
- if channel
110
- channel.close(&block)
111
- else
112
- block.call if block
113
- end
114
- end
115
- end
116
-
117
- def self.disconnect
118
- @connection.close { EM.stop if @event_machine_thread } if @connection
119
- @event_machine_thread.join if @event_machine_thread
120
- @event_machine_thread = nil
121
- @connection = nil
122
- @channels = nil
123
- @exchanges = nil
124
- end
125
-
126
- def self.connected?
127
- @connection.connected? if @connection
128
- end
129
-
130
- def self.publish(options={})
131
- EM.next_tick do
132
- get_exchange(:master).publish(options[:payload], :routing_key => options[:key], :persistent => true) do
133
- end
134
- end
135
- end
136
-
137
- def self.get_exchange(name)
138
- @exchanges[name] ||= get_channel(name).topic(Promiscuous::AMQP::EXCHANGE, :durable => true)
139
- end
140
- end
@@ -1,4 +0,0 @@
1
- module Promiscuous::Common
2
- extend Promiscuous::Autoload
3
- autoload :Options, :Lint, :ClassHelpers
4
- end
@@ -1,12 +0,0 @@
1
- module Promiscuous::Common::ClassHelpers
2
- extend ActiveSupport::Concern
3
-
4
- module ClassMethods
5
- def guess_class_name(separator)
6
- return nil if name.nil?
7
- class_name = name.split("::").reverse.take_while { |name| name != separator }.reverse.join('::')
8
- class_name = $1 if class_name =~ /^(.+)#{separator.singularize}$/
9
- class_name
10
- end
11
- end
12
- end
@@ -1,24 +0,0 @@
1
- module Promiscuous::Common::Lint::Base
2
- extend ActiveSupport::Concern
3
-
4
- included do
5
- attr_accessor :options
6
- end
7
-
8
- def initialize(options)
9
- self.options = options
10
- end
11
-
12
- def lint
13
- end
14
-
15
- module ClassMethods
16
- def use_option(*attrs)
17
- attrs.each do |attr|
18
- define_method(attr) do
19
- self.options[attr]
20
- end
21
- end
22
- end
23
- end
24
- end
@@ -1,51 +0,0 @@
1
- module Promiscuous::Common::Options
2
- extend ActiveSupport::Concern
3
-
4
- included do
5
- class_attribute :raw_options, :options, :options_mappings,
6
- :instance_reader => false,
7
- :instance_writer => false
8
- self.raw_options = {}
9
- self.options = {}
10
- self.options_mappings = {}
11
- end
12
-
13
- module ClassMethods
14
- def inherited(subclass)
15
- super
16
- subclass.options = self.options.dup
17
- end
18
-
19
- def use_option(attr, options={})
20
- instance_reader = options.fetch(:instance_reader, true)
21
- attr_alias = options.fetch(:as, attr)
22
-
23
- self.options_mappings[attr] = attr_alias
24
-
25
- base = self.ancestors[self.ancestors.index(Promiscuous::Common::Options) - 1]
26
-
27
- # We need to let all the modules overload these methods, which is
28
- # why we are injecting at the base level.
29
- base.singleton_class.class_eval do
30
- define_method("#{attr_alias}") { self.options[attr] }
31
- define_method("#{attr_alias}=") { |value| self.options[attr] = value }
32
- end
33
-
34
- if instance_reader
35
- define_method("#{attr_alias}") { self.class.__send__("#{attr_alias}") }
36
- end
37
-
38
- self.__send__("#{attr_alias}=", raw_options[attr]) if raw_options[attr]
39
- end
40
-
41
- def load_options(options)
42
- self.raw_options = self.raw_options.dup
43
- self.raw_options.merge!(options)
44
-
45
- options.each do |attr, value|
46
- attr_alias = self.options_mappings[attr]
47
- self.__send__("#{attr_alias}=", value) if attr_alias
48
- end
49
- end
50
- end
51
- end
@@ -1,14 +0,0 @@
1
- class Promiscuous::Ephemeral
2
- def initialize(attributes)
3
- attributes.each { |k, v| self.__send__("#{k}=", v) }
4
- end
5
-
6
- def self.create(attributes)
7
- new(attributes).tap { |m| m.save }
8
- end
9
-
10
- def save
11
- self.class.promiscuous_publisher.new(:instance => self).publish
12
- end
13
- alias save! save
14
- end
@@ -1 +0,0 @@
1
- class Promiscuous::Error::Recover < RuntimeError; end
@@ -1,5 +0,0 @@
1
- class Promiscuous::Observer
2
- extend ActiveModel::Callbacks
3
- attr_accessor :id
4
- define_model_callbacks :create, :update, :destroy, :only => :after
5
- end
@@ -1,7 +0,0 @@
1
- class Promiscuous::Publisher::ActiveRecord < Promiscuous::Publisher::Base
2
- include Promiscuous::Publisher::Class
3
- include Promiscuous::Publisher::Attributes
4
- include Promiscuous::Publisher::AMQP
5
- include Promiscuous::Publisher::Model
6
- include Promiscuous::Publisher::Model::ActiveRecord
7
- end
@@ -1,18 +0,0 @@
1
- module Promiscuous::Publisher::AMQP
2
- extend ActiveSupport::Concern
3
- include Promiscuous::Publisher::Envelope
4
-
5
- def publish
6
- exchange_name = Promiscuous::AMQP::EXCHANGE
7
- exchange_name += ".#{options[:personality]}" if options[:personality]
8
- Promiscuous::AMQP.publish(:exchange_name => exchange_name, :key => to, :payload => payload.to_json)
9
- rescue Exception => e
10
- raise_out_of_sync(e, payload.to_json)
11
- end
12
-
13
- def payload
14
- super.merge(:__amqp__ => to)
15
- end
16
-
17
- included { use_option :to }
18
- end
@@ -1,32 +0,0 @@
1
- module Promiscuous::Publisher::Attributes
2
- extend ActiveSupport::Concern
3
-
4
- def payload
5
- return nil unless include_attributes?
6
-
7
- Hash[attributes.map { |field| [field, payload_for(field)] }]
8
- end
9
-
10
- def payload_for(field)
11
- value = instance.__send__(field)
12
- if value.respond_to?(:promiscuous_publisher)
13
- value.promiscuous_publisher.new(options.merge(:instance => value)).payload
14
- elsif value.class.respond_to?(:promiscuous_publisher)
15
- value.class.promiscuous_publisher.new(options.merge(:instance => value)).payload
16
- else
17
- value
18
- end
19
- end
20
-
21
- def include_attributes?
22
- true
23
- end
24
-
25
- included { use_option :attributes }
26
-
27
- module ClassMethods
28
- def attributes=(value)
29
- super(superclass.attributes.to_a + value)
30
- end
31
- end
32
- end
@@ -1,23 +0,0 @@
1
- class Promiscuous::Publisher::Base
2
- include Promiscuous::Common::Options
3
-
4
- cattr_accessor :published
5
- attr_accessor :options
6
-
7
- def initialize(options)
8
- self.options = options
9
- end
10
-
11
- def instance
12
- options[:instance]
13
- end
14
-
15
- def self.publish(options)
16
- load_options(options)
17
- self.published = true
18
- end
19
-
20
- def raise_out_of_sync(exception, payload)
21
- raise Promiscuous::Error::Publisher.new(exception, :instance => instance, :out_of_sync => true, :payload => payload)
22
- end
23
- end
@@ -1,36 +0,0 @@
1
- module Promiscuous::Publisher::Class
2
- extend ActiveSupport::Concern
3
- include Promiscuous::Common::ClassHelpers
4
-
5
- included { use_option :class, :as => :klass }
6
-
7
- module ClassMethods
8
- def setup_class_binding
9
- publisher_class = self
10
- klass.class_eval do
11
- class_attribute :promiscuous_publisher
12
- self.promiscuous_publisher = publisher_class
13
- end if klass
14
- end
15
-
16
- def self.publish(options)
17
- super
18
- setup_class_binding
19
- end
20
-
21
- def inherited(subclass)
22
- super
23
- subclass.setup_class_binding unless options[:class]
24
- end
25
-
26
- def klass=(value)
27
- super
28
- setup_class_binding
29
- end
30
-
31
- def klass
32
- return nil if name.nil?
33
- "::#{super ? super : guess_class_name('Publishers')}".constantize
34
- end
35
- end
36
- end
@@ -1,7 +0,0 @@
1
- module Promiscuous::Publisher::Envelope
2
- extend ActiveSupport::Concern
3
-
4
- def payload
5
- { :payload => super }
6
- end
7
- end
@@ -1,9 +0,0 @@
1
- class Promiscuous::Publisher::Ephemeral < Promiscuous::Publisher::Base
2
- include Promiscuous::Publisher::Class
3
- include Promiscuous::Publisher::Attributes
4
- include Promiscuous::Publisher::AMQP
5
-
6
- def payload
7
- super.merge(:operation => :create)
8
- end
9
- end