promiscuous 1.0.0.beta5 → 1.0.0.beta6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/promiscuous.rb +1 -1
- data/lib/promiscuous/amqp.rb +0 -2
- data/lib/promiscuous/amqp/bunny.rb +66 -28
- data/lib/promiscuous/amqp/fake.rb +4 -1
- data/lib/promiscuous/cli.rb +1 -1
- data/lib/promiscuous/config.rb +37 -9
- data/lib/promiscuous/publisher/model/base.rb +6 -2
- data/lib/promiscuous/publisher/model/mongoid.rb +1 -1
- data/lib/promiscuous/publisher/operation/active_record.rb +53 -12
- data/lib/promiscuous/publisher/operation/base.rb +3 -3
- data/lib/promiscuous/publisher/operation/ephemeral.rb +3 -1
- data/lib/promiscuous/publisher/transport.rb +14 -7
- data/lib/promiscuous/publisher/transport/batch.rb +14 -10
- data/lib/promiscuous/publisher/transport/persistence.rb +1 -1
- data/lib/promiscuous/publisher/transport/persistence/active_record.rb +7 -9
- data/lib/promiscuous/publisher/transport/persistence/redis.rb +34 -0
- data/lib/promiscuous/publisher/transport/worker.rb +2 -1
- data/lib/promiscuous/rabbit.rb +41 -0
- data/lib/promiscuous/subscriber/message.rb +9 -0
- data/lib/promiscuous/subscriber/model/active_record.rb +2 -1
- data/lib/promiscuous/subscriber/unit_of_work.rb +3 -9
- data/lib/promiscuous/subscriber/worker/pump.rb +8 -1
- data/lib/promiscuous/version.rb +1 -1
- metadata +19 -17
- data/lib/promiscuous/amqp/hot_bunnies.rb +0 -66
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 2b5c5915b32dc2c76ffc02404b79933d27314a25
|
4
|
+
data.tar.gz: 3251ac561c92ae3e69f7ca002ea8a32236b604ee
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 7db223aa4a4ca2fd2a9af489468b610f5e00d55ba6b9ea9798c82eca7ae146bf1391b6c77940079feb103ab0ecb5a62d1aee301a8a576b7f29420024e1859aca
|
7
|
+
data.tar.gz: f60196280ba6bcca0b1a1011824f16193d0efcc52cd92615da89b2e712d4b2a9f0daf3ed5ad8b4af7bcaf447553bbba26ef6496da20d6c0af7b885d6ab655857
|
data/lib/promiscuous.rb
CHANGED
@@ -23,7 +23,7 @@ module Promiscuous
|
|
23
23
|
extend Promiscuous::Autoload
|
24
24
|
autoload :Common, :Publisher, :Subscriber, :Observer, :Worker, :Ephemeral,
|
25
25
|
:CLI, :Error, :Loader, :AMQP, :Redis, :ZK, :Config, :DSL, :Key,
|
26
|
-
:Convenience, :Dependency, :Timer
|
26
|
+
:Convenience, :Dependency, :Timer, :Rabbit
|
27
27
|
|
28
28
|
extend Promiscuous::DSL
|
29
29
|
|
data/lib/promiscuous/amqp.rb
CHANGED
@@ -45,22 +45,19 @@ class Promiscuous::AMQP::Bunny
|
|
45
45
|
channel.basic_qos(options[:prefetch]) if options[:prefetch]
|
46
46
|
raw_confirm_select(channel, &method(:on_confirm)) if options[:confirm]
|
47
47
|
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
end
|
52
|
-
[connection, channel, exchanges]
|
53
|
-
else
|
54
|
-
exchange = channel.exchange(options[:exchange], :type => :topic, :durable => true)
|
55
|
-
[connection, channel, exchange]
|
48
|
+
exchanges = {}
|
49
|
+
options[:exchanges].each do |exchange_name|
|
50
|
+
exchanges[exchange_name] = channel.exchange(exchange_name, :type => :topic, :durable => true)
|
56
51
|
end
|
52
|
+
[connection, channel, exchanges]
|
57
53
|
end
|
58
54
|
|
59
55
|
def connect
|
60
|
-
connection_options = { :url
|
61
|
-
:
|
62
|
-
|
63
|
-
|
56
|
+
connection_options = { :url => Promiscuous::Config.publisher_amqp_url,
|
57
|
+
:exchanges => [Promiscuous::Config.publisher_exchange,
|
58
|
+
Promiscuous::Config.sync_exchange],
|
59
|
+
:confirm => true }
|
60
|
+
@connection, @channel, @exchanges = new_connection(connection_options)
|
64
61
|
end
|
65
62
|
|
66
63
|
def disconnect
|
@@ -76,12 +73,13 @@ class Promiscuous::AMQP::Bunny
|
|
76
73
|
end
|
77
74
|
|
78
75
|
def raw_publish(options)
|
79
|
-
options[:exchange].publish(options[:payload], :key => options[:key], :persistent => true)
|
76
|
+
@exchanges[options[:exchange]].publish(options[:payload], :key => options[:key], :persistent => true)
|
80
77
|
end
|
81
78
|
|
82
79
|
def publish(options={})
|
83
|
-
options[:exchange]
|
84
|
-
|
80
|
+
raise "Exchange '#{options[:exchange]}' not one of: #{@exchanges.keys}" unless @exchanges[options[:exchange]]
|
81
|
+
|
82
|
+
Promiscuous.debug "[publish] #{options[:exchange]}/#{options[:key]} #{options[:payload]}"
|
85
83
|
|
86
84
|
@connection_lock.synchronize do
|
87
85
|
tag = @channel.next_publish_seq_no if options[:on_confirm]
|
@@ -111,28 +109,73 @@ class Promiscuous::AMQP::Bunny
|
|
111
109
|
@lock = Mutex.new
|
112
110
|
@prefetch = Promiscuous::Config.prefetch
|
113
111
|
|
112
|
+
configure_rabbit
|
113
|
+
|
114
114
|
connection_options = { :url => Promiscuous::Config.subscriber_amqp_url,
|
115
115
|
:exchanges => options[:bindings].keys,
|
116
116
|
:prefetch => @prefetch }
|
117
117
|
@connection, @channel, exchanges = Promiscuous::AMQP.new_connection(connection_options)
|
118
118
|
|
119
|
-
@
|
120
|
-
|
119
|
+
create_queues(@channel)
|
120
|
+
|
121
|
+
# Main queue binding
|
122
|
+
exchanges.keys.zip(options[:bindings].values).each do |exchange, bindings|
|
121
123
|
bindings.each do |binding|
|
122
124
|
@queue.bind(exchange, :routing_key => binding)
|
123
|
-
Promiscuous.debug "[bind] #{exchange
|
125
|
+
Promiscuous.debug "[bind] #{exchange}/#{binding}/#{Promiscuous::Config.queue_name}"
|
124
126
|
end
|
125
127
|
end
|
126
128
|
|
129
|
+
# Error queue binding
|
130
|
+
@error_queue.bind(Promiscuous::Config.error_exchange, :routing_key => Promiscuous::Config.error_routing)
|
131
|
+
|
127
132
|
@subscription = subscribe_queue(@queue, &block)
|
128
133
|
end
|
129
134
|
|
135
|
+
def configure_rabbit
|
136
|
+
policy = {
|
137
|
+
"dead-letter-routing-key" => Promiscuous::Config.error_routing,
|
138
|
+
"dead-letter-exchange" => Promiscuous::Config.error_exchange
|
139
|
+
}.merge(Promiscuous::Config.queue_policy)
|
140
|
+
|
141
|
+
Promiscuous::Rabbit::Policy.set Promiscuous::Config.queue_name,
|
142
|
+
{
|
143
|
+
"pattern" => Promiscuous::Config.queue_name,
|
144
|
+
"apply-to" => "queues",
|
145
|
+
"definition" => policy
|
146
|
+
}
|
147
|
+
|
148
|
+
policy = {
|
149
|
+
"message-ttl" => Promiscuous::Config.error_ttl,
|
150
|
+
"dead-letter-routing-key" => Promiscuous::Config.retry_routing,
|
151
|
+
"dead-letter-exchange" => Promiscuous::Config.error_exchange
|
152
|
+
}.merge(Promiscuous::Config.queue_policy)
|
153
|
+
Promiscuous::Rabbit::Policy.set Promiscuous::Config.error_queue_name,
|
154
|
+
{
|
155
|
+
"pattern" => Promiscuous::Config.error_queue_name,
|
156
|
+
"apply-to" => "queues",
|
157
|
+
"definition" => policy
|
158
|
+
}
|
159
|
+
end
|
160
|
+
|
130
161
|
def subscribe_queue(queue, &block)
|
131
|
-
queue.subscribe(:
|
162
|
+
queue.subscribe(:manual_ack => true) do |delivery_info, metadata, payload|
|
132
163
|
block.call(MetaData.new(self, delivery_info), payload)
|
133
164
|
end
|
134
165
|
end
|
135
166
|
|
167
|
+
def create_queues(channel)
|
168
|
+
@queue = channel.queue(Promiscuous::Config.queue_name,
|
169
|
+
Promiscuous::Config.queue_options)
|
170
|
+
|
171
|
+
@error_queue = channel.queue(Promiscuous::Config.error_queue_name,
|
172
|
+
Promiscuous::Config.queue_options)
|
173
|
+
end
|
174
|
+
|
175
|
+
def delete_queues
|
176
|
+
[@error_queue, @queue].each { |queue| queue.try(:delete) }
|
177
|
+
end
|
178
|
+
|
136
179
|
class MetaData
|
137
180
|
def initialize(subscriber, delivery_info)
|
138
181
|
@subscriber = subscriber
|
@@ -143,8 +186,8 @@ class Promiscuous::AMQP::Bunny
|
|
143
186
|
@subscriber.ack_message(@delivery_info.delivery_tag)
|
144
187
|
end
|
145
188
|
|
146
|
-
def
|
147
|
-
@subscriber.
|
189
|
+
def nack
|
190
|
+
@subscriber.nack_message(@delivery_info.delivery_tag)
|
148
191
|
end
|
149
192
|
end
|
150
193
|
|
@@ -152,13 +195,8 @@ class Promiscuous::AMQP::Bunny
|
|
152
195
|
@lock.synchronize { @channel.ack(tag) } if @channel
|
153
196
|
end
|
154
197
|
|
155
|
-
def
|
156
|
-
|
157
|
-
# is a no-op.
|
158
|
-
|
159
|
-
# TODO: Even though the prefetch window is set to 10mil we should still
|
160
|
-
# check that the unacked messages doesn't exceed this limit and increase
|
161
|
-
# the prefetch window.
|
198
|
+
def nack_message(tag)
|
199
|
+
@lock.synchronize { @channel.nack(tag) } if @channel
|
162
200
|
end
|
163
201
|
|
164
202
|
def recover
|
@@ -20,7 +20,7 @@ class Promiscuous::AMQP::Fake
|
|
20
20
|
end
|
21
21
|
|
22
22
|
def publish(options={})
|
23
|
-
Promiscuous.debug "[publish (fake)] #{options[:exchange]
|
23
|
+
Promiscuous.debug "[publish (fake)] #{options[:exchange] || "default"}/#{options[:key]} #{options[:payload]}"
|
24
24
|
@messages << options
|
25
25
|
options[:on_confirm].try(:call)
|
26
26
|
end
|
@@ -47,5 +47,8 @@ class Promiscuous::AMQP::Fake
|
|
47
47
|
|
48
48
|
def disconnect
|
49
49
|
end
|
50
|
+
|
51
|
+
def delete_queues
|
52
|
+
end
|
50
53
|
end
|
51
54
|
end
|
data/lib/promiscuous/cli.rb
CHANGED
@@ -55,7 +55,7 @@ class Promiscuous::CLI
|
|
55
55
|
bar = ProgressBar.create(:format => '%t |%b>%i| %c/%C %e', :title => title, :total => criteria.count)
|
56
56
|
criteria.each do |doc|
|
57
57
|
break if @stop
|
58
|
-
doc.promiscuous.sync
|
58
|
+
doc.promiscuous.sync(Promiscuous::Config.sync_all_routing)
|
59
59
|
bar.increment
|
60
60
|
end
|
61
61
|
end
|
data/lib/promiscuous/config.rb
CHANGED
@@ -1,12 +1,13 @@
|
|
1
1
|
module Promiscuous::Config
|
2
2
|
mattr_accessor :app, :backend, :amqp_url,
|
3
3
|
:publisher_amqp_url, :subscriber_amqp_url, :publisher_exchange,
|
4
|
-
:subscriber_exchanges, :queue_name, :queue_options,
|
5
|
-
:redis_url, :redis_stats_url, :stats_interval,
|
6
|
-
:socket_timeout, :heartbeat,
|
4
|
+
:subscriber_exchanges, :sync_exchange, :queue_name, :queue_options,
|
5
|
+
:redis_url, :redis_stats_url, :stats_interval, :error_queue_name,
|
6
|
+
:socket_timeout, :heartbeat, :sync_all_routing, :rabbit_mgmt_url,
|
7
7
|
:prefetch, :recovery_timeout, :recovery_interval, :logger, :subscriber_threads,
|
8
|
-
:version_field, :error_notifier, :transport_collection,
|
9
|
-
:on_stats, :max_retries, :generation, :destroy_timeout, :destroy_check_interval
|
8
|
+
:version_field, :error_notifier, :transport_collection, :queue_policy, :test_mode,
|
9
|
+
:on_stats, :max_retries, :generation, :destroy_timeout, :destroy_check_interval,
|
10
|
+
:error_exchange, :error_routing, :retry_routing, :error_ttl, :transport_persistence
|
10
11
|
|
11
12
|
def self.backend=(value)
|
12
13
|
@@backend = value
|
@@ -31,18 +32,35 @@ module Promiscuous::Config
|
|
31
32
|
end
|
32
33
|
end
|
33
34
|
|
35
|
+
def self.best_transport_persistence
|
36
|
+
if defined?(Mongoid::Document)
|
37
|
+
self.transport_persistence = :mongoid
|
38
|
+
elsif defined?(ActiveRecord::Base)
|
39
|
+
self.transport_persistence = :active_record
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
34
43
|
def self._configure(&block)
|
35
44
|
block.call(self) if block
|
36
45
|
|
37
46
|
self.app ||= Rails.application.class.parent_name.underscore rescue nil if defined?(Rails)
|
38
47
|
self.backend ||= best_amqp_backend
|
39
48
|
self.amqp_url ||= 'amqp://guest:guest@localhost:5672'
|
49
|
+
self.rabbit_mgmt_url ||= 'http://guest:guest@localhost:15672'
|
40
50
|
self.publisher_amqp_url ||= self.amqp_url
|
41
51
|
self.subscriber_amqp_url ||= self.amqp_url
|
42
|
-
self.publisher_exchange ||=
|
43
|
-
self.
|
44
|
-
self.
|
45
|
-
self.
|
52
|
+
self.publisher_exchange ||= 'promiscuous'
|
53
|
+
self.sync_exchange ||= 'promiscuous.sync'
|
54
|
+
self.subscriber_exchanges ||= [self.publisher_exchange]
|
55
|
+
self.sync_all_routing ||= :__all__
|
56
|
+
self.queue_name ||= "#{self.app}.subscriber"
|
57
|
+
self.error_exchange ||= "#{self.app}.error"
|
58
|
+
self.error_queue_name ||= "#{self.app}.error"
|
59
|
+
self.error_routing ||= :__error__
|
60
|
+
self.retry_routing ||= :__retry__
|
61
|
+
self.error_ttl ||= 30000
|
62
|
+
self.queue_policy ||= { 'ha-mode' => 'all' }
|
63
|
+
self.queue_options ||= { :durable => true }
|
46
64
|
self.redis_url ||= 'redis://localhost/'
|
47
65
|
# TODO self.redis_slave_url ||= nil
|
48
66
|
self.redis_stats_url ||= self.redis_url
|
@@ -62,6 +80,16 @@ module Promiscuous::Config
|
|
62
80
|
self.generation ||= 0
|
63
81
|
self.destroy_timeout ||= 1.hour
|
64
82
|
self.destroy_check_interval ||= 10.minutes
|
83
|
+
self.transport_persistence ||= best_transport_persistence
|
84
|
+
self.test_mode = set_test_mode
|
85
|
+
end
|
86
|
+
|
87
|
+
def self.set_test_mode
|
88
|
+
if self.test_mode.nil?
|
89
|
+
defined?(Rails) ? Rails.env.test? ? true : false : false
|
90
|
+
else
|
91
|
+
self.test_mode
|
92
|
+
end
|
65
93
|
end
|
66
94
|
|
67
95
|
def self.configure(&block)
|
@@ -40,11 +40,15 @@ module Promiscuous::Publisher::Model::Base
|
|
40
40
|
@instance.id
|
41
41
|
end
|
42
42
|
|
43
|
-
def sync(
|
43
|
+
def sync(target)
|
44
44
|
raise "Model cannot be dirty (have changes) when syncing" if @instance.changed?
|
45
|
+
raise "Model has to be reloaded if it was saved" if @instance.previous_changes.present?
|
45
46
|
|
46
47
|
# We can use the ephemeral because both are mongoid and ephemerals are atomic operations.
|
47
|
-
Promiscuous::Publisher::Operation::Ephemeral.new(:instance => @instance,
|
48
|
+
Promiscuous::Publisher::Operation::Ephemeral.new(:instance => @instance,
|
49
|
+
:operation => :update,
|
50
|
+
:routing => target,
|
51
|
+
:exchange => Promiscuous::Config.sync_exchange).execute
|
48
52
|
end
|
49
53
|
|
50
54
|
end
|
@@ -23,7 +23,7 @@ module Promiscuous::Publisher::Model::Mongoid
|
|
23
23
|
class PromiscuousMethods
|
24
24
|
include Promiscuous::Publisher::Model::Base::PromiscuousMethodsBase
|
25
25
|
|
26
|
-
def sync(
|
26
|
+
def sync(target)
|
27
27
|
raise "Use promiscuous.sync on the parent instance" if @instance.embedded?
|
28
28
|
super
|
29
29
|
end
|
@@ -59,6 +59,11 @@ class ActiveRecord::Base
|
|
59
59
|
with_promiscuous_transaction_context { |tx| tx.commit }
|
60
60
|
end
|
61
61
|
|
62
|
+
def supports_returning_statments?
|
63
|
+
@supports_returning_statments ||= ["ActiveRecord::ConnectionAdapters::PostgreSQLAdapter",
|
64
|
+
"ActiveRecord::ConnectionAdapters::OracleEnhancedAdapter"].include?(self.class.name)
|
65
|
+
end
|
66
|
+
|
62
67
|
alias_method :insert_without_promiscuous, :insert
|
63
68
|
alias_method :update_without_promiscuous, :update
|
64
69
|
alias_method :delete_without_promiscuous, :delete
|
@@ -152,10 +157,21 @@ class ActiveRecord::Base
|
|
152
157
|
def db_operation_and_select
|
153
158
|
# XXX This is only supported by Postgres and should be in the postgres driver
|
154
159
|
@connection.transaction do
|
155
|
-
@connection.
|
156
|
-
@
|
157
|
-
|
158
|
-
|
160
|
+
if @connection.supports_returning_statments?
|
161
|
+
@connection.exec_insert("#{@connection.to_sql(@arel, @binds)} RETURNING *", @operation_name, @binds).tap do |result|
|
162
|
+
@instances = result.map do |row|
|
163
|
+
instance = model.instantiate(row)
|
164
|
+
instance
|
165
|
+
end
|
166
|
+
end
|
167
|
+
else
|
168
|
+
@connection.exec_insert("#{@connection.to_sql(@arel, @binds)}", @operation_name, @binds)
|
169
|
+
|
170
|
+
id = @binds.select { |k,v| k.name == 'id' }.first.last rescue nil
|
171
|
+
id ||= @connection.instance_eval { @connection.last_id }
|
172
|
+
id.tap do |last_id|
|
173
|
+
result = @connection.exec_query("SELECT * FROM #{model.table_name} WHERE #{@pk} = #{last_id}")
|
174
|
+
@instances = result.map { |row| model.instantiate(row) }
|
159
175
|
end
|
160
176
|
end
|
161
177
|
end
|
@@ -168,6 +184,7 @@ class ActiveRecord::Base
|
|
168
184
|
def initialize(arel, name, binds, options={})
|
169
185
|
super
|
170
186
|
@operation = :update
|
187
|
+
return if Promiscuous.disabled?
|
171
188
|
raise unless @arel.is_a?(Arel::UpdateManager)
|
172
189
|
end
|
173
190
|
|
@@ -191,16 +208,31 @@ class ActiveRecord::Base
|
|
191
208
|
(updated_fields_in_query.keys & model.published_db_fields).present?
|
192
209
|
end
|
193
210
|
|
211
|
+
def sql_select_statment
|
212
|
+
arel = @arel.dup
|
213
|
+
arel.instance_eval { @ast = @ast.dup }
|
214
|
+
arel.ast.values = []
|
215
|
+
arel.to_sql.sub(/^UPDATE /, 'SELECT * FROM ')
|
216
|
+
end
|
217
|
+
|
194
218
|
def db_operation_and_select
|
195
219
|
# TODO this should be in the postgres driver (to also leverage the cache)
|
196
|
-
@arel.ast.values << Arel::Nodes::SqlLiteral.new("
|
197
|
-
|
198
|
-
@connection.
|
199
|
-
@
|
200
|
-
|
220
|
+
@arel.ast.values << Arel::Nodes::SqlLiteral.new("#{Promiscuous::Config.version_field} = COALESCE(#{Promiscuous::Config.version_field}, 0) + 1")
|
221
|
+
|
222
|
+
if @connection.supports_returning_statments?
|
223
|
+
@connection.exec_query("#{@connection.to_sql(@arel, @binds)} RETURNING *", @operation_name, @binds).tap do |result|
|
224
|
+
@instances = result.map { |row| model.instantiate(row) }
|
225
|
+
end.rows.size
|
226
|
+
else
|
227
|
+
@connection.exec_update(@connection.to_sql(@arel, @binds), @operation_name, @binds).tap do
|
228
|
+
result = @connection.exec_query(sql_select_statment, @operation_name)
|
229
|
+
@instances = result.map { |row| model.instantiate(row) }
|
230
|
+
end
|
231
|
+
end
|
201
232
|
end
|
202
233
|
|
203
234
|
def execute(&db_operation)
|
235
|
+
return db_operation.call if Promiscuous.disabled?
|
204
236
|
return db_operation.call unless model
|
205
237
|
return db_operation.call unless any_published_field_changed?
|
206
238
|
super
|
@@ -214,11 +246,20 @@ class ActiveRecord::Base
|
|
214
246
|
raise unless @arel.is_a?(Arel::DeleteManager)
|
215
247
|
end
|
216
248
|
|
249
|
+
def sql_select_statment
|
250
|
+
@connection.to_sql(@arel.dup, @binds.dup).sub(/^DELETE /, 'SELECT * ')
|
251
|
+
end
|
252
|
+
|
217
253
|
def db_operation_and_select
|
218
|
-
|
219
|
-
|
254
|
+
if @connection.supports_returning_statments?
|
255
|
+
@connection.exec_query("#{@connection.to_sql(@arel, @binds)} RETURNING *", @operation_name, @binds).tap do |result|
|
256
|
+
@instances = result.map { |row| model.instantiate(row) }
|
257
|
+
end.rows.size
|
258
|
+
else
|
259
|
+
result = @connection.exec_query(sql_select_statment, @operation_name, @binds)
|
220
260
|
@instances = result.map { |row| model.instantiate(row) }
|
221
|
-
|
261
|
+
@connection.exec_delete(@connection.to_sql(@arel, @binds), @operation_name, @binds)
|
262
|
+
end
|
222
263
|
end
|
223
264
|
end
|
224
265
|
|
@@ -1,5 +1,5 @@
|
|
1
1
|
class Promiscuous::Publisher::Operation::Base
|
2
|
-
attr_accessor :operation, :recovering
|
2
|
+
attr_accessor :operation, :recovering, :routing, :exchange
|
3
3
|
|
4
4
|
def initialize(options={})
|
5
5
|
@operation = options[:operation]
|
@@ -44,8 +44,8 @@ class Promiscuous::Publisher::Operation::Base
|
|
44
44
|
"Unknown database operation"
|
45
45
|
end
|
46
46
|
|
47
|
-
def create_transport_batch(operations)
|
48
|
-
Promiscuous::Publisher::Transport::Batch.new.tap do |batch|
|
47
|
+
def create_transport_batch(operations, options={})
|
48
|
+
Promiscuous::Publisher::Transport::Batch.new(options).tap do |batch|
|
49
49
|
operations.map do |operation|
|
50
50
|
batch.add operation.operation, operation.instances
|
51
51
|
end
|
@@ -2,6 +2,8 @@ class Promiscuous::Publisher::Operation::Ephemeral < Promiscuous::Publisher::Ope
|
|
2
2
|
def initialize(options={})
|
3
3
|
super
|
4
4
|
@instance = options[:instance]
|
5
|
+
@routing = options[:routing]
|
6
|
+
@exchange = options[:exchange]
|
5
7
|
end
|
6
8
|
|
7
9
|
def instances
|
@@ -9,7 +11,7 @@ class Promiscuous::Publisher::Operation::Ephemeral < Promiscuous::Publisher::Ope
|
|
9
11
|
end
|
10
12
|
|
11
13
|
def execute_instrumented(query)
|
12
|
-
create_transport_batch([self]).publish(true)
|
14
|
+
create_transport_batch([self], :exchange => @exchange, :routing => @routing).publish(true)
|
13
15
|
end
|
14
16
|
|
15
17
|
def increment_version_in_document
|
@@ -2,13 +2,20 @@ class Promiscuous::Publisher::Transport
|
|
2
2
|
extend Promiscuous::Autoload
|
3
3
|
autoload :Batch, :Worker, :Persistence
|
4
4
|
|
5
|
-
|
5
|
+
def self.persistence
|
6
|
+
unless @persistence_key == Promiscuous::Config.transport_persistence
|
7
|
+
fix_inflections
|
8
|
+
@persistence = Persistence.const_get(Promiscuous::Config.transport_persistence.to_s.classify).new
|
9
|
+
@persistence_key = Promiscuous::Config.transport_persistence
|
10
|
+
end
|
11
|
+
@persistence
|
12
|
+
end
|
13
|
+
|
14
|
+
private
|
6
15
|
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
else
|
12
|
-
raise "Either Mongoid or ActiveRecord support required"
|
16
|
+
def self.fix_inflections
|
17
|
+
ActiveSupport::Inflector.inflections do |inflect|
|
18
|
+
inflect.uncountable "redis"
|
19
|
+
end
|
13
20
|
end
|
14
21
|
end
|
@@ -1,25 +1,25 @@
|
|
1
1
|
class Promiscuous::Publisher::Transport::Batch
|
2
|
-
attr_accessor :operations, :payload_attributes, :timestamp, :id
|
2
|
+
attr_accessor :operations, :payload_attributes, :timestamp, :id, :exchange, :routing
|
3
3
|
|
4
4
|
SERIALIZER = MultiJson
|
5
5
|
|
6
6
|
def self.load(id, dump)
|
7
|
-
data = SERIALIZER.load(dump)
|
7
|
+
data = SERIALIZER.load(dump).with_indifferent_access
|
8
8
|
|
9
|
-
batch = self.new
|
9
|
+
batch = self.new(data)
|
10
10
|
batch.id = id
|
11
|
-
batch.timestamp = data['timestamp']
|
12
|
-
batch.payload_attributes = data['payload_attributes']
|
13
11
|
|
14
12
|
data['operations'].each { |op_data| batch.operations << Operation.load(op_data) }
|
15
13
|
|
16
14
|
batch
|
17
15
|
end
|
18
16
|
|
19
|
-
def initialize
|
17
|
+
def initialize(options={})
|
20
18
|
self.operations = []
|
21
|
-
self.payload_attributes = {}
|
22
|
-
self.
|
19
|
+
self.payload_attributes = options[:payload_attributes] || {}
|
20
|
+
self.exchange = options[:exchange] || Promiscuous::Config.publisher_exchange
|
21
|
+
self.routing = options[:routing] || Promiscuous::Config.sync_all_routing
|
22
|
+
self.timestamp = options[:timestamp] || Time.now
|
23
23
|
end
|
24
24
|
|
25
25
|
def add(type, instances)
|
@@ -39,7 +39,9 @@ class Promiscuous::Publisher::Transport::Batch
|
|
39
39
|
|
40
40
|
begin
|
41
41
|
if self.operations.present?
|
42
|
-
Promiscuous::AMQP.publish(:
|
42
|
+
Promiscuous::AMQP.publish(:exchange => self.exchange,
|
43
|
+
:key => self.routing.to_s,
|
44
|
+
:payload => self.payload,
|
43
45
|
:on_confirm => method(:on_rabbitmq_confirm))
|
44
46
|
else
|
45
47
|
on_rabbitmq_confirm
|
@@ -79,7 +81,9 @@ class Promiscuous::Publisher::Transport::Batch
|
|
79
81
|
{
|
80
82
|
:operations => self.operations.map(&:dump),
|
81
83
|
:payload_attributes => self.payload_attributes,
|
82
|
-
:timestamp => self.timestamp
|
84
|
+
:timestamp => self.timestamp,
|
85
|
+
:exchange => self.exchange,
|
86
|
+
:routing => self.routing
|
83
87
|
})
|
84
88
|
end
|
85
89
|
|
@@ -2,19 +2,17 @@ class Promiscuous::Publisher::Transport::Persistence::ActiveRecord
|
|
2
2
|
def save(batch)
|
3
3
|
check_schema
|
4
4
|
|
5
|
-
q = "INSERT INTO #{table} (
|
6
|
-
"VALUES ('#{batch.dump}')
|
5
|
+
q = "INSERT INTO #{table} (batch) " +
|
6
|
+
"VALUES ('#{batch.dump}')"
|
7
7
|
|
8
|
-
|
9
|
-
|
10
|
-
batch.id = result.rows.first.first.to_i
|
8
|
+
batch.id = connection.insert_sql(q, 'Promiscuous Recovery Save')
|
11
9
|
end
|
12
10
|
|
13
11
|
def expired
|
14
12
|
check_schema
|
15
13
|
|
16
14
|
q = "SELECT id, p.batch FROM #{table} p " +
|
17
|
-
"WHERE at <
|
15
|
+
"WHERE at < CURRENT_TIMESTAMP - INTERVAL '#{Promiscuous::Config.recovery_timeout}' second"
|
18
16
|
|
19
17
|
connection.exec_query(q, 'Promiscuous Recovery Expired').rows
|
20
18
|
end
|
@@ -24,7 +22,7 @@ class Promiscuous::Publisher::Transport::Persistence::ActiveRecord
|
|
24
22
|
|
25
23
|
q = "DELETE FROM #{table} WHERE id = #{batch.id}"
|
26
24
|
|
27
|
-
connection.
|
25
|
+
connection.exec_delete(q, 'Promiscuous Recovery Delete', [])
|
28
26
|
end
|
29
27
|
|
30
28
|
private
|
@@ -36,8 +34,8 @@ class Promiscuous::Publisher::Transport::Persistence::ActiveRecord
|
|
36
34
|
raise <<-help
|
37
35
|
Promiscuous requires the following migration to be run:
|
38
36
|
create_table :_promiscuous do |t|
|
39
|
-
t.
|
40
|
-
t.timestamp :at,
|
37
|
+
t.text :batch
|
38
|
+
t.timestamp :at, 'TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP'
|
41
39
|
end
|
42
40
|
help
|
43
41
|
end
|
@@ -0,0 +1,34 @@
|
|
1
|
+
class Promiscuous::Publisher::Transport::Persistence::Redis
|
2
|
+
def save(batch)
|
3
|
+
batch.id = SecureRandom.uuid
|
4
|
+
|
5
|
+
redis.multi do
|
6
|
+
redis.zadd(key, Time.now.utc.to_i, batch.id)
|
7
|
+
redis.set(key(batch.id), batch.dump)
|
8
|
+
end
|
9
|
+
end
|
10
|
+
|
11
|
+
def expired
|
12
|
+
redis.zrangebyscore(key, 0, Promiscuous::Config.recovery_timeout.seconds.ago.utc.to_i).map do |batch_id|
|
13
|
+
[batch_id, redis.get(key(batch_id))]
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
def delete(batch)
|
18
|
+
redis.multi do
|
19
|
+
redis.zrem(key, batch.id)
|
20
|
+
redis.del(key(batch.id))
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
private
|
25
|
+
|
26
|
+
def redis
|
27
|
+
Promiscuous.ensure_connected
|
28
|
+
Promiscuous::Redis.connection
|
29
|
+
end
|
30
|
+
|
31
|
+
def key(id=nil)
|
32
|
+
Promiscuous::Key.new(:pub).join('transport').join(id)
|
33
|
+
end
|
34
|
+
end
|
@@ -0,0 +1,41 @@
|
|
1
|
+
require 'net/http'
|
2
|
+
require 'uri'
|
3
|
+
|
4
|
+
module Promiscuous::Rabbit
|
5
|
+
class Policy
|
6
|
+
def self.set(queue, attributes)
|
7
|
+
uri = uri_for(queue)
|
8
|
+
|
9
|
+
request = Net::HTTP::Put.new(uri.request_uri, http_headers)
|
10
|
+
request.basic_auth(uri.user, uri.password) if uri.user
|
11
|
+
request.body = MultiJson.dump(attributes)
|
12
|
+
|
13
|
+
http = Net::HTTP.new(uri.host, uri.port)
|
14
|
+
response = http.request(request)
|
15
|
+
|
16
|
+
raise "Unable to connect to Rabbit #{response.body}" unless response.is_a?(Net::HTTPSuccess)
|
17
|
+
end
|
18
|
+
|
19
|
+
def self.delete(queue)
|
20
|
+
uri = uri_for(queue)
|
21
|
+
|
22
|
+
request = Net::HTTP::Delete.new(uri.request_uri, http_headers)
|
23
|
+
request.basic_auth(uri.user, uri.password) if uri.user
|
24
|
+
|
25
|
+
http = Net::HTTP.new(uri.host, uri.port)
|
26
|
+
response = http.request(request)
|
27
|
+
|
28
|
+
raise "Unable to connect to Rabbit #{response.body}" unless response.is_a?(Net::HTTPSuccess) || response.is_a?(Net::HTTPNotFound)
|
29
|
+
end
|
30
|
+
|
31
|
+
private
|
32
|
+
|
33
|
+
def self.uri_for(queue)
|
34
|
+
URI.parse("#{Promiscuous::Config.rabbit_mgmt_url}/api/policies/%2f/#{queue}-retry")
|
35
|
+
end
|
36
|
+
|
37
|
+
def self.http_headers
|
38
|
+
{'Content-Type' =>'application/json'}
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
@@ -42,6 +42,15 @@ class Promiscuous::Subscriber::Message
|
|
42
42
|
Promiscuous::Config.error_notifier.call(e)
|
43
43
|
end
|
44
44
|
|
45
|
+
def nack
|
46
|
+
Promiscuous.debug "[receive][failed] #{payload}"
|
47
|
+
@metadata.try(:nack)
|
48
|
+
rescue Exception => e
|
49
|
+
# We don't care if we fail, the message will be redelivered at some point
|
50
|
+
Promiscuous.warn "[receive] Some exception happened, but it's okay: #{e}\n#{e.backtrace.join("\n")}"
|
51
|
+
Promiscuous::Config.error_notifier.call(e)
|
52
|
+
end
|
53
|
+
|
45
54
|
def process
|
46
55
|
Promiscuous::Subscriber::UnitOfWork.process(self)
|
47
56
|
rescue Exception => orig_e
|
@@ -39,15 +39,9 @@ class Promiscuous::Subscriber::UnitOfWork
|
|
39
39
|
begin
|
40
40
|
on_message
|
41
41
|
rescue Exception => e
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
Promiscuous.warn("[receive] #{e.message} #{@fail_count.ordinalize} retry: #{@message}")
|
46
|
-
sleep @fail_count ** 2
|
47
|
-
process_message
|
48
|
-
else
|
49
|
-
raise e
|
50
|
-
end
|
42
|
+
Promiscuous::Config.error_notifier.call(e)
|
43
|
+
message.nack
|
44
|
+
raise e if Promiscuous::Config.test_mode
|
51
45
|
end
|
52
46
|
end
|
53
47
|
|
@@ -9,11 +9,18 @@ class Promiscuous::Subscriber::Worker::Pump
|
|
9
9
|
def connect
|
10
10
|
options = {}
|
11
11
|
options[:bindings] = {}
|
12
|
-
# We need to subscribe to everything to keep
|
12
|
+
# We need to subscribe to everything to keep routing simple
|
13
13
|
Promiscuous::Config.subscriber_exchanges.each do |exchange|
|
14
14
|
options[:bindings][exchange] = ['*']
|
15
15
|
end
|
16
16
|
|
17
|
+
# Subscribe to the sync exchange to make syncing not require any command
|
18
|
+
# line ops
|
19
|
+
options[:bindings][Promiscuous::Config.sync_exchange] = [Promiscuous::Config.app, Promiscuous::Config.sync_all_routing]
|
20
|
+
|
21
|
+
# Subscribe to the error exchange but only to retries
|
22
|
+
options[:bindings][Promiscuous::Config.error_exchange] = [Promiscuous::Config.retry_routing]
|
23
|
+
|
17
24
|
subscribe(options, &method(:on_message))
|
18
25
|
end
|
19
26
|
|
data/lib/promiscuous/version.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: promiscuous
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.0.
|
4
|
+
version: 1.0.0.beta6
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Nicolas Viennot
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2014-
|
12
|
+
date: 2014-11-12 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: activesupport
|
@@ -160,17 +160,19 @@ executables:
|
|
160
160
|
extensions: []
|
161
161
|
extra_rdoc_files: []
|
162
162
|
files:
|
163
|
+
- bin/promiscuous
|
164
|
+
- lib/promiscuous.rb
|
165
|
+
- lib/promiscuous/amqp.rb
|
163
166
|
- lib/promiscuous/amqp/bunny.rb
|
164
167
|
- lib/promiscuous/amqp/fake.rb
|
165
168
|
- lib/promiscuous/amqp/file.rb
|
166
|
-
- lib/promiscuous/amqp/hot_bunnies.rb
|
167
169
|
- lib/promiscuous/amqp/null.rb
|
168
|
-
- lib/promiscuous/amqp.rb
|
169
170
|
- lib/promiscuous/autoload.rb
|
170
171
|
- lib/promiscuous/cli.rb
|
171
172
|
- lib/promiscuous/config.rb
|
172
173
|
- lib/promiscuous/convenience.rb
|
173
174
|
- lib/promiscuous/dsl.rb
|
175
|
+
- lib/promiscuous/error.rb
|
174
176
|
- lib/promiscuous/error/base.rb
|
175
177
|
- lib/promiscuous/error/connection.rb
|
176
178
|
- lib/promiscuous/error/lock_unavailable.rb
|
@@ -178,20 +180,21 @@ files:
|
|
178
180
|
- lib/promiscuous/error/publisher.rb
|
179
181
|
- lib/promiscuous/error/recovery.rb
|
180
182
|
- lib/promiscuous/error/subscriber.rb
|
181
|
-
- lib/promiscuous/error.rb
|
182
183
|
- lib/promiscuous/key.rb
|
183
184
|
- lib/promiscuous/loader.rb
|
184
185
|
- lib/promiscuous/mongoid.rb
|
186
|
+
- lib/promiscuous/publisher.rb
|
187
|
+
- lib/promiscuous/publisher/context.rb
|
185
188
|
- lib/promiscuous/publisher/context/base.rb
|
186
189
|
- lib/promiscuous/publisher/context/transaction.rb
|
187
|
-
- lib/promiscuous/publisher/context.rb
|
188
190
|
- lib/promiscuous/publisher/mock_generator.rb
|
191
|
+
- lib/promiscuous/publisher/model.rb
|
189
192
|
- lib/promiscuous/publisher/model/active_record.rb
|
190
193
|
- lib/promiscuous/publisher/model/base.rb
|
191
194
|
- lib/promiscuous/publisher/model/ephemeral.rb
|
192
195
|
- lib/promiscuous/publisher/model/mock.rb
|
193
196
|
- lib/promiscuous/publisher/model/mongoid.rb
|
194
|
-
- lib/promiscuous/publisher/
|
197
|
+
- lib/promiscuous/publisher/operation.rb
|
195
198
|
- lib/promiscuous/publisher/operation/active_record.rb
|
196
199
|
- lib/promiscuous/publisher/operation/atomic.rb
|
197
200
|
- lib/promiscuous/publisher/operation/base.rb
|
@@ -200,36 +203,34 @@ files:
|
|
200
203
|
- lib/promiscuous/publisher/operation/non_persistent.rb
|
201
204
|
- lib/promiscuous/publisher/operation/proxy_for_query.rb
|
202
205
|
- lib/promiscuous/publisher/operation/transaction.rb
|
203
|
-
- lib/promiscuous/publisher/
|
206
|
+
- lib/promiscuous/publisher/transport.rb
|
204
207
|
- lib/promiscuous/publisher/transport/batch.rb
|
208
|
+
- lib/promiscuous/publisher/transport/persistence.rb
|
205
209
|
- lib/promiscuous/publisher/transport/persistence/active_record.rb
|
206
210
|
- lib/promiscuous/publisher/transport/persistence/mongoid.rb
|
207
|
-
- lib/promiscuous/publisher/transport/persistence.rb
|
211
|
+
- lib/promiscuous/publisher/transport/persistence/redis.rb
|
208
212
|
- lib/promiscuous/publisher/transport/worker.rb
|
209
|
-
- lib/promiscuous/publisher/transport.rb
|
210
213
|
- lib/promiscuous/publisher/worker.rb
|
211
|
-
- lib/promiscuous/
|
214
|
+
- lib/promiscuous/rabbit.rb
|
212
215
|
- lib/promiscuous/railtie.rb
|
213
216
|
- lib/promiscuous/redis.rb
|
217
|
+
- lib/promiscuous/subscriber.rb
|
214
218
|
- lib/promiscuous/subscriber/message.rb
|
219
|
+
- lib/promiscuous/subscriber/model.rb
|
215
220
|
- lib/promiscuous/subscriber/model/active_record.rb
|
216
221
|
- lib/promiscuous/subscriber/model/base.rb
|
217
222
|
- lib/promiscuous/subscriber/model/mongoid.rb
|
218
223
|
- lib/promiscuous/subscriber/model/observer.rb
|
219
|
-
- lib/promiscuous/subscriber/model.rb
|
220
224
|
- lib/promiscuous/subscriber/operation.rb
|
221
225
|
- lib/promiscuous/subscriber/unit_of_work.rb
|
226
|
+
- lib/promiscuous/subscriber/worker.rb
|
222
227
|
- lib/promiscuous/subscriber/worker/eventual_destroyer.rb
|
223
228
|
- lib/promiscuous/subscriber/worker/pump.rb
|
224
229
|
- lib/promiscuous/subscriber/worker/recorder.rb
|
225
230
|
- lib/promiscuous/subscriber/worker/runner.rb
|
226
231
|
- lib/promiscuous/subscriber/worker/stats.rb
|
227
|
-
- lib/promiscuous/subscriber/worker.rb
|
228
|
-
- lib/promiscuous/subscriber.rb
|
229
232
|
- lib/promiscuous/timer.rb
|
230
233
|
- lib/promiscuous/version.rb
|
231
|
-
- lib/promiscuous.rb
|
232
|
-
- bin/promiscuous
|
233
234
|
homepage: http://github.com/crowdtap/promiscuous
|
234
235
|
licenses: []
|
235
236
|
metadata: {}
|
@@ -249,8 +250,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
249
250
|
version: 1.3.1
|
250
251
|
requirements: []
|
251
252
|
rubyforge_project:
|
252
|
-
rubygems_version: 2.
|
253
|
+
rubygems_version: 2.2.0
|
253
254
|
signing_key:
|
254
255
|
specification_version: 4
|
255
256
|
summary: Replicate models across applications
|
256
257
|
test_files: []
|
258
|
+
has_rdoc: false
|
@@ -1,66 +0,0 @@
|
|
1
|
-
class Promiscuous::AMQP::HotBunnies < Promiscuous::AMQP::Bunny
|
2
|
-
attr_accessor :connection
|
3
|
-
|
4
|
-
def initialize_driver
|
5
|
-
require 'hot_bunnies'
|
6
|
-
end
|
7
|
-
|
8
|
-
# TODO auto reconnect
|
9
|
-
|
10
|
-
def raw_new_connection(options={})
|
11
|
-
::HotBunnies.connect(:uri => options[:url],
|
12
|
-
:heartbeat_interval => Promiscuous::Config.heartbeat,
|
13
|
-
:connection_timeout => Promiscuous::Config.socket_timeout)
|
14
|
-
end
|
15
|
-
|
16
|
-
def raw_confirm_select(channel, &callback)
|
17
|
-
channel.add_confirm_listener(&callback)
|
18
|
-
channel.confirm_select
|
19
|
-
end
|
20
|
-
|
21
|
-
def raw_publish(options={})
|
22
|
-
options[:exchange].publish(options[:payload], :routing_key => options[:key], :persistent => true)
|
23
|
-
end
|
24
|
-
|
25
|
-
def disconnect
|
26
|
-
@connection_lock.synchronize do
|
27
|
-
return unless connected?
|
28
|
-
@channel.close rescue nil
|
29
|
-
@connection.close rescue nil
|
30
|
-
@connection = @channel = nil
|
31
|
-
end
|
32
|
-
end
|
33
|
-
|
34
|
-
def connected?
|
35
|
-
!!@connection.try(:is_open)
|
36
|
-
end
|
37
|
-
|
38
|
-
module Subscriber
|
39
|
-
include Promiscuous::AMQP::Bunny::Subscriber
|
40
|
-
|
41
|
-
def subscribe_queue(queue, &block)
|
42
|
-
queue.subscribe(:ack => true, :blocking => false) do |metadata, payload|
|
43
|
-
block.call(MetaData.new(self, metadata), payload)
|
44
|
-
end
|
45
|
-
end
|
46
|
-
|
47
|
-
class MetaData < Promiscuous::AMQP::Bunny::Subscriber::MetaData
|
48
|
-
def initialize(subscriber, metadata)
|
49
|
-
@subscriber = subscriber
|
50
|
-
@metadata = metadata
|
51
|
-
end
|
52
|
-
|
53
|
-
def ack
|
54
|
-
@metadata.ack
|
55
|
-
end
|
56
|
-
end
|
57
|
-
|
58
|
-
def disconnect
|
59
|
-
@lock.synchronize do
|
60
|
-
@channel = nil
|
61
|
-
@subscription.shutdown! rescue nil
|
62
|
-
@connection.close rescue nil
|
63
|
-
end
|
64
|
-
end
|
65
|
-
end
|
66
|
-
end
|