hutch 0.21.0-java → 0.25.0-java
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/.gitignore +3 -0
- data/.rspec +1 -0
- data/.travis.yml +11 -12
- data/.yardopts +5 -0
- data/CHANGELOG.md +118 -1
- data/Gemfile +15 -4
- data/Guardfile +13 -4
- data/README.md +274 -24
- data/Rakefile +8 -1
- data/hutch.gemspec +6 -7
- data/lib/hutch.rb +11 -8
- data/lib/hutch/adapters/march_hare.rb +1 -1
- data/lib/hutch/broker.rb +113 -110
- data/lib/hutch/cli.rb +42 -11
- data/lib/hutch/config.rb +209 -59
- data/lib/hutch/error_handlers.rb +1 -0
- data/lib/hutch/error_handlers/airbrake.rb +44 -16
- data/lib/hutch/error_handlers/base.rb +15 -0
- data/lib/hutch/error_handlers/honeybadger.rb +33 -18
- data/lib/hutch/error_handlers/logger.rb +12 -6
- data/lib/hutch/error_handlers/opbeat.rb +30 -0
- data/lib/hutch/error_handlers/sentry.rb +14 -6
- data/lib/hutch/logging.rb +5 -5
- data/lib/hutch/publisher.rb +75 -0
- data/lib/hutch/tracers.rb +1 -0
- data/lib/hutch/tracers/opbeat.rb +37 -0
- data/lib/hutch/version.rb +1 -1
- data/lib/hutch/waiter.rb +104 -0
- data/lib/hutch/worker.rb +50 -66
- data/lib/yard-settings/handler.rb +38 -0
- data/lib/yard-settings/yard-settings.rb +2 -0
- data/spec/hutch/broker_spec.rb +162 -77
- data/spec/hutch/cli_spec.rb +16 -3
- data/spec/hutch/config_spec.rb +83 -22
- data/spec/hutch/error_handlers/airbrake_spec.rb +25 -10
- data/spec/hutch/error_handlers/honeybadger_spec.rb +24 -2
- data/spec/hutch/error_handlers/logger_spec.rb +14 -1
- data/spec/hutch/error_handlers/opbeat_spec.rb +37 -0
- data/spec/hutch/error_handlers/sentry_spec.rb +18 -1
- data/spec/hutch/logger_spec.rb +12 -6
- data/spec/hutch/waiter_spec.rb +51 -0
- data/spec/hutch/worker_spec.rb +33 -4
- data/spec/spec_helper.rb +7 -5
- data/spec/tracers/opbeat_spec.rb +44 -0
- data/templates/default/class/html/settings.erb +0 -0
- data/templates/default/class/setup.rb +4 -0
- data/templates/default/fulldoc/html/css/hutch.css +13 -0
- data/templates/default/layout/html/setup.rb +7 -0
- data/templates/default/method_details/html/settings.erb +5 -0
- data/templates/default/method_details/setup.rb +4 -0
- data/templates/default/method_details/text/settings.erb +0 -0
- data/templates/default/module/html/settings.erb +40 -0
- data/templates/default/module/setup.rb +4 -0
- metadata +41 -38
data/Rakefile
CHANGED
@@ -11,4 +11,11 @@ RSpec::Core::RakeTask.new(:spec) do |t|
|
|
11
11
|
t.rspec_opts = %w(--color --format doc)
|
12
12
|
end
|
13
13
|
|
14
|
-
task :
|
14
|
+
task default: :spec
|
15
|
+
|
16
|
+
#
|
17
|
+
# Re-generate API docs
|
18
|
+
#
|
19
|
+
require 'yard'
|
20
|
+
require 'yard/rake/yardoc_task'
|
21
|
+
YARD::Rake::YardocTask.new
|
data/hutch.gemspec
CHANGED
@@ -3,22 +3,21 @@ require File.expand_path('../lib/hutch/version', __FILE__)
|
|
3
3
|
Gem::Specification.new do |gem|
|
4
4
|
if defined?(JRUBY_VERSION)
|
5
5
|
gem.platform = 'java'
|
6
|
-
gem.add_runtime_dependency 'march_hare', '>=
|
6
|
+
gem.add_runtime_dependency 'march_hare', '>= 3.0.0'
|
7
7
|
else
|
8
8
|
gem.platform = Gem::Platform::RUBY
|
9
|
-
gem.add_runtime_dependency 'bunny', '
|
9
|
+
gem.add_runtime_dependency 'bunny', '~> 2.9.0'
|
10
10
|
end
|
11
11
|
gem.add_runtime_dependency 'carrot-top', '~> 0.0.7'
|
12
|
-
gem.add_runtime_dependency 'multi_json', '~> 1.
|
13
|
-
gem.add_runtime_dependency 'activesupport', '>=
|
14
|
-
gem.add_development_dependency 'rspec', '~> 3.0'
|
15
|
-
gem.add_development_dependency 'simplecov', '~> 0.7.1'
|
12
|
+
gem.add_runtime_dependency 'multi_json', '~> 1.12'
|
13
|
+
gem.add_runtime_dependency 'activesupport', '>= 4.2', '< 6'
|
16
14
|
|
17
15
|
gem.name = 'hutch'
|
18
16
|
gem.summary = 'Easy inter-service communication using RabbitMQ.'
|
19
|
-
gem.description = 'Hutch is a Ruby library for enabling asynchronous '
|
17
|
+
gem.description = 'Hutch is a Ruby library for enabling asynchronous ' \
|
20
18
|
'inter-service communication using RabbitMQ.'
|
21
19
|
gem.version = Hutch::VERSION.dup
|
20
|
+
gem.required_ruby_version = '>= 2.2'
|
22
21
|
gem.authors = ['Harry Marr']
|
23
22
|
gem.email = ['developers@gocardless.com']
|
24
23
|
gem.homepage = 'https://github.com/gocardless/hutch'
|
data/lib/hutch.rb
CHANGED
@@ -14,7 +14,6 @@ require 'hutch/exceptions'
|
|
14
14
|
require 'hutch/tracers'
|
15
15
|
|
16
16
|
module Hutch
|
17
|
-
|
18
17
|
def self.register_consumer(consumer)
|
19
18
|
self.consumers << consumer
|
20
19
|
end
|
@@ -35,11 +34,16 @@ module Hutch
|
|
35
34
|
@global_properties ||= {}
|
36
35
|
end
|
37
36
|
|
37
|
+
# Connects to broker, if not yet connected.
|
38
|
+
#
|
39
|
+
# @param options [Hash] Connection options
|
40
|
+
# @param config [Hash] Configuration
|
41
|
+
# @option options [Boolean] :enable_http_api_use
|
38
42
|
def self.connect(options = {}, config = Hutch::Config)
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
+
return if connected?
|
44
|
+
|
45
|
+
@broker = Hutch::Broker.new(config)
|
46
|
+
@broker.connect(options)
|
43
47
|
end
|
44
48
|
|
45
49
|
def self.disconnect
|
@@ -50,10 +54,9 @@ module Hutch
|
|
50
54
|
@broker
|
51
55
|
end
|
52
56
|
|
57
|
+
# @return [Boolean]
|
53
58
|
def self.connected?
|
54
|
-
|
55
|
-
return false unless broker.connection
|
56
|
-
broker.connection.open?
|
59
|
+
broker && broker.connection && broker.connection.open?
|
57
60
|
end
|
58
61
|
|
59
62
|
def self.publish(*args)
|
data/lib/hutch/broker.rb
CHANGED
@@ -1,7 +1,9 @@
|
|
1
|
+
require 'active_support/core_ext/object/blank'
|
2
|
+
|
1
3
|
require 'carrot-top'
|
2
|
-
require 'securerandom'
|
3
4
|
require 'hutch/logging'
|
4
5
|
require 'hutch/exceptions'
|
6
|
+
require 'hutch/publisher'
|
5
7
|
|
6
8
|
module Hutch
|
7
9
|
class Broker
|
@@ -9,10 +11,38 @@ module Hutch
|
|
9
11
|
|
10
12
|
attr_accessor :connection, :channel, :exchange, :api_client
|
11
13
|
|
14
|
+
|
15
|
+
DEFAULT_AMQP_PORT =
|
16
|
+
case RUBY_ENGINE
|
17
|
+
when "jruby" then
|
18
|
+
com.rabbitmq.client.ConnectionFactory::DEFAULT_AMQP_PORT
|
19
|
+
when "ruby" then
|
20
|
+
AMQ::Protocol::DEFAULT_PORT
|
21
|
+
end
|
22
|
+
|
23
|
+
DEFAULT_AMQPS_PORT =
|
24
|
+
case RUBY_ENGINE
|
25
|
+
when "jruby" then
|
26
|
+
com.rabbitmq.client.ConnectionFactory::DEFAULT_AMQP_OVER_SSL_PORT
|
27
|
+
when "ruby" then
|
28
|
+
AMQ::Protocol::TLS_PORT
|
29
|
+
end
|
30
|
+
|
31
|
+
|
32
|
+
# @param config [nil,Hash] Configuration override
|
12
33
|
def initialize(config = nil)
|
13
34
|
@config = config || Hutch::Config
|
14
35
|
end
|
15
36
|
|
37
|
+
# Connect to broker
|
38
|
+
#
|
39
|
+
# @example
|
40
|
+
# Hutch::Broker.new.connect(enable_http_api_use: true) do
|
41
|
+
# # will disconnect after this block
|
42
|
+
# end
|
43
|
+
#
|
44
|
+
# @param [Hash] options The options to connect with
|
45
|
+
# @option options [Boolean] :enable_http_api_use
|
16
46
|
def connect(options = {})
|
17
47
|
@options = options
|
18
48
|
set_up_amqp_connection
|
@@ -41,42 +71,44 @@ module Hutch
|
|
41
71
|
def disconnect
|
42
72
|
@channel.close if @channel
|
43
73
|
@connection.close if @connection
|
44
|
-
@channel
|
74
|
+
@channel = nil
|
75
|
+
@connection = nil
|
76
|
+
@exchange = nil
|
77
|
+
@api_client = nil
|
45
78
|
end
|
46
79
|
|
47
|
-
# Connect to RabbitMQ via AMQP
|
48
|
-
#
|
49
|
-
# the
|
80
|
+
# Connect to RabbitMQ via AMQP
|
81
|
+
#
|
82
|
+
# This sets up the main connection and channel we use for talking to
|
83
|
+
# RabbitMQ. It also ensures the existence of the exchange we'll be using.
|
50
84
|
def set_up_amqp_connection
|
51
85
|
open_connection!
|
52
86
|
open_channel!
|
53
|
-
|
54
|
-
|
55
|
-
exchange_options = { durable: true }.merge @config[:mq_exchange_options]
|
56
|
-
logger.info "using topic exchange '#{exchange_name}'"
|
57
|
-
|
58
|
-
with_bunny_precondition_handler('exchange') do
|
59
|
-
@exchange = @channel.topic(exchange_name, exchange_options)
|
60
|
-
end
|
87
|
+
declare_exchange!
|
88
|
+
declare_publisher!
|
61
89
|
end
|
62
90
|
|
63
|
-
def open_connection
|
91
|
+
def open_connection
|
64
92
|
logger.info "connecting to rabbitmq (#{sanitized_uri})"
|
65
93
|
|
66
|
-
|
94
|
+
connection = Hutch::Adapter.new(connection_params)
|
67
95
|
|
68
96
|
with_bunny_connection_handler(sanitized_uri) do
|
69
|
-
|
97
|
+
connection.start
|
70
98
|
end
|
71
99
|
|
72
100
|
logger.info "connected to RabbitMQ at #{connection_params[:host]} as #{connection_params[:username]}"
|
73
|
-
|
101
|
+
connection
|
74
102
|
end
|
75
103
|
|
76
|
-
def
|
77
|
-
|
78
|
-
|
79
|
-
|
104
|
+
def open_connection!
|
105
|
+
@connection = open_connection
|
106
|
+
end
|
107
|
+
|
108
|
+
def open_channel
|
109
|
+
logger.info "opening rabbitmq channel with pool size #{consumer_pool_size}, abort on exception #{consumer_pool_abort_on_exception}"
|
110
|
+
connection.create_channel(nil, consumer_pool_size, consumer_pool_abort_on_exception).tap do |ch|
|
111
|
+
connection.prefetch_channel(ch, @config[:channel_prefetch])
|
80
112
|
if @config[:publisher_confirms] || @config[:force_publisher_confirms]
|
81
113
|
logger.info 'enabling publisher confirms'
|
82
114
|
ch.confirm_select
|
@@ -84,6 +116,28 @@ module Hutch
|
|
84
116
|
end
|
85
117
|
end
|
86
118
|
|
119
|
+
def open_channel!
|
120
|
+
@channel = open_channel
|
121
|
+
end
|
122
|
+
|
123
|
+
def declare_exchange(ch = channel)
|
124
|
+
exchange_name = @config[:mq_exchange]
|
125
|
+
exchange_options = { durable: true }.merge(@config[:mq_exchange_options])
|
126
|
+
logger.info "using topic exchange '#{exchange_name}'"
|
127
|
+
|
128
|
+
with_bunny_precondition_handler('exchange') do
|
129
|
+
ch.topic(exchange_name, exchange_options)
|
130
|
+
end
|
131
|
+
end
|
132
|
+
|
133
|
+
def declare_exchange!(*args)
|
134
|
+
@exchange = declare_exchange(*args)
|
135
|
+
end
|
136
|
+
|
137
|
+
def declare_publisher!
|
138
|
+
@publisher = Hutch::Publisher.new(connection, channel, exchange, @config)
|
139
|
+
end
|
140
|
+
|
87
141
|
# Set up the connection to the RabbitMQ management API. Unfortunately, this
|
88
142
|
# is necessary to do a few things that are impossible over AMQP. E.g.
|
89
143
|
# listing queues and bindings.
|
@@ -119,7 +173,7 @@ module Hutch
|
|
119
173
|
def queue(name, arguments = {})
|
120
174
|
with_bunny_precondition_handler('queue') do
|
121
175
|
namespace = @config[:namespace].to_s.downcase.gsub(/[^-_:\.\w]/, "")
|
122
|
-
name = name.prepend(namespace + ":")
|
176
|
+
name = name.prepend(namespace + ":") if namespace.present?
|
123
177
|
channel.queue(name, durable: true, arguments: arguments)
|
124
178
|
end
|
125
179
|
end
|
@@ -127,7 +181,7 @@ module Hutch
|
|
127
181
|
# Return a mapping of queue names to the routing keys they're bound to.
|
128
182
|
def bindings
|
129
183
|
results = Hash.new { |hash, key| hash[key] = [] }
|
130
|
-
|
184
|
+
api_client.bindings.each do |binding|
|
131
185
|
next if binding['destination'] == binding['routing_key']
|
132
186
|
next unless binding['source'] == @config[:mq_exchange]
|
133
187
|
next unless binding['vhost'] == @config[:mq_vhost]
|
@@ -136,37 +190,32 @@ module Hutch
|
|
136
190
|
results
|
137
191
|
end
|
138
192
|
|
193
|
+
# Find the existing bindings, and unbind any redundant bindings
|
194
|
+
def unbind_redundant_bindings(queue, routing_keys)
|
195
|
+
return unless http_api_use_enabled?
|
196
|
+
|
197
|
+
bindings.each do |dest, keys|
|
198
|
+
next unless dest == queue.name
|
199
|
+
keys.reject { |key| routing_keys.include?(key) }.each do |key|
|
200
|
+
logger.debug "removing redundant binding #{queue.name} <--> #{key}"
|
201
|
+
queue.unbind(exchange, routing_key: key)
|
202
|
+
end
|
203
|
+
end
|
204
|
+
end
|
205
|
+
|
139
206
|
# Bind a queue to the broker's exchange on the routing keys provided. Any
|
140
207
|
# existing bindings on the queue that aren't present in the array of
|
141
208
|
# routing keys will be unbound.
|
142
209
|
def bind_queue(queue, routing_keys)
|
143
|
-
|
144
|
-
# Find the existing bindings, and unbind any redundant bindings
|
145
|
-
queue_bindings = bindings.select { |dest, keys| dest == queue.name }
|
146
|
-
queue_bindings.each do |dest, keys|
|
147
|
-
keys.reject { |key| routing_keys.include?(key) }.each do |key|
|
148
|
-
logger.debug "removing redundant binding #{queue.name} <--> #{key}"
|
149
|
-
queue.unbind(@exchange, routing_key: key)
|
150
|
-
end
|
151
|
-
end
|
152
|
-
end
|
210
|
+
unbind_redundant_bindings(queue, routing_keys)
|
153
211
|
|
154
212
|
# Ensure all the desired bindings are present
|
155
213
|
routing_keys.each do |routing_key|
|
156
214
|
logger.debug "creating binding #{queue.name} <--> #{routing_key}"
|
157
|
-
queue.bind(
|
215
|
+
queue.bind(exchange, routing_key: routing_key)
|
158
216
|
end
|
159
217
|
end
|
160
218
|
|
161
|
-
# Each subscriber is run in a thread. This calls Thread#join on each of the
|
162
|
-
# subscriber threads.
|
163
|
-
def wait_on_threads(timeout)
|
164
|
-
# Thread#join returns nil when the timeout is hit. If any return nil,
|
165
|
-
# the threads didn't all join so we return false.
|
166
|
-
per_thread_timeout = timeout.to_f / work_pool_threads.length
|
167
|
-
work_pool_threads.none? { |thread| thread.join(per_thread_timeout).nil? }
|
168
|
-
end
|
169
|
-
|
170
219
|
def stop
|
171
220
|
if defined?(JRUBY_VERSION)
|
172
221
|
channel.close
|
@@ -181,78 +230,40 @@ module Hutch
|
|
181
230
|
end
|
182
231
|
|
183
232
|
def requeue(delivery_tag)
|
184
|
-
|
233
|
+
channel.reject(delivery_tag, true)
|
185
234
|
end
|
186
235
|
|
187
236
|
def reject(delivery_tag, requeue=false)
|
188
|
-
|
237
|
+
channel.reject(delivery_tag, requeue)
|
189
238
|
end
|
190
239
|
|
191
240
|
def ack(delivery_tag)
|
192
|
-
|
241
|
+
channel.ack(delivery_tag, false)
|
193
242
|
end
|
194
243
|
|
195
244
|
def nack(delivery_tag)
|
196
|
-
|
245
|
+
channel.nack(delivery_tag, false, false)
|
197
246
|
end
|
198
247
|
|
199
|
-
def publish(
|
200
|
-
|
201
|
-
|
202
|
-
serializer = options[:serializer] || @config[:serializer]
|
203
|
-
|
204
|
-
non_overridable_properties = {
|
205
|
-
routing_key: routing_key,
|
206
|
-
timestamp: @connection.current_timestamp,
|
207
|
-
content_type: serializer.content_type,
|
208
|
-
}
|
209
|
-
properties[:message_id] ||= generate_id
|
210
|
-
|
211
|
-
payload = serializer.encode(message)
|
212
|
-
logger.info {
|
213
|
-
spec =
|
214
|
-
if serializer.binary?
|
215
|
-
"#{payload.bytesize} bytes message"
|
216
|
-
else
|
217
|
-
"message '#{payload}'"
|
218
|
-
end
|
219
|
-
"publishing #{spec} to #{routing_key}"
|
220
|
-
}
|
221
|
-
|
222
|
-
response = @exchange.publish(payload, {persistent: true}.
|
223
|
-
merge(properties).
|
224
|
-
merge(global_properties).
|
225
|
-
merge(non_overridable_properties))
|
226
|
-
|
227
|
-
channel.wait_for_confirms if @config[:force_publisher_confirms]
|
228
|
-
response
|
248
|
+
def publish(*args)
|
249
|
+
@publisher.publish(*args)
|
229
250
|
end
|
230
251
|
|
231
252
|
def confirm_select(*args)
|
232
|
-
|
253
|
+
channel.confirm_select(*args)
|
233
254
|
end
|
234
255
|
|
235
256
|
def wait_for_confirms
|
236
|
-
|
257
|
+
channel.wait_for_confirms
|
237
258
|
end
|
238
259
|
|
260
|
+
# @return [Boolean] True if channel is set up to use publisher confirmations.
|
239
261
|
def using_publisher_confirmations?
|
240
|
-
|
262
|
+
channel.using_publisher_confirmations?
|
241
263
|
end
|
242
264
|
|
243
265
|
private
|
244
266
|
|
245
|
-
def raise_publish_error(reason, routing_key, message)
|
246
|
-
msg = "unable to publish - #{reason}. Message: #{JSON.dump(message)}, Routing key: #{routing_key}."
|
247
|
-
logger.error(msg)
|
248
|
-
raise PublishError, msg
|
249
|
-
end
|
250
|
-
|
251
|
-
def ensure_connection!(routing_key, message)
|
252
|
-
raise_publish_error('no connection to broker', routing_key, message) unless @connection
|
253
|
-
raise_publish_error('connection is closed', routing_key, message) unless @connection.open?
|
254
|
-
end
|
255
|
-
|
256
267
|
def api_config
|
257
268
|
@api_config ||= OpenStruct.new.tap do |config|
|
258
269
|
config.host = @config[:mq_api_host]
|
@@ -271,11 +282,7 @@ module Hutch
|
|
271
282
|
{}.tap do |params|
|
272
283
|
params[:host] = @config[:mq_host]
|
273
284
|
params[:port] = @config[:mq_port]
|
274
|
-
params[:vhost] =
|
275
|
-
@config[:mq_vhost]
|
276
|
-
else
|
277
|
-
Hutch::Adapter::DEFAULT_VHOST
|
278
|
-
end
|
285
|
+
params[:vhost] = @config[:mq_vhost].presence || Hutch::Adapter::DEFAULT_VHOST
|
279
286
|
params[:username] = @config[:mq_username]
|
280
287
|
params[:password] = @config[:mq_password]
|
281
288
|
params[:tls] = @config[:mq_tls]
|
@@ -299,17 +306,22 @@ module Hutch
|
|
299
306
|
end
|
300
307
|
|
301
308
|
def parse_uri
|
302
|
-
return
|
309
|
+
return if @config[:uri].blank?
|
303
310
|
|
304
311
|
u = URI.parse(@config[:uri])
|
305
312
|
|
313
|
+
@config[:mq_tls] = u.scheme == 'amqps'
|
306
314
|
@config[:mq_host] = u.host
|
307
|
-
@config[:mq_port] = u.port
|
315
|
+
@config[:mq_port] = u.port || default_mq_port
|
308
316
|
@config[:mq_vhost] = u.path.sub(/^\//, "")
|
309
317
|
@config[:mq_username] = u.user
|
310
318
|
@config[:mq_password] = u.password
|
311
319
|
end
|
312
320
|
|
321
|
+
def default_mq_port
|
322
|
+
@config[:mq_tls] ? DEFAULT_AMQPS_PORT : DEFAULT_AMQP_PORT
|
323
|
+
end
|
324
|
+
|
313
325
|
def sanitized_uri
|
314
326
|
p = connection_params
|
315
327
|
scheme = p[:tls] ? "amqps" : "amqp"
|
@@ -351,25 +363,16 @@ module Hutch
|
|
351
363
|
raise ConnectionError.new("couldn't connect to rabbitmq at #{uri}. Check your configuration, network connectivity and RabbitMQ logs.")
|
352
364
|
end
|
353
365
|
|
354
|
-
def work_pool_threads
|
355
|
-
channel_work_pool.threads || []
|
356
|
-
end
|
357
|
-
|
358
366
|
def channel_work_pool
|
359
|
-
|
367
|
+
channel.work_pool
|
360
368
|
end
|
361
369
|
|
362
370
|
def consumer_pool_size
|
363
371
|
@config[:consumer_pool_size]
|
364
372
|
end
|
365
373
|
|
366
|
-
def
|
367
|
-
|
374
|
+
def consumer_pool_abort_on_exception
|
375
|
+
@config[:consumer_pool_abort_on_exception]
|
368
376
|
end
|
369
|
-
|
370
|
-
def global_properties
|
371
|
-
Hutch.global_properties.respond_to?(:call) ? Hutch.global_properties.call : Hutch.global_properties
|
372
|
-
end
|
373
|
-
|
374
377
|
end
|
375
378
|
end
|