hutch 0.21.0-java → 0.25.0-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/.gitignore +3 -0
- data/.rspec +1 -0
- data/.travis.yml +11 -12
- data/.yardopts +5 -0
- data/CHANGELOG.md +118 -1
- data/Gemfile +15 -4
- data/Guardfile +13 -4
- data/README.md +274 -24
- data/Rakefile +8 -1
- data/hutch.gemspec +6 -7
- data/lib/hutch.rb +11 -8
- data/lib/hutch/adapters/march_hare.rb +1 -1
- data/lib/hutch/broker.rb +113 -110
- data/lib/hutch/cli.rb +42 -11
- data/lib/hutch/config.rb +209 -59
- data/lib/hutch/error_handlers.rb +1 -0
- data/lib/hutch/error_handlers/airbrake.rb +44 -16
- data/lib/hutch/error_handlers/base.rb +15 -0
- data/lib/hutch/error_handlers/honeybadger.rb +33 -18
- data/lib/hutch/error_handlers/logger.rb +12 -6
- data/lib/hutch/error_handlers/opbeat.rb +30 -0
- data/lib/hutch/error_handlers/sentry.rb +14 -6
- data/lib/hutch/logging.rb +5 -5
- data/lib/hutch/publisher.rb +75 -0
- data/lib/hutch/tracers.rb +1 -0
- data/lib/hutch/tracers/opbeat.rb +37 -0
- data/lib/hutch/version.rb +1 -1
- data/lib/hutch/waiter.rb +104 -0
- data/lib/hutch/worker.rb +50 -66
- data/lib/yard-settings/handler.rb +38 -0
- data/lib/yard-settings/yard-settings.rb +2 -0
- data/spec/hutch/broker_spec.rb +162 -77
- data/spec/hutch/cli_spec.rb +16 -3
- data/spec/hutch/config_spec.rb +83 -22
- data/spec/hutch/error_handlers/airbrake_spec.rb +25 -10
- data/spec/hutch/error_handlers/honeybadger_spec.rb +24 -2
- data/spec/hutch/error_handlers/logger_spec.rb +14 -1
- data/spec/hutch/error_handlers/opbeat_spec.rb +37 -0
- data/spec/hutch/error_handlers/sentry_spec.rb +18 -1
- data/spec/hutch/logger_spec.rb +12 -6
- data/spec/hutch/waiter_spec.rb +51 -0
- data/spec/hutch/worker_spec.rb +33 -4
- data/spec/spec_helper.rb +7 -5
- data/spec/tracers/opbeat_spec.rb +44 -0
- data/templates/default/class/html/settings.erb +0 -0
- data/templates/default/class/setup.rb +4 -0
- data/templates/default/fulldoc/html/css/hutch.css +13 -0
- data/templates/default/layout/html/setup.rb +7 -0
- data/templates/default/method_details/html/settings.erb +5 -0
- data/templates/default/method_details/setup.rb +4 -0
- data/templates/default/method_details/text/settings.erb +0 -0
- data/templates/default/module/html/settings.erb +40 -0
- data/templates/default/module/setup.rb +4 -0
- metadata +41 -38
data/Rakefile
CHANGED
@@ -11,4 +11,11 @@ RSpec::Core::RakeTask.new(:spec) do |t|
|
|
11
11
|
t.rspec_opts = %w(--color --format doc)
|
12
12
|
end
|
13
13
|
|
14
|
-
task :
|
14
|
+
task default: :spec
|
15
|
+
|
16
|
+
#
|
17
|
+
# Re-generate API docs
|
18
|
+
#
|
19
|
+
require 'yard'
|
20
|
+
require 'yard/rake/yardoc_task'
|
21
|
+
YARD::Rake::YardocTask.new
|
data/hutch.gemspec
CHANGED
@@ -3,22 +3,21 @@ require File.expand_path('../lib/hutch/version', __FILE__)
|
|
3
3
|
Gem::Specification.new do |gem|
|
4
4
|
if defined?(JRUBY_VERSION)
|
5
5
|
gem.platform = 'java'
|
6
|
-
gem.add_runtime_dependency 'march_hare', '>=
|
6
|
+
gem.add_runtime_dependency 'march_hare', '>= 3.0.0'
|
7
7
|
else
|
8
8
|
gem.platform = Gem::Platform::RUBY
|
9
|
-
gem.add_runtime_dependency 'bunny', '
|
9
|
+
gem.add_runtime_dependency 'bunny', '~> 2.9.0'
|
10
10
|
end
|
11
11
|
gem.add_runtime_dependency 'carrot-top', '~> 0.0.7'
|
12
|
-
gem.add_runtime_dependency 'multi_json', '~> 1.
|
13
|
-
gem.add_runtime_dependency 'activesupport', '>=
|
14
|
-
gem.add_development_dependency 'rspec', '~> 3.0'
|
15
|
-
gem.add_development_dependency 'simplecov', '~> 0.7.1'
|
12
|
+
gem.add_runtime_dependency 'multi_json', '~> 1.12'
|
13
|
+
gem.add_runtime_dependency 'activesupport', '>= 4.2', '< 6'
|
16
14
|
|
17
15
|
gem.name = 'hutch'
|
18
16
|
gem.summary = 'Easy inter-service communication using RabbitMQ.'
|
19
|
-
gem.description = 'Hutch is a Ruby library for enabling asynchronous '
|
17
|
+
gem.description = 'Hutch is a Ruby library for enabling asynchronous ' \
|
20
18
|
'inter-service communication using RabbitMQ.'
|
21
19
|
gem.version = Hutch::VERSION.dup
|
20
|
+
gem.required_ruby_version = '>= 2.2'
|
22
21
|
gem.authors = ['Harry Marr']
|
23
22
|
gem.email = ['developers@gocardless.com']
|
24
23
|
gem.homepage = 'https://github.com/gocardless/hutch'
|
data/lib/hutch.rb
CHANGED
@@ -14,7 +14,6 @@ require 'hutch/exceptions'
|
|
14
14
|
require 'hutch/tracers'
|
15
15
|
|
16
16
|
module Hutch
|
17
|
-
|
18
17
|
def self.register_consumer(consumer)
|
19
18
|
self.consumers << consumer
|
20
19
|
end
|
@@ -35,11 +34,16 @@ module Hutch
|
|
35
34
|
@global_properties ||= {}
|
36
35
|
end
|
37
36
|
|
37
|
+
# Connects to broker, if not yet connected.
|
38
|
+
#
|
39
|
+
# @param options [Hash] Connection options
|
40
|
+
# @param config [Hash] Configuration
|
41
|
+
# @option options [Boolean] :enable_http_api_use
|
38
42
|
def self.connect(options = {}, config = Hutch::Config)
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
+
return if connected?
|
44
|
+
|
45
|
+
@broker = Hutch::Broker.new(config)
|
46
|
+
@broker.connect(options)
|
43
47
|
end
|
44
48
|
|
45
49
|
def self.disconnect
|
@@ -50,10 +54,9 @@ module Hutch
|
|
50
54
|
@broker
|
51
55
|
end
|
52
56
|
|
57
|
+
# @return [Boolean]
|
53
58
|
def self.connected?
|
54
|
-
|
55
|
-
return false unless broker.connection
|
56
|
-
broker.connection.open?
|
59
|
+
broker && broker.connection && broker.connection.open?
|
57
60
|
end
|
58
61
|
|
59
62
|
def self.publish(*args)
|
data/lib/hutch/broker.rb
CHANGED
@@ -1,7 +1,9 @@
|
|
1
|
+
require 'active_support/core_ext/object/blank'
|
2
|
+
|
1
3
|
require 'carrot-top'
|
2
|
-
require 'securerandom'
|
3
4
|
require 'hutch/logging'
|
4
5
|
require 'hutch/exceptions'
|
6
|
+
require 'hutch/publisher'
|
5
7
|
|
6
8
|
module Hutch
|
7
9
|
class Broker
|
@@ -9,10 +11,38 @@ module Hutch
|
|
9
11
|
|
10
12
|
attr_accessor :connection, :channel, :exchange, :api_client
|
11
13
|
|
14
|
+
|
15
|
+
DEFAULT_AMQP_PORT =
|
16
|
+
case RUBY_ENGINE
|
17
|
+
when "jruby" then
|
18
|
+
com.rabbitmq.client.ConnectionFactory::DEFAULT_AMQP_PORT
|
19
|
+
when "ruby" then
|
20
|
+
AMQ::Protocol::DEFAULT_PORT
|
21
|
+
end
|
22
|
+
|
23
|
+
DEFAULT_AMQPS_PORT =
|
24
|
+
case RUBY_ENGINE
|
25
|
+
when "jruby" then
|
26
|
+
com.rabbitmq.client.ConnectionFactory::DEFAULT_AMQP_OVER_SSL_PORT
|
27
|
+
when "ruby" then
|
28
|
+
AMQ::Protocol::TLS_PORT
|
29
|
+
end
|
30
|
+
|
31
|
+
|
32
|
+
# @param config [nil,Hash] Configuration override
|
12
33
|
def initialize(config = nil)
|
13
34
|
@config = config || Hutch::Config
|
14
35
|
end
|
15
36
|
|
37
|
+
# Connect to broker
|
38
|
+
#
|
39
|
+
# @example
|
40
|
+
# Hutch::Broker.new.connect(enable_http_api_use: true) do
|
41
|
+
# # will disconnect after this block
|
42
|
+
# end
|
43
|
+
#
|
44
|
+
# @param [Hash] options The options to connect with
|
45
|
+
# @option options [Boolean] :enable_http_api_use
|
16
46
|
def connect(options = {})
|
17
47
|
@options = options
|
18
48
|
set_up_amqp_connection
|
@@ -41,42 +71,44 @@ module Hutch
|
|
41
71
|
def disconnect
|
42
72
|
@channel.close if @channel
|
43
73
|
@connection.close if @connection
|
44
|
-
@channel
|
74
|
+
@channel = nil
|
75
|
+
@connection = nil
|
76
|
+
@exchange = nil
|
77
|
+
@api_client = nil
|
45
78
|
end
|
46
79
|
|
47
|
-
# Connect to RabbitMQ via AMQP
|
48
|
-
#
|
49
|
-
# the
|
80
|
+
# Connect to RabbitMQ via AMQP
|
81
|
+
#
|
82
|
+
# This sets up the main connection and channel we use for talking to
|
83
|
+
# RabbitMQ. It also ensures the existence of the exchange we'll be using.
|
50
84
|
def set_up_amqp_connection
|
51
85
|
open_connection!
|
52
86
|
open_channel!
|
53
|
-
|
54
|
-
|
55
|
-
exchange_options = { durable: true }.merge @config[:mq_exchange_options]
|
56
|
-
logger.info "using topic exchange '#{exchange_name}'"
|
57
|
-
|
58
|
-
with_bunny_precondition_handler('exchange') do
|
59
|
-
@exchange = @channel.topic(exchange_name, exchange_options)
|
60
|
-
end
|
87
|
+
declare_exchange!
|
88
|
+
declare_publisher!
|
61
89
|
end
|
62
90
|
|
63
|
-
def open_connection
|
91
|
+
def open_connection
|
64
92
|
logger.info "connecting to rabbitmq (#{sanitized_uri})"
|
65
93
|
|
66
|
-
|
94
|
+
connection = Hutch::Adapter.new(connection_params)
|
67
95
|
|
68
96
|
with_bunny_connection_handler(sanitized_uri) do
|
69
|
-
|
97
|
+
connection.start
|
70
98
|
end
|
71
99
|
|
72
100
|
logger.info "connected to RabbitMQ at #{connection_params[:host]} as #{connection_params[:username]}"
|
73
|
-
|
101
|
+
connection
|
74
102
|
end
|
75
103
|
|
76
|
-
def
|
77
|
-
|
78
|
-
|
79
|
-
|
104
|
+
def open_connection!
|
105
|
+
@connection = open_connection
|
106
|
+
end
|
107
|
+
|
108
|
+
def open_channel
|
109
|
+
logger.info "opening rabbitmq channel with pool size #{consumer_pool_size}, abort on exception #{consumer_pool_abort_on_exception}"
|
110
|
+
connection.create_channel(nil, consumer_pool_size, consumer_pool_abort_on_exception).tap do |ch|
|
111
|
+
connection.prefetch_channel(ch, @config[:channel_prefetch])
|
80
112
|
if @config[:publisher_confirms] || @config[:force_publisher_confirms]
|
81
113
|
logger.info 'enabling publisher confirms'
|
82
114
|
ch.confirm_select
|
@@ -84,6 +116,28 @@ module Hutch
|
|
84
116
|
end
|
85
117
|
end
|
86
118
|
|
119
|
+
def open_channel!
|
120
|
+
@channel = open_channel
|
121
|
+
end
|
122
|
+
|
123
|
+
def declare_exchange(ch = channel)
|
124
|
+
exchange_name = @config[:mq_exchange]
|
125
|
+
exchange_options = { durable: true }.merge(@config[:mq_exchange_options])
|
126
|
+
logger.info "using topic exchange '#{exchange_name}'"
|
127
|
+
|
128
|
+
with_bunny_precondition_handler('exchange') do
|
129
|
+
ch.topic(exchange_name, exchange_options)
|
130
|
+
end
|
131
|
+
end
|
132
|
+
|
133
|
+
def declare_exchange!(*args)
|
134
|
+
@exchange = declare_exchange(*args)
|
135
|
+
end
|
136
|
+
|
137
|
+
def declare_publisher!
|
138
|
+
@publisher = Hutch::Publisher.new(connection, channel, exchange, @config)
|
139
|
+
end
|
140
|
+
|
87
141
|
# Set up the connection to the RabbitMQ management API. Unfortunately, this
|
88
142
|
# is necessary to do a few things that are impossible over AMQP. E.g.
|
89
143
|
# listing queues and bindings.
|
@@ -119,7 +173,7 @@ module Hutch
|
|
119
173
|
def queue(name, arguments = {})
|
120
174
|
with_bunny_precondition_handler('queue') do
|
121
175
|
namespace = @config[:namespace].to_s.downcase.gsub(/[^-_:\.\w]/, "")
|
122
|
-
name = name.prepend(namespace + ":")
|
176
|
+
name = name.prepend(namespace + ":") if namespace.present?
|
123
177
|
channel.queue(name, durable: true, arguments: arguments)
|
124
178
|
end
|
125
179
|
end
|
@@ -127,7 +181,7 @@ module Hutch
|
|
127
181
|
# Return a mapping of queue names to the routing keys they're bound to.
|
128
182
|
def bindings
|
129
183
|
results = Hash.new { |hash, key| hash[key] = [] }
|
130
|
-
|
184
|
+
api_client.bindings.each do |binding|
|
131
185
|
next if binding['destination'] == binding['routing_key']
|
132
186
|
next unless binding['source'] == @config[:mq_exchange]
|
133
187
|
next unless binding['vhost'] == @config[:mq_vhost]
|
@@ -136,37 +190,32 @@ module Hutch
|
|
136
190
|
results
|
137
191
|
end
|
138
192
|
|
193
|
+
# Find the existing bindings, and unbind any redundant bindings
|
194
|
+
def unbind_redundant_bindings(queue, routing_keys)
|
195
|
+
return unless http_api_use_enabled?
|
196
|
+
|
197
|
+
bindings.each do |dest, keys|
|
198
|
+
next unless dest == queue.name
|
199
|
+
keys.reject { |key| routing_keys.include?(key) }.each do |key|
|
200
|
+
logger.debug "removing redundant binding #{queue.name} <--> #{key}"
|
201
|
+
queue.unbind(exchange, routing_key: key)
|
202
|
+
end
|
203
|
+
end
|
204
|
+
end
|
205
|
+
|
139
206
|
# Bind a queue to the broker's exchange on the routing keys provided. Any
|
140
207
|
# existing bindings on the queue that aren't present in the array of
|
141
208
|
# routing keys will be unbound.
|
142
209
|
def bind_queue(queue, routing_keys)
|
143
|
-
|
144
|
-
# Find the existing bindings, and unbind any redundant bindings
|
145
|
-
queue_bindings = bindings.select { |dest, keys| dest == queue.name }
|
146
|
-
queue_bindings.each do |dest, keys|
|
147
|
-
keys.reject { |key| routing_keys.include?(key) }.each do |key|
|
148
|
-
logger.debug "removing redundant binding #{queue.name} <--> #{key}"
|
149
|
-
queue.unbind(@exchange, routing_key: key)
|
150
|
-
end
|
151
|
-
end
|
152
|
-
end
|
210
|
+
unbind_redundant_bindings(queue, routing_keys)
|
153
211
|
|
154
212
|
# Ensure all the desired bindings are present
|
155
213
|
routing_keys.each do |routing_key|
|
156
214
|
logger.debug "creating binding #{queue.name} <--> #{routing_key}"
|
157
|
-
queue.bind(
|
215
|
+
queue.bind(exchange, routing_key: routing_key)
|
158
216
|
end
|
159
217
|
end
|
160
218
|
|
161
|
-
# Each subscriber is run in a thread. This calls Thread#join on each of the
|
162
|
-
# subscriber threads.
|
163
|
-
def wait_on_threads(timeout)
|
164
|
-
# Thread#join returns nil when the timeout is hit. If any return nil,
|
165
|
-
# the threads didn't all join so we return false.
|
166
|
-
per_thread_timeout = timeout.to_f / work_pool_threads.length
|
167
|
-
work_pool_threads.none? { |thread| thread.join(per_thread_timeout).nil? }
|
168
|
-
end
|
169
|
-
|
170
219
|
def stop
|
171
220
|
if defined?(JRUBY_VERSION)
|
172
221
|
channel.close
|
@@ -181,78 +230,40 @@ module Hutch
|
|
181
230
|
end
|
182
231
|
|
183
232
|
def requeue(delivery_tag)
|
184
|
-
|
233
|
+
channel.reject(delivery_tag, true)
|
185
234
|
end
|
186
235
|
|
187
236
|
def reject(delivery_tag, requeue=false)
|
188
|
-
|
237
|
+
channel.reject(delivery_tag, requeue)
|
189
238
|
end
|
190
239
|
|
191
240
|
def ack(delivery_tag)
|
192
|
-
|
241
|
+
channel.ack(delivery_tag, false)
|
193
242
|
end
|
194
243
|
|
195
244
|
def nack(delivery_tag)
|
196
|
-
|
245
|
+
channel.nack(delivery_tag, false, false)
|
197
246
|
end
|
198
247
|
|
199
|
-
def publish(
|
200
|
-
|
201
|
-
|
202
|
-
serializer = options[:serializer] || @config[:serializer]
|
203
|
-
|
204
|
-
non_overridable_properties = {
|
205
|
-
routing_key: routing_key,
|
206
|
-
timestamp: @connection.current_timestamp,
|
207
|
-
content_type: serializer.content_type,
|
208
|
-
}
|
209
|
-
properties[:message_id] ||= generate_id
|
210
|
-
|
211
|
-
payload = serializer.encode(message)
|
212
|
-
logger.info {
|
213
|
-
spec =
|
214
|
-
if serializer.binary?
|
215
|
-
"#{payload.bytesize} bytes message"
|
216
|
-
else
|
217
|
-
"message '#{payload}'"
|
218
|
-
end
|
219
|
-
"publishing #{spec} to #{routing_key}"
|
220
|
-
}
|
221
|
-
|
222
|
-
response = @exchange.publish(payload, {persistent: true}.
|
223
|
-
merge(properties).
|
224
|
-
merge(global_properties).
|
225
|
-
merge(non_overridable_properties))
|
226
|
-
|
227
|
-
channel.wait_for_confirms if @config[:force_publisher_confirms]
|
228
|
-
response
|
248
|
+
def publish(*args)
|
249
|
+
@publisher.publish(*args)
|
229
250
|
end
|
230
251
|
|
231
252
|
def confirm_select(*args)
|
232
|
-
|
253
|
+
channel.confirm_select(*args)
|
233
254
|
end
|
234
255
|
|
235
256
|
def wait_for_confirms
|
236
|
-
|
257
|
+
channel.wait_for_confirms
|
237
258
|
end
|
238
259
|
|
260
|
+
# @return [Boolean] True if channel is set up to use publisher confirmations.
|
239
261
|
def using_publisher_confirmations?
|
240
|
-
|
262
|
+
channel.using_publisher_confirmations?
|
241
263
|
end
|
242
264
|
|
243
265
|
private
|
244
266
|
|
245
|
-
def raise_publish_error(reason, routing_key, message)
|
246
|
-
msg = "unable to publish - #{reason}. Message: #{JSON.dump(message)}, Routing key: #{routing_key}."
|
247
|
-
logger.error(msg)
|
248
|
-
raise PublishError, msg
|
249
|
-
end
|
250
|
-
|
251
|
-
def ensure_connection!(routing_key, message)
|
252
|
-
raise_publish_error('no connection to broker', routing_key, message) unless @connection
|
253
|
-
raise_publish_error('connection is closed', routing_key, message) unless @connection.open?
|
254
|
-
end
|
255
|
-
|
256
267
|
def api_config
|
257
268
|
@api_config ||= OpenStruct.new.tap do |config|
|
258
269
|
config.host = @config[:mq_api_host]
|
@@ -271,11 +282,7 @@ module Hutch
|
|
271
282
|
{}.tap do |params|
|
272
283
|
params[:host] = @config[:mq_host]
|
273
284
|
params[:port] = @config[:mq_port]
|
274
|
-
params[:vhost] =
|
275
|
-
@config[:mq_vhost]
|
276
|
-
else
|
277
|
-
Hutch::Adapter::DEFAULT_VHOST
|
278
|
-
end
|
285
|
+
params[:vhost] = @config[:mq_vhost].presence || Hutch::Adapter::DEFAULT_VHOST
|
279
286
|
params[:username] = @config[:mq_username]
|
280
287
|
params[:password] = @config[:mq_password]
|
281
288
|
params[:tls] = @config[:mq_tls]
|
@@ -299,17 +306,22 @@ module Hutch
|
|
299
306
|
end
|
300
307
|
|
301
308
|
def parse_uri
|
302
|
-
return
|
309
|
+
return if @config[:uri].blank?
|
303
310
|
|
304
311
|
u = URI.parse(@config[:uri])
|
305
312
|
|
313
|
+
@config[:mq_tls] = u.scheme == 'amqps'
|
306
314
|
@config[:mq_host] = u.host
|
307
|
-
@config[:mq_port] = u.port
|
315
|
+
@config[:mq_port] = u.port || default_mq_port
|
308
316
|
@config[:mq_vhost] = u.path.sub(/^\//, "")
|
309
317
|
@config[:mq_username] = u.user
|
310
318
|
@config[:mq_password] = u.password
|
311
319
|
end
|
312
320
|
|
321
|
+
def default_mq_port
|
322
|
+
@config[:mq_tls] ? DEFAULT_AMQPS_PORT : DEFAULT_AMQP_PORT
|
323
|
+
end
|
324
|
+
|
313
325
|
def sanitized_uri
|
314
326
|
p = connection_params
|
315
327
|
scheme = p[:tls] ? "amqps" : "amqp"
|
@@ -351,25 +363,16 @@ module Hutch
|
|
351
363
|
raise ConnectionError.new("couldn't connect to rabbitmq at #{uri}. Check your configuration, network connectivity and RabbitMQ logs.")
|
352
364
|
end
|
353
365
|
|
354
|
-
def work_pool_threads
|
355
|
-
channel_work_pool.threads || []
|
356
|
-
end
|
357
|
-
|
358
366
|
def channel_work_pool
|
359
|
-
|
367
|
+
channel.work_pool
|
360
368
|
end
|
361
369
|
|
362
370
|
def consumer_pool_size
|
363
371
|
@config[:consumer_pool_size]
|
364
372
|
end
|
365
373
|
|
366
|
-
def
|
367
|
-
|
374
|
+
def consumer_pool_abort_on_exception
|
375
|
+
@config[:consumer_pool_abort_on_exception]
|
368
376
|
end
|
369
|
-
|
370
|
-
def global_properties
|
371
|
-
Hutch.global_properties.respond_to?(:call) ? Hutch.global_properties.call : Hutch.global_properties
|
372
|
-
end
|
373
|
-
|
374
377
|
end
|
375
378
|
end
|