hutch 0.21.0-java → 0.25.0-java

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. checksums.yaml +5 -5
  2. data/.gitignore +3 -0
  3. data/.rspec +1 -0
  4. data/.travis.yml +11 -12
  5. data/.yardopts +5 -0
  6. data/CHANGELOG.md +118 -1
  7. data/Gemfile +15 -4
  8. data/Guardfile +13 -4
  9. data/README.md +274 -24
  10. data/Rakefile +8 -1
  11. data/hutch.gemspec +6 -7
  12. data/lib/hutch.rb +11 -8
  13. data/lib/hutch/adapters/march_hare.rb +1 -1
  14. data/lib/hutch/broker.rb +113 -110
  15. data/lib/hutch/cli.rb +42 -11
  16. data/lib/hutch/config.rb +209 -59
  17. data/lib/hutch/error_handlers.rb +1 -0
  18. data/lib/hutch/error_handlers/airbrake.rb +44 -16
  19. data/lib/hutch/error_handlers/base.rb +15 -0
  20. data/lib/hutch/error_handlers/honeybadger.rb +33 -18
  21. data/lib/hutch/error_handlers/logger.rb +12 -6
  22. data/lib/hutch/error_handlers/opbeat.rb +30 -0
  23. data/lib/hutch/error_handlers/sentry.rb +14 -6
  24. data/lib/hutch/logging.rb +5 -5
  25. data/lib/hutch/publisher.rb +75 -0
  26. data/lib/hutch/tracers.rb +1 -0
  27. data/lib/hutch/tracers/opbeat.rb +37 -0
  28. data/lib/hutch/version.rb +1 -1
  29. data/lib/hutch/waiter.rb +104 -0
  30. data/lib/hutch/worker.rb +50 -66
  31. data/lib/yard-settings/handler.rb +38 -0
  32. data/lib/yard-settings/yard-settings.rb +2 -0
  33. data/spec/hutch/broker_spec.rb +162 -77
  34. data/spec/hutch/cli_spec.rb +16 -3
  35. data/spec/hutch/config_spec.rb +83 -22
  36. data/spec/hutch/error_handlers/airbrake_spec.rb +25 -10
  37. data/spec/hutch/error_handlers/honeybadger_spec.rb +24 -2
  38. data/spec/hutch/error_handlers/logger_spec.rb +14 -1
  39. data/spec/hutch/error_handlers/opbeat_spec.rb +37 -0
  40. data/spec/hutch/error_handlers/sentry_spec.rb +18 -1
  41. data/spec/hutch/logger_spec.rb +12 -6
  42. data/spec/hutch/waiter_spec.rb +51 -0
  43. data/spec/hutch/worker_spec.rb +33 -4
  44. data/spec/spec_helper.rb +7 -5
  45. data/spec/tracers/opbeat_spec.rb +44 -0
  46. data/templates/default/class/html/settings.erb +0 -0
  47. data/templates/default/class/setup.rb +4 -0
  48. data/templates/default/fulldoc/html/css/hutch.css +13 -0
  49. data/templates/default/layout/html/setup.rb +7 -0
  50. data/templates/default/method_details/html/settings.erb +5 -0
  51. data/templates/default/method_details/setup.rb +4 -0
  52. data/templates/default/method_details/text/settings.erb +0 -0
  53. data/templates/default/module/html/settings.erb +40 -0
  54. data/templates/default/module/setup.rb +4 -0
  55. metadata +41 -38
data/Rakefile CHANGED
@@ -11,4 +11,11 @@ RSpec::Core::RakeTask.new(:spec) do |t|
11
11
  t.rspec_opts = %w(--color --format doc)
12
12
  end
13
13
 
14
- task :default => :spec
14
+ task default: :spec
15
+
16
+ #
17
+ # Re-generate API docs
18
+ #
19
+ require 'yard'
20
+ require 'yard/rake/yardoc_task'
21
+ YARD::Rake::YardocTask.new
@@ -3,22 +3,21 @@ require File.expand_path('../lib/hutch/version', __FILE__)
3
3
  Gem::Specification.new do |gem|
4
4
  if defined?(JRUBY_VERSION)
5
5
  gem.platform = 'java'
6
- gem.add_runtime_dependency 'march_hare', '>= 2.16.0'
6
+ gem.add_runtime_dependency 'march_hare', '>= 3.0.0'
7
7
  else
8
8
  gem.platform = Gem::Platform::RUBY
9
- gem.add_runtime_dependency 'bunny', '>= 2.2.2'
9
+ gem.add_runtime_dependency 'bunny', '~> 2.9.0'
10
10
  end
11
11
  gem.add_runtime_dependency 'carrot-top', '~> 0.0.7'
12
- gem.add_runtime_dependency 'multi_json', '~> 1.11.2'
13
- gem.add_runtime_dependency 'activesupport', '>= 3.0'
14
- gem.add_development_dependency 'rspec', '~> 3.0'
15
- gem.add_development_dependency 'simplecov', '~> 0.7.1'
12
+ gem.add_runtime_dependency 'multi_json', '~> 1.12'
13
+ gem.add_runtime_dependency 'activesupport', '>= 4.2', '< 6'
16
14
 
17
15
  gem.name = 'hutch'
18
16
  gem.summary = 'Easy inter-service communication using RabbitMQ.'
19
- gem.description = 'Hutch is a Ruby library for enabling asynchronous ' +
17
+ gem.description = 'Hutch is a Ruby library for enabling asynchronous ' \
20
18
  'inter-service communication using RabbitMQ.'
21
19
  gem.version = Hutch::VERSION.dup
20
+ gem.required_ruby_version = '>= 2.2'
22
21
  gem.authors = ['Harry Marr']
23
22
  gem.email = ['developers@gocardless.com']
24
23
  gem.homepage = 'https://github.com/gocardless/hutch'
@@ -14,7 +14,6 @@ require 'hutch/exceptions'
14
14
  require 'hutch/tracers'
15
15
 
16
16
  module Hutch
17
-
18
17
  def self.register_consumer(consumer)
19
18
  self.consumers << consumer
20
19
  end
@@ -35,11 +34,16 @@ module Hutch
35
34
  @global_properties ||= {}
36
35
  end
37
36
 
37
+ # Connects to broker, if not yet connected.
38
+ #
39
+ # @param options [Hash] Connection options
40
+ # @param config [Hash] Configuration
41
+ # @option options [Boolean] :enable_http_api_use
38
42
  def self.connect(options = {}, config = Hutch::Config)
39
- unless connected?
40
- @broker = Hutch::Broker.new(config)
41
- @broker.connect(options)
42
- end
43
+ return if connected?
44
+
45
+ @broker = Hutch::Broker.new(config)
46
+ @broker.connect(options)
43
47
  end
44
48
 
45
49
  def self.disconnect
@@ -50,10 +54,9 @@ module Hutch
50
54
  @broker
51
55
  end
52
56
 
57
+ # @return [Boolean]
53
58
  def self.connected?
54
- return false unless broker
55
- return false unless broker.connection
56
- broker.connection.open?
59
+ broker && broker.connection && broker.connection.open?
57
60
  end
58
61
 
59
62
  def self.publish(*args)
@@ -25,7 +25,7 @@ module Hutch
25
25
  ch.prefetch = prefetch if prefetch
26
26
  end
27
27
 
28
- def create_channel(n = nil, consumer_pool_size = 1)
28
+ def create_channel(n = nil, consumer_pool_size = 1, consumer_pool_abort_on_exception = false)
29
29
  @connection.create_channel(n)
30
30
  end
31
31
 
@@ -1,7 +1,9 @@
1
+ require 'active_support/core_ext/object/blank'
2
+
1
3
  require 'carrot-top'
2
- require 'securerandom'
3
4
  require 'hutch/logging'
4
5
  require 'hutch/exceptions'
6
+ require 'hutch/publisher'
5
7
 
6
8
  module Hutch
7
9
  class Broker
@@ -9,10 +11,38 @@ module Hutch
9
11
 
10
12
  attr_accessor :connection, :channel, :exchange, :api_client
11
13
 
14
+
15
+ DEFAULT_AMQP_PORT =
16
+ case RUBY_ENGINE
17
+ when "jruby" then
18
+ com.rabbitmq.client.ConnectionFactory::DEFAULT_AMQP_PORT
19
+ when "ruby" then
20
+ AMQ::Protocol::DEFAULT_PORT
21
+ end
22
+
23
+ DEFAULT_AMQPS_PORT =
24
+ case RUBY_ENGINE
25
+ when "jruby" then
26
+ com.rabbitmq.client.ConnectionFactory::DEFAULT_AMQP_OVER_SSL_PORT
27
+ when "ruby" then
28
+ AMQ::Protocol::TLS_PORT
29
+ end
30
+
31
+
32
+ # @param config [nil,Hash] Configuration override
12
33
  def initialize(config = nil)
13
34
  @config = config || Hutch::Config
14
35
  end
15
36
 
37
+ # Connect to broker
38
+ #
39
+ # @example
40
+ # Hutch::Broker.new.connect(enable_http_api_use: true) do
41
+ # # will disconnect after this block
42
+ # end
43
+ #
44
+ # @param [Hash] options The options to connect with
45
+ # @option options [Boolean] :enable_http_api_use
16
46
  def connect(options = {})
17
47
  @options = options
18
48
  set_up_amqp_connection
@@ -41,42 +71,44 @@ module Hutch
41
71
  def disconnect
42
72
  @channel.close if @channel
43
73
  @connection.close if @connection
44
- @channel, @connection, @exchange, @api_client = nil, nil, nil, nil
74
+ @channel = nil
75
+ @connection = nil
76
+ @exchange = nil
77
+ @api_client = nil
45
78
  end
46
79
 
47
- # Connect to RabbitMQ via AMQP. This sets up the main connection and
48
- # channel we use for talking to RabbitMQ. It also ensures the existance of
49
- # the exchange we'll be using.
80
+ # Connect to RabbitMQ via AMQP
81
+ #
82
+ # This sets up the main connection and channel we use for talking to
83
+ # RabbitMQ. It also ensures the existence of the exchange we'll be using.
50
84
  def set_up_amqp_connection
51
85
  open_connection!
52
86
  open_channel!
53
-
54
- exchange_name = @config[:mq_exchange]
55
- exchange_options = { durable: true }.merge @config[:mq_exchange_options]
56
- logger.info "using topic exchange '#{exchange_name}'"
57
-
58
- with_bunny_precondition_handler('exchange') do
59
- @exchange = @channel.topic(exchange_name, exchange_options)
60
- end
87
+ declare_exchange!
88
+ declare_publisher!
61
89
  end
62
90
 
63
- def open_connection!
91
+ def open_connection
64
92
  logger.info "connecting to rabbitmq (#{sanitized_uri})"
65
93
 
66
- @connection = Hutch::Adapter.new(connection_params)
94
+ connection = Hutch::Adapter.new(connection_params)
67
95
 
68
96
  with_bunny_connection_handler(sanitized_uri) do
69
- @connection.start
97
+ connection.start
70
98
  end
71
99
 
72
100
  logger.info "connected to RabbitMQ at #{connection_params[:host]} as #{connection_params[:username]}"
73
- @connection
101
+ connection
74
102
  end
75
103
 
76
- def open_channel!
77
- logger.info "opening rabbitmq channel with pool size #{consumer_pool_size}"
78
- @channel = @connection.create_channel(nil, consumer_pool_size).tap do |ch|
79
- @connection.prefetch_channel(ch, @config[:channel_prefetch])
104
+ def open_connection!
105
+ @connection = open_connection
106
+ end
107
+
108
+ def open_channel
109
+ logger.info "opening rabbitmq channel with pool size #{consumer_pool_size}, abort on exception #{consumer_pool_abort_on_exception}"
110
+ connection.create_channel(nil, consumer_pool_size, consumer_pool_abort_on_exception).tap do |ch|
111
+ connection.prefetch_channel(ch, @config[:channel_prefetch])
80
112
  if @config[:publisher_confirms] || @config[:force_publisher_confirms]
81
113
  logger.info 'enabling publisher confirms'
82
114
  ch.confirm_select
@@ -84,6 +116,28 @@ module Hutch
84
116
  end
85
117
  end
86
118
 
119
+ def open_channel!
120
+ @channel = open_channel
121
+ end
122
+
123
+ def declare_exchange(ch = channel)
124
+ exchange_name = @config[:mq_exchange]
125
+ exchange_options = { durable: true }.merge(@config[:mq_exchange_options])
126
+ logger.info "using topic exchange '#{exchange_name}'"
127
+
128
+ with_bunny_precondition_handler('exchange') do
129
+ ch.topic(exchange_name, exchange_options)
130
+ end
131
+ end
132
+
133
+ def declare_exchange!(*args)
134
+ @exchange = declare_exchange(*args)
135
+ end
136
+
137
+ def declare_publisher!
138
+ @publisher = Hutch::Publisher.new(connection, channel, exchange, @config)
139
+ end
140
+
87
141
  # Set up the connection to the RabbitMQ management API. Unfortunately, this
88
142
  # is necessary to do a few things that are impossible over AMQP. E.g.
89
143
  # listing queues and bindings.
@@ -119,7 +173,7 @@ module Hutch
119
173
  def queue(name, arguments = {})
120
174
  with_bunny_precondition_handler('queue') do
121
175
  namespace = @config[:namespace].to_s.downcase.gsub(/[^-_:\.\w]/, "")
122
- name = name.prepend(namespace + ":") unless namespace.empty?
176
+ name = name.prepend(namespace + ":") if namespace.present?
123
177
  channel.queue(name, durable: true, arguments: arguments)
124
178
  end
125
179
  end
@@ -127,7 +181,7 @@ module Hutch
127
181
  # Return a mapping of queue names to the routing keys they're bound to.
128
182
  def bindings
129
183
  results = Hash.new { |hash, key| hash[key] = [] }
130
- @api_client.bindings.each do |binding|
184
+ api_client.bindings.each do |binding|
131
185
  next if binding['destination'] == binding['routing_key']
132
186
  next unless binding['source'] == @config[:mq_exchange]
133
187
  next unless binding['vhost'] == @config[:mq_vhost]
@@ -136,37 +190,32 @@ module Hutch
136
190
  results
137
191
  end
138
192
 
193
+ # Find the existing bindings, and unbind any redundant bindings
194
+ def unbind_redundant_bindings(queue, routing_keys)
195
+ return unless http_api_use_enabled?
196
+
197
+ bindings.each do |dest, keys|
198
+ next unless dest == queue.name
199
+ keys.reject { |key| routing_keys.include?(key) }.each do |key|
200
+ logger.debug "removing redundant binding #{queue.name} <--> #{key}"
201
+ queue.unbind(exchange, routing_key: key)
202
+ end
203
+ end
204
+ end
205
+
139
206
  # Bind a queue to the broker's exchange on the routing keys provided. Any
140
207
  # existing bindings on the queue that aren't present in the array of
141
208
  # routing keys will be unbound.
142
209
  def bind_queue(queue, routing_keys)
143
- if http_api_use_enabled?
144
- # Find the existing bindings, and unbind any redundant bindings
145
- queue_bindings = bindings.select { |dest, keys| dest == queue.name }
146
- queue_bindings.each do |dest, keys|
147
- keys.reject { |key| routing_keys.include?(key) }.each do |key|
148
- logger.debug "removing redundant binding #{queue.name} <--> #{key}"
149
- queue.unbind(@exchange, routing_key: key)
150
- end
151
- end
152
- end
210
+ unbind_redundant_bindings(queue, routing_keys)
153
211
 
154
212
  # Ensure all the desired bindings are present
155
213
  routing_keys.each do |routing_key|
156
214
  logger.debug "creating binding #{queue.name} <--> #{routing_key}"
157
- queue.bind(@exchange, routing_key: routing_key)
215
+ queue.bind(exchange, routing_key: routing_key)
158
216
  end
159
217
  end
160
218
 
161
- # Each subscriber is run in a thread. This calls Thread#join on each of the
162
- # subscriber threads.
163
- def wait_on_threads(timeout)
164
- # Thread#join returns nil when the timeout is hit. If any return nil,
165
- # the threads didn't all join so we return false.
166
- per_thread_timeout = timeout.to_f / work_pool_threads.length
167
- work_pool_threads.none? { |thread| thread.join(per_thread_timeout).nil? }
168
- end
169
-
170
219
  def stop
171
220
  if defined?(JRUBY_VERSION)
172
221
  channel.close
@@ -181,78 +230,40 @@ module Hutch
181
230
  end
182
231
 
183
232
  def requeue(delivery_tag)
184
- @channel.reject(delivery_tag, true)
233
+ channel.reject(delivery_tag, true)
185
234
  end
186
235
 
187
236
  def reject(delivery_tag, requeue=false)
188
- @channel.reject(delivery_tag, requeue)
237
+ channel.reject(delivery_tag, requeue)
189
238
  end
190
239
 
191
240
  def ack(delivery_tag)
192
- @channel.ack(delivery_tag, false)
241
+ channel.ack(delivery_tag, false)
193
242
  end
194
243
 
195
244
  def nack(delivery_tag)
196
- @channel.nack(delivery_tag, false, false)
245
+ channel.nack(delivery_tag, false, false)
197
246
  end
198
247
 
199
- def publish(routing_key, message, properties = {}, options = {})
200
- ensure_connection!(routing_key, message)
201
-
202
- serializer = options[:serializer] || @config[:serializer]
203
-
204
- non_overridable_properties = {
205
- routing_key: routing_key,
206
- timestamp: @connection.current_timestamp,
207
- content_type: serializer.content_type,
208
- }
209
- properties[:message_id] ||= generate_id
210
-
211
- payload = serializer.encode(message)
212
- logger.info {
213
- spec =
214
- if serializer.binary?
215
- "#{payload.bytesize} bytes message"
216
- else
217
- "message '#{payload}'"
218
- end
219
- "publishing #{spec} to #{routing_key}"
220
- }
221
-
222
- response = @exchange.publish(payload, {persistent: true}.
223
- merge(properties).
224
- merge(global_properties).
225
- merge(non_overridable_properties))
226
-
227
- channel.wait_for_confirms if @config[:force_publisher_confirms]
228
- response
248
+ def publish(*args)
249
+ @publisher.publish(*args)
229
250
  end
230
251
 
231
252
  def confirm_select(*args)
232
- @channel.confirm_select(*args)
253
+ channel.confirm_select(*args)
233
254
  end
234
255
 
235
256
  def wait_for_confirms
236
- @channel.wait_for_confirms
257
+ channel.wait_for_confirms
237
258
  end
238
259
 
260
+ # @return [Boolean] True if channel is set up to use publisher confirmations.
239
261
  def using_publisher_confirmations?
240
- @channel.using_publisher_confirmations?
262
+ channel.using_publisher_confirmations?
241
263
  end
242
264
 
243
265
  private
244
266
 
245
- def raise_publish_error(reason, routing_key, message)
246
- msg = "unable to publish - #{reason}. Message: #{JSON.dump(message)}, Routing key: #{routing_key}."
247
- logger.error(msg)
248
- raise PublishError, msg
249
- end
250
-
251
- def ensure_connection!(routing_key, message)
252
- raise_publish_error('no connection to broker', routing_key, message) unless @connection
253
- raise_publish_error('connection is closed', routing_key, message) unless @connection.open?
254
- end
255
-
256
267
  def api_config
257
268
  @api_config ||= OpenStruct.new.tap do |config|
258
269
  config.host = @config[:mq_api_host]
@@ -271,11 +282,7 @@ module Hutch
271
282
  {}.tap do |params|
272
283
  params[:host] = @config[:mq_host]
273
284
  params[:port] = @config[:mq_port]
274
- params[:vhost] = if @config[:mq_vhost] && "" != @config[:mq_vhost]
275
- @config[:mq_vhost]
276
- else
277
- Hutch::Adapter::DEFAULT_VHOST
278
- end
285
+ params[:vhost] = @config[:mq_vhost].presence || Hutch::Adapter::DEFAULT_VHOST
279
286
  params[:username] = @config[:mq_username]
280
287
  params[:password] = @config[:mq_password]
281
288
  params[:tls] = @config[:mq_tls]
@@ -299,17 +306,22 @@ module Hutch
299
306
  end
300
307
 
301
308
  def parse_uri
302
- return unless @config[:uri] && !@config[:uri].empty?
309
+ return if @config[:uri].blank?
303
310
 
304
311
  u = URI.parse(@config[:uri])
305
312
 
313
+ @config[:mq_tls] = u.scheme == 'amqps'
306
314
  @config[:mq_host] = u.host
307
- @config[:mq_port] = u.port
315
+ @config[:mq_port] = u.port || default_mq_port
308
316
  @config[:mq_vhost] = u.path.sub(/^\//, "")
309
317
  @config[:mq_username] = u.user
310
318
  @config[:mq_password] = u.password
311
319
  end
312
320
 
321
+ def default_mq_port
322
+ @config[:mq_tls] ? DEFAULT_AMQPS_PORT : DEFAULT_AMQP_PORT
323
+ end
324
+
313
325
  def sanitized_uri
314
326
  p = connection_params
315
327
  scheme = p[:tls] ? "amqps" : "amqp"
@@ -351,25 +363,16 @@ module Hutch
351
363
  raise ConnectionError.new("couldn't connect to rabbitmq at #{uri}. Check your configuration, network connectivity and RabbitMQ logs.")
352
364
  end
353
365
 
354
- def work_pool_threads
355
- channel_work_pool.threads || []
356
- end
357
-
358
366
  def channel_work_pool
359
- @channel.work_pool
367
+ channel.work_pool
360
368
  end
361
369
 
362
370
  def consumer_pool_size
363
371
  @config[:consumer_pool_size]
364
372
  end
365
373
 
366
- def generate_id
367
- SecureRandom.uuid
374
+ def consumer_pool_abort_on_exception
375
+ @config[:consumer_pool_abort_on_exception]
368
376
  end
369
-
370
- def global_properties
371
- Hutch.global_properties.respond_to?(:call) ? Hutch.global_properties.call : Hutch.global_properties
372
- end
373
-
374
377
  end
375
378
  end