hutch 0.21.0 → 0.22.1

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 85403719cdd5fb675886fc2dd6a830a544b3f8ea
4
- data.tar.gz: ca485c1fe62a41867f634d46a45798570b3d441f
3
+ metadata.gz: a331137bca776d401e85057a0caa84922dda9d1d
4
+ data.tar.gz: c677873d07dfad4a6be14574d413e3870febb1bd
5
5
  SHA512:
6
- metadata.gz: 0ff8d4d207f98acadd27035454a23aa50119076621b01b71fa21d342df02ff67574454f0b26374a60a2b04d941d1b170a6dfd420d12136abb404a17990ab5ea7
7
- data.tar.gz: 62fef649721ee040dd1555310a6cd6338da4d00807ab5a6f41cb940e131df919a97e3dc173b2c2a4a9bfff311bce5835f1d37652472229470353c7787af407f7
6
+ metadata.gz: dd21d047618cf83c463b7064042270d2886d7cc232c9d1234b542912e37e7250f8a700fb47fcdc62f31ab61789a4fc3f89f3946387187081d0c705f0024d0ffe
7
+ data.tar.gz: bee9c047725370823d44a5a392e1544559998d70c36b7b084d6ba7f8cf93907f822d2ce935c024d51a3c4cfb567d1a68ab402bde62b30f497a67d4df85edac43
data/.gitignore CHANGED
@@ -5,3 +5,5 @@ Gemfile.lock
5
5
  .ruby-gemset
6
6
  .idea
7
7
  coverage
8
+ .yardoc
9
+ doc
@@ -0,0 +1,5 @@
1
+ --load "./lib/yard-settings/yard-settings.rb"
2
+ --title "Hutch API docs"
3
+ -
4
+ README.md
5
+
@@ -1,4 +1,58 @@
1
- ## 0.21.0 — (unreleased)
1
+ ## 0.23.0 — (unreleased)
2
+
3
+ No changes yet.
4
+
5
+
6
+ ## 0.22.0 — June 7th, 2016
7
+
8
+ ### Message Payload is Reported to Sentry
9
+
10
+ Contributed by Matt Thompson.
11
+
12
+ ### Daemonization Flag Ignored on JRuby
13
+
14
+ Hutch will no longer try to daemonize its process on JRuby
15
+ (since it is not supported) and will emit a warning instead.
16
+
17
+ Contributed by Olle Jonsson.
18
+
19
+ ### Custom Setup Steps in Hutch::Worker
20
+
21
+ `Hutch::Worker` now accepts a list of callables that are invoked
22
+ after queue setup.
23
+
24
+ Contributed by Kelly Stannard.
25
+
26
+ ### More Flexible and Better Abstracted Hutch::Broker
27
+
28
+ `Hutch::Broker` was refactored with some bits extracted into separate
29
+ classes or methods, making them easier to override.
30
+
31
+ Contributed by Aleksandar Ivanov and Ryan Hosford.
32
+
33
+ ### Configurable Consumer Thread Pool Exception Handling (MRI only)
34
+
35
+ `:consumer_pool_abort_on_exception` is a new option
36
+ (defaults to `false`) which defines whether Bunny's
37
+ consumer work pool threads should abort on exception.
38
+ The option is ignored on JRuby.
39
+
40
+ Contributed by Seamus Abshere.
41
+
42
+ ### Worker: Log received messages using level DEBUG instead of INFO
43
+
44
+ Received messages used to be logged using severity level INFO.
45
+ This has been lowered to DEBUG.
46
+
47
+ Contributed by Jesper Josefsson.
48
+
49
+ ### Refactoring
50
+
51
+ Olle Jonsson and Kelly Stannard have contributed
52
+ multiple internal improvements that have no behaviour changes.
53
+
54
+
55
+ ## 0.21.0 — February 7th, 2016
2
56
 
3
57
  ### JRuby Compatibility Restored
4
58
 
data/Gemfile CHANGED
@@ -13,7 +13,7 @@ group :development, :test do
13
13
  gem "honeybadger"
14
14
  gem "coveralls", require: false
15
15
  gem "newrelic_rpm"
16
- gem "airbrake"
16
+ gem "airbrake", "~> 4.0"
17
17
  end
18
18
 
19
19
  group :development, :darwin do
data/README.md CHANGED
@@ -11,7 +11,7 @@ in a service-oriented architecture, using RabbitMQ.
11
11
  To install with RubyGems:
12
12
 
13
13
  ```
14
- $ gem install hutch
14
+ gem install hutch
15
15
  ```
16
16
 
17
17
  ## Project Maturity
@@ -107,7 +107,7 @@ a [Logger object](http://ruby-doc.org/stdlib-2.1.2/libdoc/logger/rdoc/Logger.htm
107
107
  class FailedPaymentConsumer
108
108
  include Hutch::Consumer
109
109
  consume 'gc.ps.payment.failed'
110
-
110
+
111
111
  def process(message)
112
112
  logger.info "Marking payment #{message[:id]} as failed"
113
113
  mark_payment_as_failed(message[:id])
data/Rakefile CHANGED
@@ -11,4 +11,11 @@ RSpec::Core::RakeTask.new(:spec) do |t|
11
11
  t.rspec_opts = %w(--color --format doc)
12
12
  end
13
13
 
14
- task :default => :spec
14
+ task default: :spec
15
+
16
+ #
17
+ # Re-generate API docs
18
+ #
19
+ require 'yard'
20
+ require 'yard/rake/yardoc_task'
21
+ YARD::Rake::YardocTask.new
@@ -6,13 +6,16 @@ Gem::Specification.new do |gem|
6
6
  gem.add_runtime_dependency 'march_hare', '>= 2.16.0'
7
7
  else
8
8
  gem.platform = Gem::Platform::RUBY
9
- gem.add_runtime_dependency 'bunny', '>= 2.2.2'
9
+ gem.add_runtime_dependency 'bunny', '>= 2.3.1'
10
10
  end
11
11
  gem.add_runtime_dependency 'carrot-top', '~> 0.0.7'
12
12
  gem.add_runtime_dependency 'multi_json', '~> 1.11.2'
13
13
  gem.add_runtime_dependency 'activesupport', '>= 3.0'
14
14
  gem.add_development_dependency 'rspec', '~> 3.0'
15
15
  gem.add_development_dependency 'simplecov', '~> 0.7.1'
16
+ gem.add_development_dependency 'yard', '~> 0.8'
17
+ gem.add_development_dependency 'redcarpet', '> 0'
18
+ gem.add_development_dependency 'github-markup', '> 0'
16
19
 
17
20
  gem.name = 'hutch'
18
21
  gem.summary = 'Easy inter-service communication using RabbitMQ.'
@@ -14,7 +14,6 @@ require 'hutch/exceptions'
14
14
  require 'hutch/tracers'
15
15
 
16
16
  module Hutch
17
-
18
17
  def self.register_consumer(consumer)
19
18
  self.consumers << consumer
20
19
  end
@@ -35,11 +34,16 @@ module Hutch
35
34
  @global_properties ||= {}
36
35
  end
37
36
 
37
+ # Connects to broker, if not yet connected.
38
+ #
39
+ # @param options [Hash] Connection options
40
+ # @param config [Hash] Configuration
41
+ # @option options [Boolean] :enable_http_api_use
38
42
  def self.connect(options = {}, config = Hutch::Config)
39
- unless connected?
40
- @broker = Hutch::Broker.new(config)
41
- @broker.connect(options)
42
- end
43
+ return if connected?
44
+
45
+ @broker = Hutch::Broker.new(config)
46
+ @broker.connect(options)
43
47
  end
44
48
 
45
49
  def self.disconnect
@@ -50,10 +54,9 @@ module Hutch
50
54
  @broker
51
55
  end
52
56
 
57
+ # @return [Boolean]
53
58
  def self.connected?
54
- return false unless broker
55
- return false unless broker.connection
56
- broker.connection.open?
59
+ broker && broker.connection && broker.connection.open?
57
60
  end
58
61
 
59
62
  def self.publish(*args)
@@ -25,7 +25,7 @@ module Hutch
25
25
  ch.prefetch = prefetch if prefetch
26
26
  end
27
27
 
28
- def create_channel(n = nil, consumer_pool_size = 1)
28
+ def create_channel(n = nil, consumer_pool_size = 1, consumer_pool_abort_on_exception = false)
29
29
  @connection.create_channel(n)
30
30
  end
31
31
 
@@ -1,7 +1,9 @@
1
+ require 'active_support/core_ext/object/blank'
2
+
1
3
  require 'carrot-top'
2
- require 'securerandom'
3
4
  require 'hutch/logging'
4
5
  require 'hutch/exceptions'
6
+ require 'hutch/publisher'
5
7
 
6
8
  module Hutch
7
9
  class Broker
@@ -9,10 +11,20 @@ module Hutch
9
11
 
10
12
  attr_accessor :connection, :channel, :exchange, :api_client
11
13
 
14
+ # @param config [nil,Hash] Configuration override
12
15
  def initialize(config = nil)
13
16
  @config = config || Hutch::Config
14
17
  end
15
18
 
19
+ # Connect to broker
20
+ #
21
+ # @example
22
+ # Hutch::Broker.new.connect(enable_http_api_use: true) do
23
+ # # will disconnect after this block
24
+ # end
25
+ #
26
+ # @param [Hash] options The options to connect with
27
+ # @option options [Boolean] :enable_http_api_use
16
28
  def connect(options = {})
17
29
  @options = options
18
30
  set_up_amqp_connection
@@ -41,42 +53,44 @@ module Hutch
41
53
  def disconnect
42
54
  @channel.close if @channel
43
55
  @connection.close if @connection
44
- @channel, @connection, @exchange, @api_client = nil, nil, nil, nil
56
+ @channel = nil
57
+ @connection = nil
58
+ @exchange = nil
59
+ @api_client = nil
45
60
  end
46
61
 
47
- # Connect to RabbitMQ via AMQP. This sets up the main connection and
48
- # channel we use for talking to RabbitMQ. It also ensures the existance of
49
- # the exchange we'll be using.
62
+ # Connect to RabbitMQ via AMQP
63
+ #
64
+ # This sets up the main connection and channel we use for talking to
65
+ # RabbitMQ. It also ensures the existence of the exchange we'll be using.
50
66
  def set_up_amqp_connection
51
67
  open_connection!
52
68
  open_channel!
53
-
54
- exchange_name = @config[:mq_exchange]
55
- exchange_options = { durable: true }.merge @config[:mq_exchange_options]
56
- logger.info "using topic exchange '#{exchange_name}'"
57
-
58
- with_bunny_precondition_handler('exchange') do
59
- @exchange = @channel.topic(exchange_name, exchange_options)
60
- end
69
+ declare_exchange!
70
+ declare_publisher!
61
71
  end
62
72
 
63
- def open_connection!
73
+ def open_connection
64
74
  logger.info "connecting to rabbitmq (#{sanitized_uri})"
65
75
 
66
- @connection = Hutch::Adapter.new(connection_params)
76
+ connection = Hutch::Adapter.new(connection_params)
67
77
 
68
78
  with_bunny_connection_handler(sanitized_uri) do
69
- @connection.start
79
+ connection.start
70
80
  end
71
81
 
72
82
  logger.info "connected to RabbitMQ at #{connection_params[:host]} as #{connection_params[:username]}"
73
- @connection
83
+ connection
74
84
  end
75
85
 
76
- def open_channel!
77
- logger.info "opening rabbitmq channel with pool size #{consumer_pool_size}"
78
- @channel = @connection.create_channel(nil, consumer_pool_size).tap do |ch|
79
- @connection.prefetch_channel(ch, @config[:channel_prefetch])
86
+ def open_connection!
87
+ @connection = open_connection
88
+ end
89
+
90
+ def open_channel
91
+ logger.info "opening rabbitmq channel with pool size #{consumer_pool_size}, abort on exception #{consumer_pool_abort_on_exception}"
92
+ connection.create_channel(nil, consumer_pool_size, consumer_pool_abort_on_exception).tap do |ch|
93
+ connection.prefetch_channel(ch, @config[:channel_prefetch])
80
94
  if @config[:publisher_confirms] || @config[:force_publisher_confirms]
81
95
  logger.info 'enabling publisher confirms'
82
96
  ch.confirm_select
@@ -84,6 +98,28 @@ module Hutch
84
98
  end
85
99
  end
86
100
 
101
+ def open_channel!
102
+ @channel = open_channel
103
+ end
104
+
105
+ def declare_exchange(ch = channel)
106
+ exchange_name = @config[:mq_exchange]
107
+ exchange_options = { durable: true }.merge(@config[:mq_exchange_options])
108
+ logger.info "using topic exchange '#{exchange_name}'"
109
+
110
+ with_bunny_precondition_handler('exchange') do
111
+ ch.topic(exchange_name, exchange_options)
112
+ end
113
+ end
114
+
115
+ def declare_exchange!(*args)
116
+ @exchange = declare_exchange(*args)
117
+ end
118
+
119
+ def declare_publisher!
120
+ @publisher = Hutch::Publisher.new(connection, channel, exchange, @config)
121
+ end
122
+
87
123
  # Set up the connection to the RabbitMQ management API. Unfortunately, this
88
124
  # is necessary to do a few things that are impossible over AMQP. E.g.
89
125
  # listing queues and bindings.
@@ -119,7 +155,7 @@ module Hutch
119
155
  def queue(name, arguments = {})
120
156
  with_bunny_precondition_handler('queue') do
121
157
  namespace = @config[:namespace].to_s.downcase.gsub(/[^-_:\.\w]/, "")
122
- name = name.prepend(namespace + ":") unless namespace.empty?
158
+ name = name.prepend(namespace + ":") if namespace.present?
123
159
  channel.queue(name, durable: true, arguments: arguments)
124
160
  end
125
161
  end
@@ -127,7 +163,7 @@ module Hutch
127
163
  # Return a mapping of queue names to the routing keys they're bound to.
128
164
  def bindings
129
165
  results = Hash.new { |hash, key| hash[key] = [] }
130
- @api_client.bindings.each do |binding|
166
+ api_client.bindings.each do |binding|
131
167
  next if binding['destination'] == binding['routing_key']
132
168
  next unless binding['source'] == @config[:mq_exchange]
133
169
  next unless binding['vhost'] == @config[:mq_vhost]
@@ -136,37 +172,32 @@ module Hutch
136
172
  results
137
173
  end
138
174
 
175
+ # Find the existing bindings, and unbind any redundant bindings
176
+ def unbind_redundant_bindings(queue, routing_keys)
177
+ return unless http_api_use_enabled?
178
+
179
+ bindings.each do |dest, keys|
180
+ next unless dest == queue.name
181
+ keys.reject { |key| routing_keys.include?(key) }.each do |key|
182
+ logger.debug "removing redundant binding #{queue.name} <--> #{key}"
183
+ queue.unbind(exchange, routing_key: key)
184
+ end
185
+ end
186
+ end
187
+
139
188
  # Bind a queue to the broker's exchange on the routing keys provided. Any
140
189
  # existing bindings on the queue that aren't present in the array of
141
190
  # routing keys will be unbound.
142
191
  def bind_queue(queue, routing_keys)
143
- if http_api_use_enabled?
144
- # Find the existing bindings, and unbind any redundant bindings
145
- queue_bindings = bindings.select { |dest, keys| dest == queue.name }
146
- queue_bindings.each do |dest, keys|
147
- keys.reject { |key| routing_keys.include?(key) }.each do |key|
148
- logger.debug "removing redundant binding #{queue.name} <--> #{key}"
149
- queue.unbind(@exchange, routing_key: key)
150
- end
151
- end
152
- end
192
+ unbind_redundant_bindings(queue, routing_keys)
153
193
 
154
194
  # Ensure all the desired bindings are present
155
195
  routing_keys.each do |routing_key|
156
196
  logger.debug "creating binding #{queue.name} <--> #{routing_key}"
157
- queue.bind(@exchange, routing_key: routing_key)
197
+ queue.bind(exchange, routing_key: routing_key)
158
198
  end
159
199
  end
160
200
 
161
- # Each subscriber is run in a thread. This calls Thread#join on each of the
162
- # subscriber threads.
163
- def wait_on_threads(timeout)
164
- # Thread#join returns nil when the timeout is hit. If any return nil,
165
- # the threads didn't all join so we return false.
166
- per_thread_timeout = timeout.to_f / work_pool_threads.length
167
- work_pool_threads.none? { |thread| thread.join(per_thread_timeout).nil? }
168
- end
169
-
170
201
  def stop
171
202
  if defined?(JRUBY_VERSION)
172
203
  channel.close
@@ -181,78 +212,39 @@ module Hutch
181
212
  end
182
213
 
183
214
  def requeue(delivery_tag)
184
- @channel.reject(delivery_tag, true)
215
+ channel.reject(delivery_tag, true)
185
216
  end
186
217
 
187
218
  def reject(delivery_tag, requeue=false)
188
- @channel.reject(delivery_tag, requeue)
219
+ channel.reject(delivery_tag, requeue)
189
220
  end
190
221
 
191
222
  def ack(delivery_tag)
192
- @channel.ack(delivery_tag, false)
223
+ channel.ack(delivery_tag, false)
193
224
  end
194
225
 
195
226
  def nack(delivery_tag)
196
- @channel.nack(delivery_tag, false, false)
227
+ channel.nack(delivery_tag, false, false)
197
228
  end
198
229
 
199
- def publish(routing_key, message, properties = {}, options = {})
200
- ensure_connection!(routing_key, message)
201
-
202
- serializer = options[:serializer] || @config[:serializer]
203
-
204
- non_overridable_properties = {
205
- routing_key: routing_key,
206
- timestamp: @connection.current_timestamp,
207
- content_type: serializer.content_type,
208
- }
209
- properties[:message_id] ||= generate_id
210
-
211
- payload = serializer.encode(message)
212
- logger.info {
213
- spec =
214
- if serializer.binary?
215
- "#{payload.bytesize} bytes message"
216
- else
217
- "message '#{payload}'"
218
- end
219
- "publishing #{spec} to #{routing_key}"
220
- }
221
-
222
- response = @exchange.publish(payload, {persistent: true}.
223
- merge(properties).
224
- merge(global_properties).
225
- merge(non_overridable_properties))
226
-
227
- channel.wait_for_confirms if @config[:force_publisher_confirms]
228
- response
230
+ def publish(*args)
231
+ @publisher.publish(*args)
229
232
  end
230
233
 
231
234
  def confirm_select(*args)
232
- @channel.confirm_select(*args)
235
+ channel.confirm_select(*args)
233
236
  end
234
237
 
235
238
  def wait_for_confirms
236
- @channel.wait_for_confirms
239
+ channel.wait_for_confirms
237
240
  end
238
241
 
242
+ # @return [Boolean] True if channel is set up to use publisher confirmations.
239
243
  def using_publisher_confirmations?
240
- @channel.using_publisher_confirmations?
244
+ channel.using_publisher_confirmations?
241
245
  end
242
246
 
243
247
  private
244
-
245
- def raise_publish_error(reason, routing_key, message)
246
- msg = "unable to publish - #{reason}. Message: #{JSON.dump(message)}, Routing key: #{routing_key}."
247
- logger.error(msg)
248
- raise PublishError, msg
249
- end
250
-
251
- def ensure_connection!(routing_key, message)
252
- raise_publish_error('no connection to broker', routing_key, message) unless @connection
253
- raise_publish_error('connection is closed', routing_key, message) unless @connection.open?
254
- end
255
-
256
248
  def api_config
257
249
  @api_config ||= OpenStruct.new.tap do |config|
258
250
  config.host = @config[:mq_api_host]
@@ -271,11 +263,7 @@ module Hutch
271
263
  {}.tap do |params|
272
264
  params[:host] = @config[:mq_host]
273
265
  params[:port] = @config[:mq_port]
274
- params[:vhost] = if @config[:mq_vhost] && "" != @config[:mq_vhost]
275
- @config[:mq_vhost]
276
- else
277
- Hutch::Adapter::DEFAULT_VHOST
278
- end
266
+ params[:vhost] = @config[:mq_vhost].presence || Hutch::Adapter::DEFAULT_VHOST
279
267
  params[:username] = @config[:mq_username]
280
268
  params[:password] = @config[:mq_password]
281
269
  params[:tls] = @config[:mq_tls]
@@ -299,7 +287,7 @@ module Hutch
299
287
  end
300
288
 
301
289
  def parse_uri
302
- return unless @config[:uri] && !@config[:uri].empty?
290
+ return if @config[:uri].blank?
303
291
 
304
292
  u = URI.parse(@config[:uri])
305
293
 
@@ -351,25 +339,16 @@ module Hutch
351
339
  raise ConnectionError.new("couldn't connect to rabbitmq at #{uri}. Check your configuration, network connectivity and RabbitMQ logs.")
352
340
  end
353
341
 
354
- def work_pool_threads
355
- channel_work_pool.threads || []
356
- end
357
-
358
342
  def channel_work_pool
359
- @channel.work_pool
343
+ channel.work_pool
360
344
  end
361
345
 
362
346
  def consumer_pool_size
363
347
  @config[:consumer_pool_size]
364
348
  end
365
349
 
366
- def generate_id
367
- SecureRandom.uuid
350
+ def consumer_pool_abort_on_exception
351
+ @config[:consumer_pool_abort_on_exception]
368
352
  end
369
-
370
- def global_properties
371
- Hutch.global_properties.respond_to?(:call) ? Hutch.global_properties.call : Hutch.global_properties
372
- end
373
-
374
353
  end
375
354
  end