hutch 0.19.0-java

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +7 -0
  3. data/.travis.yml +11 -0
  4. data/CHANGELOG.md +438 -0
  5. data/Gemfile +22 -0
  6. data/Guardfile +5 -0
  7. data/LICENSE +22 -0
  8. data/README.md +317 -0
  9. data/Rakefile +14 -0
  10. data/bin/hutch +8 -0
  11. data/circle.yml +3 -0
  12. data/examples/consumer.rb +13 -0
  13. data/examples/producer.rb +10 -0
  14. data/hutch.gemspec +30 -0
  15. data/lib/hutch.rb +62 -0
  16. data/lib/hutch/adapter.rb +11 -0
  17. data/lib/hutch/adapters/bunny.rb +33 -0
  18. data/lib/hutch/adapters/march_hare.rb +37 -0
  19. data/lib/hutch/broker.rb +374 -0
  20. data/lib/hutch/cli.rb +205 -0
  21. data/lib/hutch/config.rb +125 -0
  22. data/lib/hutch/consumer.rb +75 -0
  23. data/lib/hutch/error_handlers.rb +8 -0
  24. data/lib/hutch/error_handlers/airbrake.rb +26 -0
  25. data/lib/hutch/error_handlers/honeybadger.rb +28 -0
  26. data/lib/hutch/error_handlers/logger.rb +16 -0
  27. data/lib/hutch/error_handlers/sentry.rb +23 -0
  28. data/lib/hutch/exceptions.rb +7 -0
  29. data/lib/hutch/logging.rb +32 -0
  30. data/lib/hutch/message.rb +31 -0
  31. data/lib/hutch/serializers/identity.rb +19 -0
  32. data/lib/hutch/serializers/json.rb +22 -0
  33. data/lib/hutch/tracers.rb +6 -0
  34. data/lib/hutch/tracers/newrelic.rb +19 -0
  35. data/lib/hutch/tracers/null_tracer.rb +15 -0
  36. data/lib/hutch/version.rb +4 -0
  37. data/lib/hutch/worker.rb +143 -0
  38. data/spec/hutch/broker_spec.rb +377 -0
  39. data/spec/hutch/cli_spec.rb +80 -0
  40. data/spec/hutch/config_spec.rb +126 -0
  41. data/spec/hutch/consumer_spec.rb +130 -0
  42. data/spec/hutch/error_handlers/airbrake_spec.rb +34 -0
  43. data/spec/hutch/error_handlers/honeybadger_spec.rb +36 -0
  44. data/spec/hutch/error_handlers/logger_spec.rb +15 -0
  45. data/spec/hutch/error_handlers/sentry_spec.rb +20 -0
  46. data/spec/hutch/logger_spec.rb +28 -0
  47. data/spec/hutch/message_spec.rb +38 -0
  48. data/spec/hutch/serializers/json_spec.rb +17 -0
  49. data/spec/hutch/worker_spec.rb +99 -0
  50. data/spec/hutch_spec.rb +87 -0
  51. data/spec/spec_helper.rb +40 -0
  52. metadata +194 -0
data/lib/hutch.rb ADDED
@@ -0,0 +1,62 @@
1
+ require 'hutch/adapter'
2
+ require 'hutch/consumer'
3
+ require 'hutch/worker'
4
+ require 'hutch/broker'
5
+ require 'hutch/logging'
6
+ require 'hutch/serializers/identity'
7
+ require 'hutch/serializers/json'
8
+ require 'hutch/config'
9
+ require 'hutch/message'
10
+ require 'hutch/cli'
11
+ require 'hutch/version'
12
+ require 'hutch/error_handlers'
13
+ require 'hutch/exceptions'
14
+ require 'hutch/tracers'
15
+
16
+ module Hutch
17
+
18
+ def self.register_consumer(consumer)
19
+ self.consumers << consumer
20
+ end
21
+
22
+ def self.consumers
23
+ @consumers ||= []
24
+ end
25
+
26
+ def self.logger
27
+ Hutch::Logging.logger
28
+ end
29
+
30
+ def self.global_properties=(properties)
31
+ @global_properties = properties
32
+ end
33
+
34
+ def self.global_properties
35
+ @global_properties ||= {}
36
+ end
37
+
38
+ def self.connect(options = {}, config = Hutch::Config)
39
+ unless connected?
40
+ @broker = Hutch::Broker.new(config)
41
+ @broker.connect(options)
42
+ end
43
+ end
44
+
45
+ def self.disconnect
46
+ @broker.disconnect if @broker
47
+ end
48
+
49
+ def self.broker
50
+ @broker
51
+ end
52
+
53
+ def self.connected?
54
+ return false unless broker
55
+ return false unless broker.connection
56
+ broker.connection.open?
57
+ end
58
+
59
+ def self.publish(*args)
60
+ broker.publish(*args)
61
+ end
62
+ end
@@ -0,0 +1,11 @@
1
+ if defined?(JRUBY_VERSION)
2
+ require 'hutch/adapters/march_hare'
3
+ module Hutch
4
+ Adapter = Adapters::MarchHareAdapter
5
+ end
6
+ else
7
+ require 'hutch/adapters/bunny'
8
+ module Hutch
9
+ Adapter = Adapters::BunnyAdapter
10
+ end
11
+ end
@@ -0,0 +1,33 @@
1
+ require 'bunny'
2
+ require 'forwardable'
3
+
4
+ module Hutch
5
+ module Adapters
6
+ class BunnyAdapter
7
+ extend Forwardable
8
+
9
+ DEFAULT_VHOST = Bunny::Session::DEFAULT_VHOST
10
+
11
+ ConnectionRefused = Bunny::TCPConnectionFailed
12
+ PreconditionFailed = Bunny::PreconditionFailed
13
+
14
+ def_delegators :@connection, :start, :disconnect, :close, :create_channel, :open?
15
+
16
+ def initialize(opts={})
17
+ @connection = Bunny.new(opts)
18
+ end
19
+
20
+ def self.decode_message(delivery_info, properties, payload)
21
+ [delivery_info, properties, payload]
22
+ end
23
+
24
+ def prefetch_channel(ch, prefetch)
25
+ ch.prefetch(prefetch) if prefetch
26
+ end
27
+
28
+ def current_timestamp
29
+ Time.now.to_i
30
+ end
31
+ end
32
+ end
33
+ end
@@ -0,0 +1,37 @@
1
+ require 'march_hare'
2
+ require 'forwardable'
3
+
4
+ module Hutch
5
+ module Adapters
6
+ class MarchHareAdapter
7
+ extend Forwardable
8
+
9
+ DEFAULT_VHOST = "/"
10
+
11
+ ConnectionRefused = MarchHare::ConnectionRefused
12
+ PreconditionFailed = MarchHare::PreconditionFailed
13
+
14
+ def_delegators :@connection, :start, :disconnect, :close, :open?
15
+
16
+ def initialize(opts = {})
17
+ @connection = MarchHare.connect(opts)
18
+ end
19
+
20
+ def self.decode_message(delivery_info, payload)
21
+ [delivery_info, delivery_info.properties, payload]
22
+ end
23
+
24
+ def prefetch_channel(ch, prefetch)
25
+ ch.prefetch = prefetch if prefetch
26
+ end
27
+
28
+ def create_channel(n = nil, consumer_pool_size = 1)
29
+ @connection.create_channel(n)
30
+ end
31
+
32
+ def current_timestamp
33
+ Time.now
34
+ end
35
+ end
36
+ end
37
+ end
@@ -0,0 +1,374 @@
1
+ require 'carrot-top'
2
+ require 'securerandom'
3
+ require 'hutch/logging'
4
+ require 'hutch/exceptions'
5
+
6
+ module Hutch
7
+ class Broker
8
+ include Logging
9
+
10
+ attr_accessor :connection, :channel, :exchange, :api_client
11
+
12
+ def initialize(config = nil)
13
+ @config = config || Hutch::Config
14
+ end
15
+
16
+ def connect(options = {})
17
+ @options = options
18
+ set_up_amqp_connection
19
+ if http_api_use_enabled?
20
+ logger.info "HTTP API use is enabled"
21
+ set_up_api_connection
22
+ else
23
+ logger.info "HTTP API use is disabled"
24
+ end
25
+
26
+ if tracing_enabled?
27
+ logger.info "tracing is enabled using #{@config[:tracer]}"
28
+ else
29
+ logger.info "tracing is disabled"
30
+ end
31
+
32
+ if block_given?
33
+ begin
34
+ yield
35
+ ensure
36
+ disconnect
37
+ end
38
+ end
39
+ end
40
+
41
+ def disconnect
42
+ @channel.close if @channel
43
+ @connection.close if @connection
44
+ @channel, @connection, @exchange, @api_client = nil, nil, nil, nil
45
+ end
46
+
47
+ # Connect to RabbitMQ via AMQP. This sets up the main connection and
48
+ # channel we use for talking to RabbitMQ. It also ensures the existance of
49
+ # the exchange we'll be using.
50
+ def set_up_amqp_connection
51
+ open_connection!
52
+ open_channel!
53
+
54
+ exchange_name = @config[:mq_exchange]
55
+ logger.info "using topic exchange '#{exchange_name}'"
56
+
57
+ with_bunny_precondition_handler('exchange') do
58
+ @exchange = @channel.topic(exchange_name, durable: true)
59
+ end
60
+ end
61
+
62
+ def open_connection!
63
+ logger.info "connecting to rabbitmq (#{sanitized_uri})"
64
+
65
+ @connection = Hutch::Adapter.new(connection_params)
66
+
67
+ with_bunny_connection_handler(sanitized_uri) do
68
+ @connection.start
69
+ end
70
+
71
+ logger.info "connected to RabbitMQ at #{connection_params[:host]} as #{connection_params[:username]}"
72
+ @connection
73
+ end
74
+
75
+ def open_channel!
76
+ logger.info "opening rabbitmq channel with pool size #{consumer_pool_size}"
77
+ @channel = @connection.create_channel(nil, consumer_pool_size).tap do |ch|
78
+ @connection.prefetch_channel(ch, @config[:channel_prefetch])
79
+ if @config[:publisher_confirms] || @config[:force_publisher_confirms]
80
+ logger.info 'enabling publisher confirms'
81
+ ch.confirm_select
82
+ end
83
+ end
84
+ end
85
+
86
+ # Set up the connection to the RabbitMQ management API. Unfortunately, this
87
+ # is necessary to do a few things that are impossible over AMQP. E.g.
88
+ # listing queues and bindings.
89
+ def set_up_api_connection
90
+ logger.info "connecting to rabbitmq HTTP API (#{api_config.sanitized_uri})"
91
+
92
+ with_authentication_error_handler do
93
+ with_connection_error_handler do
94
+ @api_client = CarrotTop.new(host: api_config.host, port: api_config.port,
95
+ user: api_config.username, password: api_config.password,
96
+ ssl: api_config.ssl)
97
+ @api_client.exchanges
98
+ end
99
+ end
100
+ end
101
+
102
+ def http_api_use_enabled?
103
+ op = @options.fetch(:enable_http_api_use, true)
104
+ cf = if @config[:enable_http_api_use].nil?
105
+ true
106
+ else
107
+ @config[:enable_http_api_use]
108
+ end
109
+
110
+ op && cf
111
+ end
112
+
113
+ def tracing_enabled?
114
+ @config[:tracer] && @config[:tracer] != Hutch::Tracers::NullTracer
115
+ end
116
+
117
+ # Create / get a durable queue and apply namespace if it exists.
118
+ def queue(name, arguments = {})
119
+ with_bunny_precondition_handler('queue') do
120
+ namespace = @config[:namespace].to_s.downcase.gsub(/[^-_:\.\w]/, "")
121
+ name = name.prepend(namespace + ":") unless namespace.empty?
122
+ channel.queue(name, durable: true, arguments: arguments)
123
+ end
124
+ end
125
+
126
+ # Return a mapping of queue names to the routing keys they're bound to.
127
+ def bindings
128
+ results = Hash.new { |hash, key| hash[key] = [] }
129
+ @api_client.bindings.each do |binding|
130
+ next if binding['destination'] == binding['routing_key']
131
+ next unless binding['source'] == @config[:mq_exchange]
132
+ next unless binding['vhost'] == @config[:mq_vhost]
133
+ results[binding['destination']] << binding['routing_key']
134
+ end
135
+ results
136
+ end
137
+
138
+ # Bind a queue to the broker's exchange on the routing keys provided. Any
139
+ # existing bindings on the queue that aren't present in the array of
140
+ # routing keys will be unbound.
141
+ def bind_queue(queue, routing_keys)
142
+ if http_api_use_enabled?
143
+ # Find the existing bindings, and unbind any redundant bindings
144
+ queue_bindings = bindings.select { |dest, keys| dest == queue.name }
145
+ queue_bindings.each do |dest, keys|
146
+ keys.reject { |key| routing_keys.include?(key) }.each do |key|
147
+ logger.debug "removing redundant binding #{queue.name} <--> #{key}"
148
+ queue.unbind(@exchange, routing_key: key)
149
+ end
150
+ end
151
+ end
152
+
153
+ # Ensure all the desired bindings are present
154
+ routing_keys.each do |routing_key|
155
+ logger.debug "creating binding #{queue.name} <--> #{routing_key}"
156
+ queue.bind(@exchange, routing_key: routing_key)
157
+ end
158
+ end
159
+
160
+ # Each subscriber is run in a thread. This calls Thread#join on each of the
161
+ # subscriber threads.
162
+ def wait_on_threads(timeout)
163
+ # Thread#join returns nil when the timeout is hit. If any return nil,
164
+ # the threads didn't all join so we return false.
165
+ per_thread_timeout = timeout.to_f / work_pool_threads.length
166
+ work_pool_threads.none? { |thread| thread.join(per_thread_timeout).nil? }
167
+ end
168
+
169
+ def stop
170
+ if defined?(JRUBY_VERSION)
171
+ channel.close
172
+ else
173
+ # Enqueue a failing job that kills the consumer loop
174
+ channel_work_pool.shutdown
175
+ # Give `timeout` seconds to jobs that are still being processed
176
+ channel_work_pool.join(@config[:graceful_exit_timeout])
177
+ # If after `timeout` they are still running, they are killed
178
+ channel_work_pool.kill
179
+ end
180
+ end
181
+
182
+ def requeue(delivery_tag)
183
+ @channel.reject(delivery_tag, true)
184
+ end
185
+
186
+ def reject(delivery_tag, requeue=false)
187
+ @channel.reject(delivery_tag, requeue)
188
+ end
189
+
190
+ def ack(delivery_tag)
191
+ @channel.ack(delivery_tag, false)
192
+ end
193
+
194
+ def nack(delivery_tag)
195
+ @channel.nack(delivery_tag, false, false)
196
+ end
197
+
198
+ def publish(routing_key, message, properties = {}, options = {})
199
+ ensure_connection!(routing_key, message)
200
+
201
+ serializer = options[:serializer] || @config[:serializer]
202
+
203
+ non_overridable_properties = {
204
+ routing_key: routing_key,
205
+ timestamp: @connection.current_timestamp,
206
+ content_type: serializer.content_type,
207
+ }
208
+ properties[:message_id] ||= generate_id
209
+
210
+ payload = serializer.encode(message)
211
+ logger.info {
212
+ spec =
213
+ if serializer.binary?
214
+ "#{payload.bytesize} bytes message"
215
+ else
216
+ "message '#{payload}'"
217
+ end
218
+ "publishing #{spec} to #{routing_key}"
219
+ }
220
+
221
+ response = @exchange.publish(payload, {persistent: true}.
222
+ merge(properties).
223
+ merge(global_properties).
224
+ merge(non_overridable_properties))
225
+
226
+ channel.wait_for_confirms if @config[:force_publisher_confirms]
227
+ response
228
+ end
229
+
230
+ def confirm_select(*args)
231
+ @channel.confirm_select(*args)
232
+ end
233
+
234
+ def wait_for_confirms
235
+ @channel.wait_for_confirms
236
+ end
237
+
238
+ def using_publisher_confirmations?
239
+ @channel.using_publisher_confirmations?
240
+ end
241
+
242
+ private
243
+
244
+ def raise_publish_error(reason, routing_key, message)
245
+ msg = "unable to publish - #{reason}. Message: #{JSON.dump(message)}, Routing key: #{routing_key}."
246
+ logger.error(msg)
247
+ raise PublishError, msg
248
+ end
249
+
250
+ def ensure_connection!(routing_key, message)
251
+ raise_publish_error('no connection to broker', routing_key, message) unless @connection
252
+ raise_publish_error('connection is closed', routing_key, message) unless @connection.open?
253
+ end
254
+
255
+ def api_config
256
+ @api_config ||= OpenStruct.new.tap do |config|
257
+ config.host = @config[:mq_api_host]
258
+ config.port = @config[:mq_api_port]
259
+ config.username = @config[:mq_username]
260
+ config.password = @config[:mq_password]
261
+ config.ssl = @config[:mq_api_ssl]
262
+ config.protocol = config.ssl ? "https://" : "http://"
263
+ config.sanitized_uri = "#{config.protocol}#{config.username}@#{config.host}:#{config.port}/"
264
+ end
265
+ end
266
+
267
+ def connection_params
268
+ parse_uri
269
+
270
+ {}.tap do |params|
271
+ params[:host] = @config[:mq_host]
272
+ params[:port] = @config[:mq_port]
273
+ params[:vhost] = if @config[:mq_vhost] && "" != @config[:mq_vhost]
274
+ @config[:mq_vhost]
275
+ else
276
+ Hutch::Adapter::DEFAULT_VHOST
277
+ end
278
+ params[:username] = @config[:mq_username]
279
+ params[:password] = @config[:mq_password]
280
+ params[:tls] = @config[:mq_tls]
281
+ params[:tls_key] = @config[:mq_tls_key]
282
+ params[:tls_cert] = @config[:mq_tls_cert]
283
+ params[:verify_peer] = @config[:mq_verify_peer]
284
+ if @config[:mq_tls_ca_certificates]
285
+ params[:tls_ca_certificates] = @config[:mq_tls_ca_certificates]
286
+ end
287
+ params[:heartbeat] = @config[:heartbeat]
288
+ params[:connection_timeout] = @config[:connection_timeout]
289
+ params[:read_timeout] = @config[:read_timeout]
290
+ params[:write_timeout] = @config[:write_timeout]
291
+
292
+
293
+ params[:automatically_recover] = true
294
+ params[:network_recovery_interval] = 1
295
+
296
+ params[:client_logger] = @config[:client_logger] if @config[:client_logger]
297
+ end
298
+ end
299
+
300
+ def parse_uri
301
+ return unless @config[:uri] && !@config[:uri].empty?
302
+
303
+ u = URI.parse(@config[:uri])
304
+
305
+ @config[:mq_host] = u.host
306
+ @config[:mq_port] = u.port
307
+ @config[:mq_vhost] = u.path.sub(/^\//, "")
308
+ @config[:mq_username] = u.user
309
+ @config[:mq_password] = u.password
310
+ end
311
+
312
+ def sanitized_uri
313
+ p = connection_params
314
+ scheme = p[:tls] ? "amqps" : "amqp"
315
+
316
+ "#{scheme}://#{p[:username]}@#{p[:host]}:#{p[:port]}/#{p[:vhost].sub(/^\//, '')}"
317
+ end
318
+
319
+ def with_authentication_error_handler
320
+ yield
321
+ rescue Net::HTTPServerException => ex
322
+ logger.error "HTTP API connection error: #{ex.message.downcase}"
323
+ if ex.response.code == '401'
324
+ raise AuthenticationError.new('invalid HTTP API credentials')
325
+ else
326
+ raise
327
+ end
328
+ end
329
+
330
+ def with_connection_error_handler
331
+ yield
332
+ rescue Errno::ECONNREFUSED => ex
333
+ logger.error "HTTP API connection error: #{ex.message.downcase}"
334
+ raise ConnectionError.new("couldn't connect to HTTP API at #{api_config.sanitized_uri}")
335
+ end
336
+
337
+ def with_bunny_precondition_handler(item)
338
+ yield
339
+ rescue Hutch::Adapter::PreconditionFailed => ex
340
+ logger.error ex.message
341
+ s = "RabbitMQ responded with 406 Precondition Failed when creating this #{item}. " +
342
+ "Perhaps it is being redeclared with non-matching attributes"
343
+ raise WorkerSetupError.new(s)
344
+ end
345
+
346
+ def with_bunny_connection_handler(uri)
347
+ yield
348
+ rescue Hutch::Adapter::ConnectionRefused => ex
349
+ logger.error "amqp connection error: #{ex.message.downcase}"
350
+ raise ConnectionError.new("couldn't connect to rabbitmq at #{uri}. Check your configuration, network connectivity and RabbitMQ logs.")
351
+ end
352
+
353
+ def work_pool_threads
354
+ channel_work_pool.threads || []
355
+ end
356
+
357
+ def channel_work_pool
358
+ @channel.work_pool
359
+ end
360
+
361
+ def consumer_pool_size
362
+ @config[:consumer_pool_size]
363
+ end
364
+
365
+ def generate_id
366
+ SecureRandom.uuid
367
+ end
368
+
369
+ def global_properties
370
+ Hutch.global_properties.respond_to?(:call) ? Hutch.global_properties.call : Hutch.global_properties
371
+ end
372
+
373
+ end
374
+ end