hutch-java 1.3.0-java
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.github/workflows/test.yml +57 -0
- data/.gitignore +10 -0
- data/.rspec +1 -0
- data/.yardopts +5 -0
- data/CHANGELOG.md +899 -0
- data/Gemfile +35 -0
- data/Guardfile +14 -0
- data/LICENSE +23 -0
- data/README.md +679 -0
- data/Rakefile +21 -0
- data/bin/ci/before_build.sh +20 -0
- data/bin/ci/before_build_docker.sh +20 -0
- data/bin/ci/install_on_debian.sh +46 -0
- data/bin/hutch +8 -0
- data/examples/consumer.rb +13 -0
- data/examples/producer.rb +10 -0
- data/hutch.gemspec +27 -0
- data/lib/hutch/acknowledgements/base.rb +16 -0
- data/lib/hutch/acknowledgements/nack_on_all_failures.rb +19 -0
- data/lib/hutch/adapter.rb +11 -0
- data/lib/hutch/adapters/bunny.rb +37 -0
- data/lib/hutch/adapters/march_hare.rb +41 -0
- data/lib/hutch/broker.rb +383 -0
- data/lib/hutch/cli.rb +246 -0
- data/lib/hutch/config.rb +304 -0
- data/lib/hutch/consumer.rb +125 -0
- data/lib/hutch/error_handlers/airbrake.rb +54 -0
- data/lib/hutch/error_handlers/base.rb +15 -0
- data/lib/hutch/error_handlers/bugsnag.rb +30 -0
- data/lib/hutch/error_handlers/honeybadger.rb +43 -0
- data/lib/hutch/error_handlers/logger.rb +22 -0
- data/lib/hutch/error_handlers/rollbar.rb +28 -0
- data/lib/hutch/error_handlers/sentry.rb +26 -0
- data/lib/hutch/error_handlers/sentry_raven.rb +31 -0
- data/lib/hutch/error_handlers.rb +11 -0
- data/lib/hutch/exceptions.rb +14 -0
- data/lib/hutch/logging.rb +32 -0
- data/lib/hutch/message.rb +31 -0
- data/lib/hutch/publisher.rb +75 -0
- data/lib/hutch/serializers/identity.rb +19 -0
- data/lib/hutch/serializers/json.rb +22 -0
- data/lib/hutch/tracers/datadog.rb +18 -0
- data/lib/hutch/tracers/newrelic.rb +19 -0
- data/lib/hutch/tracers/null_tracer.rb +15 -0
- data/lib/hutch/tracers.rb +7 -0
- data/lib/hutch/version.rb +3 -0
- data/lib/hutch/waiter.rb +104 -0
- data/lib/hutch/worker.rb +145 -0
- data/lib/hutch.rb +69 -0
- data/lib/yard-settings/handler.rb +38 -0
- data/lib/yard-settings/yard-settings.rb +2 -0
- data/spec/hutch/broker_spec.rb +462 -0
- data/spec/hutch/cli_spec.rb +93 -0
- data/spec/hutch/config_spec.rb +259 -0
- data/spec/hutch/consumer_spec.rb +208 -0
- data/spec/hutch/error_handlers/airbrake_spec.rb +49 -0
- data/spec/hutch/error_handlers/bugsnag_spec.rb +55 -0
- data/spec/hutch/error_handlers/honeybadger_spec.rb +58 -0
- data/spec/hutch/error_handlers/logger_spec.rb +28 -0
- data/spec/hutch/error_handlers/rollbar_spec.rb +45 -0
- data/spec/hutch/error_handlers/sentry_raven_spec.rb +37 -0
- data/spec/hutch/error_handlers/sentry_spec.rb +47 -0
- data/spec/hutch/logger_spec.rb +34 -0
- data/spec/hutch/message_spec.rb +38 -0
- data/spec/hutch/serializers/json_spec.rb +17 -0
- data/spec/hutch/tracers/datadog_spec.rb +44 -0
- data/spec/hutch/waiter_spec.rb +51 -0
- data/spec/hutch/worker_spec.rb +184 -0
- data/spec/hutch_spec.rb +87 -0
- data/spec/spec_helper.rb +42 -0
- data/templates/default/class/html/settings.erb +0 -0
- data/templates/default/class/setup.rb +4 -0
- data/templates/default/fulldoc/html/css/hutch.css +13 -0
- data/templates/default/layout/html/setup.rb +7 -0
- data/templates/default/method_details/html/settings.erb +5 -0
- data/templates/default/method_details/setup.rb +4 -0
- data/templates/default/method_details/text/settings.erb +0 -0
- data/templates/default/module/html/settings.erb +40 -0
- data/templates/default/module/setup.rb +4 -0
- metadata +204 -0
@@ -0,0 +1,20 @@
|
|
1
|
+
#!/usr/bin/env sh
|
2
|
+
|
3
|
+
CTL=${BUNNY_RABBITMQCTL:-"sudo rabbitmqctl"}
|
4
|
+
PLUGINS=${BUNNY_RABBITMQ_PLUGINS:-"sudo rabbitmq-plugins"}
|
5
|
+
|
6
|
+
echo "Will use rabbitmqctl at ${CTL}"
|
7
|
+
echo "Will use rabbitmq-plugins at ${PLUGINS}"
|
8
|
+
|
9
|
+
$PLUGINS enable rabbitmq_management
|
10
|
+
|
11
|
+
sleep 3
|
12
|
+
|
13
|
+
# guest:guest has full access to /
|
14
|
+
$CTL add_vhost /
|
15
|
+
$CTL add_user guest guest
|
16
|
+
$CTL set_permissions -p / guest ".*" ".*" ".*"
|
17
|
+
|
18
|
+
# Reduce retention policy for faster publishing of stats
|
19
|
+
$CTL eval 'supervisor2:terminate_child(rabbit_mgmt_sup_sup, rabbit_mgmt_sup), application:set_env(rabbitmq_management, sample_retention_policies, [{global, [{605, 1}]}, {basic, [{605, 1}]}, {detailed, [{10, 1}]}]), rabbit_mgmt_sup_sup:start_child().' || true
|
20
|
+
$CTL eval 'supervisor2:terminate_child(rabbit_mgmt_agent_sup_sup, rabbit_mgmt_agent_sup), application:set_env(rabbitmq_management_agent, sample_retention_policies, [{global, [{605, 1}]}, {basic, [{605, 1}]}, {detailed, [{10, 1}]}]), rabbit_mgmt_agent_sup_sup:start_child().' || true
|
@@ -0,0 +1,20 @@
|
|
1
|
+
#!/bin/sh
|
2
|
+
|
3
|
+
CTL=${MARCH_HARE_RABBITMQCTL:="docker exec rabbitmq rabbitmqctl"}
|
4
|
+
PLUGINS=${MARCH_HARE_RABBITMQ_PLUGINS:="docker exec rabbitmq rabbitmq-plugins"}
|
5
|
+
|
6
|
+
$PLUGINS enable rabbitmq_management
|
7
|
+
|
8
|
+
sleep 3
|
9
|
+
|
10
|
+
# guest:guest has full access to /
|
11
|
+
|
12
|
+
$CTL add_vhost /
|
13
|
+
# $CTL add_user guest guest # already exists
|
14
|
+
$CTL set_permissions -p / guest ".*" ".*" ".*"
|
15
|
+
|
16
|
+
# Reduce retention policy for faster publishing of stats
|
17
|
+
$CTL eval 'supervisor2:terminate_child(rabbit_mgmt_sup_sup, rabbit_mgmt_sup), application:set_env(rabbitmq_management, sample_retention_policies, [{global, [{605, 1}]}, {basic, [{605, 1}]}, {detailed, [{10, 1}]}]), rabbit_mgmt_sup_sup:start_child().'
|
18
|
+
$CTL eval 'supervisor2:terminate_child(rabbit_mgmt_agent_sup_sup, rabbit_mgmt_agent_sup), application:set_env(rabbitmq_management_agent, sample_retention_policies, [{global, [{605, 1}]}, {basic, [{605, 1}]}, {detailed, [{10, 1}]}]), rabbit_mgmt_agent_sup_sup:start_child().'
|
19
|
+
|
20
|
+
sleep 3
|
@@ -0,0 +1,46 @@
|
|
1
|
+
#!/bin/sh
|
2
|
+
|
3
|
+
sudo apt-get install curl gnupg debian-keyring debian-archive-keyring apt-transport-https -y
|
4
|
+
|
5
|
+
## Team RabbitMQ's main signing key
|
6
|
+
sudo apt-key adv --keyserver "hkps://keys.openpgp.org" --recv-keys "0x0A9AF2115F4687BD29803A206B73A36E6026DFCA"
|
7
|
+
## Launchpad PPA that provides modern Erlang releases
|
8
|
+
sudo apt-key adv --keyserver "keyserver.ubuntu.com" --recv-keys "F77F1EDA57EBB1CC"
|
9
|
+
## PackageCloud RabbitMQ repository
|
10
|
+
sudo apt-key adv --keyserver "keyserver.ubuntu.com" --recv-keys "F6609E60DC62814E"
|
11
|
+
|
12
|
+
## Add apt repositories maintained by Team RabbitMQ
|
13
|
+
sudo tee /etc/apt/sources.list.d/rabbitmq.list <<EOF
|
14
|
+
## Provides modern Erlang/OTP releases
|
15
|
+
##
|
16
|
+
## "bionic" as distribution name should work for any reasonably recent Ubuntu or Debian release.
|
17
|
+
## See the release to distribution mapping table in RabbitMQ doc guides to learn more.
|
18
|
+
deb http://ppa.launchpad.net/rabbitmq/rabbitmq-erlang/ubuntu bionic main
|
19
|
+
deb-src http://ppa.launchpad.net/rabbitmq/rabbitmq-erlang/ubuntu bionic main
|
20
|
+
|
21
|
+
## Provides RabbitMQ
|
22
|
+
##
|
23
|
+
## "bionic" as distribution name should work for any reasonably recent Ubuntu or Debian release.
|
24
|
+
## See the release to distribution mapping table in RabbitMQ doc guides to learn more.
|
25
|
+
deb https://packagecloud.io/rabbitmq/rabbitmq-server/ubuntu/ bionic main
|
26
|
+
deb-src https://packagecloud.io/rabbitmq/rabbitmq-server/ubuntu/ bionic main
|
27
|
+
EOF
|
28
|
+
|
29
|
+
## Update package indices
|
30
|
+
sudo apt-get update -y
|
31
|
+
|
32
|
+
## Install Erlang packages
|
33
|
+
sudo apt-get install -y erlang-base \
|
34
|
+
erlang-asn1 erlang-crypto erlang-eldap erlang-ftp erlang-inets \
|
35
|
+
erlang-mnesia erlang-os-mon erlang-parsetools erlang-public-key \
|
36
|
+
erlang-runtime-tools erlang-snmp erlang-ssl \
|
37
|
+
erlang-syntax-tools erlang-tftp erlang-tools erlang-xmerl
|
38
|
+
|
39
|
+
## Install rabbitmq-server and its dependencies
|
40
|
+
sudo apt-get install rabbitmq-server -y --fix-missing
|
41
|
+
|
42
|
+
sudo service rabbitmq-server start
|
43
|
+
|
44
|
+
sudo rabbitmqctl await_startup --timeout 120
|
45
|
+
|
46
|
+
until sudo lsof -i:5672; do echo "Waiting for RabbitMQ to start..."; sleep 1; done
|
data/bin/hutch
ADDED
data/hutch.gemspec
ADDED
@@ -0,0 +1,27 @@
|
|
1
|
+
require File.expand_path('../lib/hutch/version', __FILE__)
|
2
|
+
|
3
|
+
Gem::Specification.new do |gem|
|
4
|
+
if defined?(JRUBY_VERSION)
|
5
|
+
gem.platform = 'java'
|
6
|
+
gem.add_runtime_dependency 'march_hare', '>= 3.0.0'
|
7
|
+
else
|
8
|
+
gem.platform = Gem::Platform::RUBY
|
9
|
+
gem.add_runtime_dependency 'bunny', '>= 2.20', '< 3.0'
|
10
|
+
end
|
11
|
+
gem.add_runtime_dependency 'carrot-top', '~> 0.0.7'
|
12
|
+
gem.add_runtime_dependency 'multi_json', '~> 1.15'
|
13
|
+
gem.add_runtime_dependency 'activesupport', '>= 4.2', '< 8'
|
14
|
+
|
15
|
+
gem.name = 'hutch-java'
|
16
|
+
gem.summary = 'Opinionated asynchronous inter-service communication using RabbitMQ'
|
17
|
+
gem.description = 'Hutch is a Ruby library for enabling asynchronous inter-service communication using RabbitMQ'
|
18
|
+
gem.version = Hutch::VERSION.dup
|
19
|
+
gem.required_ruby_version = '>= 2.6'
|
20
|
+
gem.authors = ['Shivam Shah']
|
21
|
+
gem.homepage = 'https://github.com/nulllvoid/hutch'
|
22
|
+
gem.require_paths = ['lib']
|
23
|
+
gem.license = 'MIT'
|
24
|
+
gem.executables = ['hutch']
|
25
|
+
gem.files = `git ls-files`.split("\n")
|
26
|
+
gem.test_files = `git ls-files -- spec/*`.split("\n")
|
27
|
+
end
|
@@ -0,0 +1,16 @@
|
|
1
|
+
module Hutch
|
2
|
+
module Acknowledgements
|
3
|
+
# Defines acknowledgement handler interface.
|
4
|
+
class Base
|
5
|
+
# Implements negative acknowledgement/requeueing logic
|
6
|
+
# and returns a boolean to indicate whether acknowledgement
|
7
|
+
# was performed. If false is returned, next handler in the
|
8
|
+
# chain will be invoked.
|
9
|
+
#
|
10
|
+
# The chain always falls back to unconditional nacking.
|
11
|
+
def handle(delivery_info, properties, broker, ex)
|
12
|
+
raise NotImplementedError.new
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
require 'hutch/logging'
|
2
|
+
require 'hutch/acknowledgements/base'
|
3
|
+
|
4
|
+
module Hutch
|
5
|
+
module Acknowledgements
|
6
|
+
class NackOnAllFailures < Base
|
7
|
+
include Logging
|
8
|
+
|
9
|
+
def handle(delivery_info, properties, broker, ex)
|
10
|
+
prefix = "message(#{properties.message_id || '-'}): "
|
11
|
+
logger.debug "#{prefix} nacking message"
|
12
|
+
|
13
|
+
broker.nack(delivery_info.delivery_tag)
|
14
|
+
|
15
|
+
true
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
@@ -0,0 +1,37 @@
|
|
1
|
+
require 'bunny'
|
2
|
+
require 'forwardable'
|
3
|
+
|
4
|
+
module Hutch
|
5
|
+
module Adapters
|
6
|
+
class BunnyAdapter
|
7
|
+
extend Forwardable
|
8
|
+
|
9
|
+
DEFAULT_VHOST = Bunny::Session::DEFAULT_VHOST
|
10
|
+
|
11
|
+
ConnectionRefused = Bunny::TCPConnectionFailed
|
12
|
+
PreconditionFailed = Bunny::PreconditionFailed
|
13
|
+
|
14
|
+
def_delegators :@connection, :start, :disconnect, :close, :create_channel, :open?
|
15
|
+
|
16
|
+
def initialize(opts={})
|
17
|
+
@connection = Bunny.new(opts)
|
18
|
+
end
|
19
|
+
|
20
|
+
def self.decode_message(delivery_info, properties, payload)
|
21
|
+
[delivery_info, properties, payload]
|
22
|
+
end
|
23
|
+
|
24
|
+
def prefetch_channel(ch, prefetch)
|
25
|
+
ch.prefetch(prefetch) if prefetch
|
26
|
+
end
|
27
|
+
|
28
|
+
def current_timestamp
|
29
|
+
Time.now.to_i
|
30
|
+
end
|
31
|
+
|
32
|
+
def self.new_exchange(ch, exchange_type, exchange_name, exchange_options)
|
33
|
+
Bunny::Exchange.new(ch, exchange_type, exchange_name, exchange_options)
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
@@ -0,0 +1,41 @@
|
|
1
|
+
require 'march_hare'
|
2
|
+
require 'forwardable'
|
3
|
+
|
4
|
+
module Hutch
|
5
|
+
module Adapters
|
6
|
+
class MarchHareAdapter
|
7
|
+
extend Forwardable
|
8
|
+
|
9
|
+
DEFAULT_VHOST = "/"
|
10
|
+
|
11
|
+
ConnectionRefused = MarchHare::ConnectionRefused
|
12
|
+
PreconditionFailed = MarchHare::PreconditionFailed
|
13
|
+
|
14
|
+
def_delegators :@connection, :start, :disconnect, :close, :open?
|
15
|
+
|
16
|
+
def initialize(opts = {})
|
17
|
+
@connection = MarchHare.connect(opts)
|
18
|
+
end
|
19
|
+
|
20
|
+
def self.decode_message(delivery_info, payload)
|
21
|
+
[delivery_info, delivery_info.properties, payload]
|
22
|
+
end
|
23
|
+
|
24
|
+
def prefetch_channel(ch, prefetch)
|
25
|
+
ch.prefetch = prefetch if prefetch
|
26
|
+
end
|
27
|
+
|
28
|
+
def create_channel(n = nil, consumer_pool_size = 1, consumer_pool_abort_on_exception = false)
|
29
|
+
@connection.create_channel(n)
|
30
|
+
end
|
31
|
+
|
32
|
+
def current_timestamp
|
33
|
+
Time.now
|
34
|
+
end
|
35
|
+
|
36
|
+
def self.new_exchange(ch, exchange_type, exchange_name, exchange_options)
|
37
|
+
MarchHare::Exchange.new(ch, exchange_name, exchange_options.merge(type: exchange_type))
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
data/lib/hutch/broker.rb
ADDED
@@ -0,0 +1,383 @@
|
|
1
|
+
require 'active_support/core_ext/object/blank'
|
2
|
+
|
3
|
+
require 'carrot-top'
|
4
|
+
require 'hutch/logging'
|
5
|
+
require 'hutch/exceptions'
|
6
|
+
require 'hutch/publisher'
|
7
|
+
|
8
|
+
module Hutch
|
9
|
+
class Broker
|
10
|
+
include Logging
|
11
|
+
|
12
|
+
attr_accessor :connection, :channel, :exchange, :api_client
|
13
|
+
|
14
|
+
|
15
|
+
DEFAULT_AMQP_PORT =
|
16
|
+
case RUBY_ENGINE
|
17
|
+
when "jruby" then
|
18
|
+
com.rabbitmq.client.ConnectionFactory::DEFAULT_AMQP_PORT
|
19
|
+
when "ruby" then
|
20
|
+
AMQ::Protocol::DEFAULT_PORT
|
21
|
+
end
|
22
|
+
|
23
|
+
DEFAULT_AMQPS_PORT =
|
24
|
+
case RUBY_ENGINE
|
25
|
+
when "jruby" then
|
26
|
+
com.rabbitmq.client.ConnectionFactory::DEFAULT_AMQP_OVER_SSL_PORT
|
27
|
+
when "ruby" then
|
28
|
+
AMQ::Protocol::TLS_PORT
|
29
|
+
end
|
30
|
+
|
31
|
+
|
32
|
+
# @param config [nil,Hash] Configuration override
|
33
|
+
def initialize(config = nil)
|
34
|
+
@config = config || Hutch::Config
|
35
|
+
end
|
36
|
+
|
37
|
+
# Connect to broker
|
38
|
+
#
|
39
|
+
# @example
|
40
|
+
# Hutch::Broker.new.connect(enable_http_api_use: true) do
|
41
|
+
# # will disconnect after this block
|
42
|
+
# end
|
43
|
+
#
|
44
|
+
# @param [Hash] options The options to connect with
|
45
|
+
# @option options [Boolean] :enable_http_api_use
|
46
|
+
def connect(options = {})
|
47
|
+
@options = options
|
48
|
+
set_up_amqp_connection
|
49
|
+
if http_api_use_enabled?
|
50
|
+
logger.info "HTTP API use is enabled"
|
51
|
+
set_up_api_connection
|
52
|
+
else
|
53
|
+
logger.info "HTTP API use is disabled"
|
54
|
+
end
|
55
|
+
|
56
|
+
if tracing_enabled?
|
57
|
+
logger.info "tracing is enabled using #{@config[:tracer]}"
|
58
|
+
else
|
59
|
+
logger.info "tracing is disabled"
|
60
|
+
end
|
61
|
+
|
62
|
+
if block_given?
|
63
|
+
begin
|
64
|
+
yield
|
65
|
+
ensure
|
66
|
+
disconnect
|
67
|
+
end
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
def disconnect
|
72
|
+
@channel.close if @channel
|
73
|
+
@connection.close if @connection
|
74
|
+
@channel = nil
|
75
|
+
@connection = nil
|
76
|
+
@exchange = nil
|
77
|
+
@api_client = nil
|
78
|
+
end
|
79
|
+
|
80
|
+
# Connect to RabbitMQ via AMQP
|
81
|
+
#
|
82
|
+
# This sets up the main connection and channel we use for talking to
|
83
|
+
# RabbitMQ. It also ensures the existence of the exchange we'll be using.
|
84
|
+
def set_up_amqp_connection
|
85
|
+
open_connection!
|
86
|
+
open_channel!
|
87
|
+
declare_exchange!
|
88
|
+
declare_publisher!
|
89
|
+
end
|
90
|
+
|
91
|
+
def open_connection
|
92
|
+
logger.info "connecting to rabbitmq (#{sanitized_uri})"
|
93
|
+
|
94
|
+
connection = Hutch::Adapter.new(connection_params)
|
95
|
+
|
96
|
+
with_bunny_connection_handler(sanitized_uri) do
|
97
|
+
connection.start
|
98
|
+
end
|
99
|
+
|
100
|
+
logger.info "connected to RabbitMQ at #{connection_params[:host]} as #{connection_params[:username]}"
|
101
|
+
connection
|
102
|
+
end
|
103
|
+
|
104
|
+
def open_connection!
|
105
|
+
@connection = open_connection
|
106
|
+
end
|
107
|
+
|
108
|
+
def open_channel
|
109
|
+
logger.info "opening rabbitmq channel with pool size #{consumer_pool_size}, abort on exception #{consumer_pool_abort_on_exception}"
|
110
|
+
connection.create_channel(nil, consumer_pool_size, consumer_pool_abort_on_exception).tap do |ch|
|
111
|
+
connection.prefetch_channel(ch, @config[:channel_prefetch])
|
112
|
+
if @config[:publisher_confirms] || @config[:force_publisher_confirms]
|
113
|
+
logger.info 'enabling publisher confirms'
|
114
|
+
ch.confirm_select
|
115
|
+
end
|
116
|
+
end
|
117
|
+
end
|
118
|
+
|
119
|
+
def open_channel!
|
120
|
+
@channel = open_channel
|
121
|
+
end
|
122
|
+
|
123
|
+
def declare_exchange(ch = channel)
|
124
|
+
exchange_name = @config[:mq_exchange]
|
125
|
+
exchange_type = @config[:mq_exchange_type]
|
126
|
+
exchange_options = { durable: true }.merge(@config[:mq_exchange_options])
|
127
|
+
logger.info "using topic exchange '#{exchange_name}'"
|
128
|
+
|
129
|
+
with_bunny_precondition_handler('exchange') do
|
130
|
+
Adapter.new_exchange(ch, exchange_type, exchange_name, exchange_options)
|
131
|
+
end
|
132
|
+
end
|
133
|
+
|
134
|
+
def declare_exchange!(*args)
|
135
|
+
@exchange = declare_exchange(*args)
|
136
|
+
end
|
137
|
+
|
138
|
+
def declare_publisher!
|
139
|
+
@publisher = Hutch::Publisher.new(connection, channel, exchange, @config)
|
140
|
+
end
|
141
|
+
|
142
|
+
# Set up the connection to the RabbitMQ management API. Unfortunately, this
|
143
|
+
# is necessary to do a few things that are impossible over AMQP. E.g.
|
144
|
+
# listing queues and bindings.
|
145
|
+
def set_up_api_connection
|
146
|
+
logger.info "connecting to rabbitmq HTTP API (#{api_config.sanitized_uri})"
|
147
|
+
|
148
|
+
with_authentication_error_handler do
|
149
|
+
with_connection_error_handler do
|
150
|
+
@api_client = CarrotTop.new(host: api_config.host, port: api_config.port,
|
151
|
+
user: api_config.username, password: api_config.password,
|
152
|
+
ssl: api_config.ssl)
|
153
|
+
@api_client.exchanges
|
154
|
+
end
|
155
|
+
end
|
156
|
+
end
|
157
|
+
|
158
|
+
def http_api_use_enabled?
|
159
|
+
op = @options.fetch(:enable_http_api_use, true)
|
160
|
+
cf = if @config[:enable_http_api_use].nil?
|
161
|
+
true
|
162
|
+
else
|
163
|
+
@config[:enable_http_api_use]
|
164
|
+
end
|
165
|
+
|
166
|
+
op && cf
|
167
|
+
end
|
168
|
+
|
169
|
+
def tracing_enabled?
|
170
|
+
@config[:tracer] && @config[:tracer] != Hutch::Tracers::NullTracer
|
171
|
+
end
|
172
|
+
|
173
|
+
# Create / get a durable queue and apply namespace if it exists.
|
174
|
+
def queue(name, options = {})
|
175
|
+
with_bunny_precondition_handler('queue') do
|
176
|
+
namespace = @config[:namespace].to_s.downcase.gsub(/[^-_:\.\w]/, "")
|
177
|
+
queue_name = namespace.present? ? "#{namespace}:#{name}" : name
|
178
|
+
channel.queue(queue_name, **options)
|
179
|
+
end
|
180
|
+
end
|
181
|
+
|
182
|
+
# Return a mapping of queue names to the routing keys they're bound to.
|
183
|
+
def bindings
|
184
|
+
results = Hash.new { |hash, key| hash[key] = [] }
|
185
|
+
|
186
|
+
filtered = api_client.bindings.
|
187
|
+
reject { |b| b['destination'] == b['routing_key'] }.
|
188
|
+
select { |b| b['source'] == @config[:mq_exchange] && b['vhost'] == @config[:mq_vhost] }
|
189
|
+
|
190
|
+
filtered.each do |binding|
|
191
|
+
results[binding['destination']] << binding['routing_key']
|
192
|
+
end
|
193
|
+
|
194
|
+
results
|
195
|
+
end
|
196
|
+
|
197
|
+
# Find the existing bindings, and unbind any redundant bindings
|
198
|
+
def unbind_redundant_bindings(queue, routing_keys)
|
199
|
+
return unless http_api_use_enabled?
|
200
|
+
|
201
|
+
filtered = bindings.select { |dest, keys| dest == queue.name }
|
202
|
+
filtered.each do |dest, keys|
|
203
|
+
keys.reject { |key| routing_keys.include?(key) }.each do |key|
|
204
|
+
logger.debug "removing redundant binding #{queue.name} <--> #{key}"
|
205
|
+
queue.unbind(exchange, routing_key: key)
|
206
|
+
end
|
207
|
+
end
|
208
|
+
end
|
209
|
+
|
210
|
+
# Bind a queue to the broker's exchange on the routing keys provided. Any
|
211
|
+
# existing bindings on the queue that aren't present in the array of
|
212
|
+
# routing keys will be unbound.
|
213
|
+
def bind_queue(queue, routing_keys)
|
214
|
+
unbind_redundant_bindings(queue, routing_keys)
|
215
|
+
|
216
|
+
# Ensure all the desired bindings are present
|
217
|
+
routing_keys.each do |routing_key|
|
218
|
+
logger.debug "creating binding #{queue.name} <--> #{routing_key}"
|
219
|
+
queue.bind(exchange, routing_key: routing_key)
|
220
|
+
end
|
221
|
+
end
|
222
|
+
|
223
|
+
def stop
|
224
|
+
if defined?(JRUBY_VERSION)
|
225
|
+
channel.close
|
226
|
+
else
|
227
|
+
# Enqueue a failing job that kills the consumer loop
|
228
|
+
channel_work_pool.shutdown
|
229
|
+
# Give `timeout` seconds to jobs that are still being processed
|
230
|
+
channel_work_pool.join(@config[:graceful_exit_timeout])
|
231
|
+
# If after `timeout` they are still running, they are killed
|
232
|
+
channel_work_pool.kill
|
233
|
+
end
|
234
|
+
end
|
235
|
+
|
236
|
+
def requeue(delivery_tag)
|
237
|
+
channel.reject(delivery_tag, true)
|
238
|
+
end
|
239
|
+
|
240
|
+
def reject(delivery_tag, requeue=false)
|
241
|
+
channel.reject(delivery_tag, requeue)
|
242
|
+
end
|
243
|
+
|
244
|
+
def ack(delivery_tag)
|
245
|
+
channel.ack(delivery_tag, false)
|
246
|
+
end
|
247
|
+
|
248
|
+
def nack(delivery_tag)
|
249
|
+
channel.nack(delivery_tag, false, false)
|
250
|
+
end
|
251
|
+
|
252
|
+
def publish(*args)
|
253
|
+
@publisher.publish(*args)
|
254
|
+
end
|
255
|
+
|
256
|
+
def confirm_select(*args)
|
257
|
+
channel.confirm_select(*args)
|
258
|
+
end
|
259
|
+
|
260
|
+
def wait_for_confirms
|
261
|
+
channel.wait_for_confirms
|
262
|
+
end
|
263
|
+
|
264
|
+
# @return [Boolean] True if channel is set up to use publisher confirmations.
|
265
|
+
def using_publisher_confirmations?
|
266
|
+
channel.using_publisher_confirmations?
|
267
|
+
end
|
268
|
+
|
269
|
+
private
|
270
|
+
|
271
|
+
def api_config
|
272
|
+
@api_config ||= OpenStruct.new.tap do |config|
|
273
|
+
config.host = @config[:mq_api_host]
|
274
|
+
config.port = @config[:mq_api_port]
|
275
|
+
config.username = @config[:mq_username]
|
276
|
+
config.password = @config[:mq_password]
|
277
|
+
config.ssl = @config[:mq_api_ssl]
|
278
|
+
config.protocol = config.ssl ? "https://" : "http://"
|
279
|
+
config.sanitized_uri = "#{config.protocol}#{config.username}@#{config.host}:#{config.port}/"
|
280
|
+
end
|
281
|
+
end
|
282
|
+
|
283
|
+
def connection_params
|
284
|
+
parse_uri
|
285
|
+
|
286
|
+
{}.tap do |params|
|
287
|
+
params[:host] = @config[:mq_host]
|
288
|
+
params[:port] = @config[:mq_port]
|
289
|
+
params[:vhost] = @config[:mq_vhost].presence || Hutch::Adapter::DEFAULT_VHOST
|
290
|
+
params[:auth_mechanism] = @config[:mq_auth_mechanism]
|
291
|
+
params[:username] = @config[:mq_username]
|
292
|
+
params[:password] = @config[:mq_password]
|
293
|
+
params[:tls] = @config[:mq_tls]
|
294
|
+
params[:tls_key] = @config[:mq_tls_key]
|
295
|
+
params[:tls_cert] = @config[:mq_tls_cert]
|
296
|
+
params[:verify_peer] = @config[:mq_verify_peer]
|
297
|
+
if @config[:mq_tls_ca_certificates]
|
298
|
+
params[:tls_ca_certificates] = @config[:mq_tls_ca_certificates]
|
299
|
+
end
|
300
|
+
params[:heartbeat] = @config[:heartbeat]
|
301
|
+
params[:connection_timeout] = @config[:connection_timeout]
|
302
|
+
params[:read_timeout] = @config[:read_timeout]
|
303
|
+
params[:write_timeout] = @config[:write_timeout]
|
304
|
+
|
305
|
+
|
306
|
+
params[:automatically_recover] = @config[:automatically_recover]
|
307
|
+
params[:network_recovery_interval] = @config[:network_recovery_interval]
|
308
|
+
|
309
|
+
params[:logger] = @config[:client_logger] if @config[:client_logger]
|
310
|
+
end
|
311
|
+
end
|
312
|
+
|
313
|
+
def parse_uri
|
314
|
+
return if @config[:uri].blank?
|
315
|
+
|
316
|
+
u = URI.parse(@config[:uri])
|
317
|
+
|
318
|
+
@config[:mq_tls] = u.scheme == 'amqps'
|
319
|
+
@config[:mq_host] = u.host
|
320
|
+
@config[:mq_port] = u.port || default_mq_port
|
321
|
+
@config[:mq_vhost] = u.path.sub(/^\//, "")
|
322
|
+
@config[:mq_username] = u.user
|
323
|
+
@config[:mq_password] = u.password
|
324
|
+
end
|
325
|
+
|
326
|
+
def default_mq_port
|
327
|
+
@config[:mq_tls] ? DEFAULT_AMQPS_PORT : DEFAULT_AMQP_PORT
|
328
|
+
end
|
329
|
+
|
330
|
+
def sanitized_uri
|
331
|
+
p = connection_params
|
332
|
+
scheme = p[:tls] ? "amqps" : "amqp"
|
333
|
+
|
334
|
+
"#{scheme}://#{p[:username]}@#{p[:host]}:#{p[:port]}/#{p[:vhost].sub(/^\//, '')}"
|
335
|
+
end
|
336
|
+
|
337
|
+
def with_authentication_error_handler
|
338
|
+
yield
|
339
|
+
rescue Net::HTTPServerException => ex
|
340
|
+
logger.error "HTTP API connection error: #{ex.message.downcase}"
|
341
|
+
if ex.response.code == '401'
|
342
|
+
raise AuthenticationError.new('invalid HTTP API credentials')
|
343
|
+
else
|
344
|
+
raise
|
345
|
+
end
|
346
|
+
end
|
347
|
+
|
348
|
+
def with_connection_error_handler
|
349
|
+
yield
|
350
|
+
rescue Errno::ECONNREFUSED => ex
|
351
|
+
logger.error "HTTP API connection error: #{ex.message.downcase}"
|
352
|
+
raise ConnectionError.new("couldn't connect to HTTP API at #{api_config.sanitized_uri}")
|
353
|
+
end
|
354
|
+
|
355
|
+
def with_bunny_precondition_handler(item)
|
356
|
+
yield
|
357
|
+
rescue Hutch::Adapter::PreconditionFailed => ex
|
358
|
+
logger.error ex.message
|
359
|
+
s = "RabbitMQ responded with 406 Precondition Failed when creating this #{item}. " +
|
360
|
+
"Perhaps it is being redeclared with non-matching attributes"
|
361
|
+
raise WorkerSetupError.new(s)
|
362
|
+
end
|
363
|
+
|
364
|
+
def with_bunny_connection_handler(uri)
|
365
|
+
yield
|
366
|
+
rescue Hutch::Adapter::ConnectionRefused => ex
|
367
|
+
logger.error "amqp connection error: #{ex.message.downcase}"
|
368
|
+
raise ConnectionError.new("couldn't connect to rabbitmq at #{uri}. Check your configuration, network connectivity and RabbitMQ logs.")
|
369
|
+
end
|
370
|
+
|
371
|
+
def channel_work_pool
|
372
|
+
channel.work_pool
|
373
|
+
end
|
374
|
+
|
375
|
+
def consumer_pool_size
|
376
|
+
@config[:consumer_pool_size]
|
377
|
+
end
|
378
|
+
|
379
|
+
def consumer_pool_abort_on_exception
|
380
|
+
@config[:consumer_pool_abort_on_exception]
|
381
|
+
end
|
382
|
+
end
|
383
|
+
end
|