hutch 0.19.0-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +7 -0
- data/.travis.yml +11 -0
- data/CHANGELOG.md +438 -0
- data/Gemfile +22 -0
- data/Guardfile +5 -0
- data/LICENSE +22 -0
- data/README.md +317 -0
- data/Rakefile +14 -0
- data/bin/hutch +8 -0
- data/circle.yml +3 -0
- data/examples/consumer.rb +13 -0
- data/examples/producer.rb +10 -0
- data/hutch.gemspec +30 -0
- data/lib/hutch.rb +62 -0
- data/lib/hutch/adapter.rb +11 -0
- data/lib/hutch/adapters/bunny.rb +33 -0
- data/lib/hutch/adapters/march_hare.rb +37 -0
- data/lib/hutch/broker.rb +374 -0
- data/lib/hutch/cli.rb +205 -0
- data/lib/hutch/config.rb +125 -0
- data/lib/hutch/consumer.rb +75 -0
- data/lib/hutch/error_handlers.rb +8 -0
- data/lib/hutch/error_handlers/airbrake.rb +26 -0
- data/lib/hutch/error_handlers/honeybadger.rb +28 -0
- data/lib/hutch/error_handlers/logger.rb +16 -0
- data/lib/hutch/error_handlers/sentry.rb +23 -0
- data/lib/hutch/exceptions.rb +7 -0
- data/lib/hutch/logging.rb +32 -0
- data/lib/hutch/message.rb +31 -0
- data/lib/hutch/serializers/identity.rb +19 -0
- data/lib/hutch/serializers/json.rb +22 -0
- data/lib/hutch/tracers.rb +6 -0
- data/lib/hutch/tracers/newrelic.rb +19 -0
- data/lib/hutch/tracers/null_tracer.rb +15 -0
- data/lib/hutch/version.rb +4 -0
- data/lib/hutch/worker.rb +143 -0
- data/spec/hutch/broker_spec.rb +377 -0
- data/spec/hutch/cli_spec.rb +80 -0
- data/spec/hutch/config_spec.rb +126 -0
- data/spec/hutch/consumer_spec.rb +130 -0
- data/spec/hutch/error_handlers/airbrake_spec.rb +34 -0
- data/spec/hutch/error_handlers/honeybadger_spec.rb +36 -0
- data/spec/hutch/error_handlers/logger_spec.rb +15 -0
- data/spec/hutch/error_handlers/sentry_spec.rb +20 -0
- data/spec/hutch/logger_spec.rb +28 -0
- data/spec/hutch/message_spec.rb +38 -0
- data/spec/hutch/serializers/json_spec.rb +17 -0
- data/spec/hutch/worker_spec.rb +99 -0
- data/spec/hutch_spec.rb +87 -0
- data/spec/spec_helper.rb +40 -0
- metadata +194 -0
data/lib/hutch/cli.rb
ADDED
@@ -0,0 +1,205 @@
|
|
1
|
+
require 'optparse'
|
2
|
+
|
3
|
+
require 'hutch/version'
|
4
|
+
require 'hutch/logging'
|
5
|
+
require 'hutch/exceptions'
|
6
|
+
require 'hutch/config'
|
7
|
+
|
8
|
+
module Hutch
|
9
|
+
class CLI
|
10
|
+
include Logging
|
11
|
+
|
12
|
+
# Run a Hutch worker with the command line interface.
|
13
|
+
def run(argv = ARGV)
|
14
|
+
parse_options(argv)
|
15
|
+
|
16
|
+
::Process.daemon(true) if Hutch::Config.daemonise
|
17
|
+
|
18
|
+
write_pid if Hutch::Config.pidfile
|
19
|
+
|
20
|
+
Hutch.logger.info "hutch booted with pid #{::Process.pid}"
|
21
|
+
|
22
|
+
if load_app && start_work_loop == :success
|
23
|
+
# If we got here, the worker was shut down nicely
|
24
|
+
Hutch.logger.info 'hutch shut down gracefully'
|
25
|
+
exit 0
|
26
|
+
else
|
27
|
+
Hutch.logger.info 'hutch terminated due to an error'
|
28
|
+
exit 1
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
def load_app
|
33
|
+
# Try to load a Rails app in the current directory
|
34
|
+
load_rails_app('.') if Hutch::Config.autoload_rails
|
35
|
+
Hutch::Config.require_paths.each do |path|
|
36
|
+
# See if each path is a Rails app. If so, try to load it.
|
37
|
+
next if load_rails_app(path)
|
38
|
+
|
39
|
+
# Given path is not a Rails app, try requiring it as a file
|
40
|
+
logger.info "requiring '#{path}'"
|
41
|
+
begin
|
42
|
+
# Need to add '.' to load path for relative requires
|
43
|
+
$LOAD_PATH << '.'
|
44
|
+
require path
|
45
|
+
rescue LoadError
|
46
|
+
logger.fatal "could not load file '#{path}'"
|
47
|
+
return false
|
48
|
+
ensure
|
49
|
+
# Clean up load path
|
50
|
+
$LOAD_PATH.pop
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
# Because of the order things are required when we run the Hutch binary
|
55
|
+
# in hutch/bin, the Sentry Raven gem gets required **after** the error
|
56
|
+
# handlers are set up. Due to this, we never got any Sentry notifications
|
57
|
+
# when an error occurred in any of the consumers.
|
58
|
+
if defined?(Raven)
|
59
|
+
Hutch::Config[:error_handlers] << Hutch::ErrorHandlers::Sentry.new
|
60
|
+
end
|
61
|
+
|
62
|
+
true
|
63
|
+
end
|
64
|
+
|
65
|
+
def load_rails_app(path)
|
66
|
+
# path should point to the app's top level directory
|
67
|
+
if File.directory?(path)
|
68
|
+
# Smells like a Rails app if it's got a config/environment.rb file
|
69
|
+
rails_path = File.expand_path(File.join(path, 'config/environment.rb'))
|
70
|
+
if File.exists?(rails_path)
|
71
|
+
logger.info "found rails project (#{path}), booting app"
|
72
|
+
ENV['RACK_ENV'] ||= ENV['RAILS_ENV'] || 'development'
|
73
|
+
require rails_path
|
74
|
+
::Rails.application.eager_load!
|
75
|
+
return true
|
76
|
+
end
|
77
|
+
end
|
78
|
+
false
|
79
|
+
end
|
80
|
+
|
81
|
+
# Kick off the work loop. This method returns when the worker is shut down
|
82
|
+
# gracefully (with a SIGQUIT, SIGTERM or SIGINT).
|
83
|
+
def start_work_loop
|
84
|
+
Hutch.connect
|
85
|
+
@worker = Hutch::Worker.new(Hutch.broker, Hutch.consumers)
|
86
|
+
@worker.run
|
87
|
+
:success
|
88
|
+
rescue ConnectionError, AuthenticationError, WorkerSetupError => ex
|
89
|
+
logger.fatal ex.message
|
90
|
+
:error
|
91
|
+
end
|
92
|
+
|
93
|
+
def parse_options(args = ARGV)
|
94
|
+
OptionParser.new do |opts|
|
95
|
+
opts.banner = 'usage: hutch [options]'
|
96
|
+
|
97
|
+
opts.on('--mq-host HOST', 'Set the RabbitMQ host') do |host|
|
98
|
+
Hutch::Config.mq_host = host
|
99
|
+
end
|
100
|
+
|
101
|
+
opts.on('--mq-port PORT', 'Set the RabbitMQ port') do |port|
|
102
|
+
Hutch::Config.mq_port = port
|
103
|
+
end
|
104
|
+
|
105
|
+
opts.on("-t", "--[no-]mq-tls", 'Use TLS for the AMQP connection') do |tls|
|
106
|
+
Hutch::Config.mq_tls = tls
|
107
|
+
end
|
108
|
+
|
109
|
+
opts.on('--mq-tls-cert FILE', 'Certificate for TLS client verification') do |file|
|
110
|
+
abort "Certificate file '#{file}' not found" unless File.exists?(file)
|
111
|
+
Hutch::Config.mq_tls_cert = file
|
112
|
+
end
|
113
|
+
|
114
|
+
opts.on('--mq-tls-key FILE', 'Private key for TLS client verification') do |file|
|
115
|
+
abort "Private key file '#{file}' not found" unless File.exists?(file)
|
116
|
+
Hutch::Config.mq_tls_key = file
|
117
|
+
end
|
118
|
+
|
119
|
+
opts.on('--mq-exchange EXCHANGE',
|
120
|
+
'Set the RabbitMQ exchange') do |exchange|
|
121
|
+
Hutch::Config.mq_exchange = exchange
|
122
|
+
end
|
123
|
+
|
124
|
+
opts.on('--mq-vhost VHOST', 'Set the RabbitMQ vhost') do |vhost|
|
125
|
+
Hutch::Config.mq_vhost = vhost
|
126
|
+
end
|
127
|
+
|
128
|
+
opts.on('--mq-username USERNAME',
|
129
|
+
'Set the RabbitMQ username') do |username|
|
130
|
+
Hutch::Config.mq_username = username
|
131
|
+
end
|
132
|
+
|
133
|
+
opts.on('--mq-password PASSWORD',
|
134
|
+
'Set the RabbitMQ password') do |password|
|
135
|
+
Hutch::Config.mq_password = password
|
136
|
+
end
|
137
|
+
|
138
|
+
opts.on('--mq-api-host HOST', 'Set the RabbitMQ API host') do |host|
|
139
|
+
Hutch::Config.mq_api_host = host
|
140
|
+
end
|
141
|
+
|
142
|
+
opts.on('--mq-api-port PORT', 'Set the RabbitMQ API port') do |port|
|
143
|
+
Hutch::Config.mq_api_port = port
|
144
|
+
end
|
145
|
+
|
146
|
+
opts.on("-s", "--[no-]mq-api-ssl", 'Use SSL for the RabbitMQ API') do |api_ssl|
|
147
|
+
Hutch::Config.mq_api_ssl = api_ssl
|
148
|
+
end
|
149
|
+
|
150
|
+
opts.on('--config FILE', 'Load Hutch configuration from a file') do |file|
|
151
|
+
begin
|
152
|
+
File.open(file) { |fp| Hutch::Config.load_from_file(fp) }
|
153
|
+
rescue Errno::ENOENT
|
154
|
+
abort "Config file '#{file}' not found"
|
155
|
+
end
|
156
|
+
end
|
157
|
+
|
158
|
+
opts.on('--require PATH', 'Require a Rails app or path') do |path|
|
159
|
+
Hutch::Config.require_paths << path
|
160
|
+
end
|
161
|
+
|
162
|
+
opts.on('--[no-]autoload-rails', 'Require the current rails app directory') do |autoload_rails|
|
163
|
+
Hutch::Config.autoload_rails = autoload_rails
|
164
|
+
end
|
165
|
+
|
166
|
+
opts.on('-q', '--quiet', 'Quiet logging') do
|
167
|
+
Hutch::Config.log_level = Logger::WARN
|
168
|
+
end
|
169
|
+
|
170
|
+
opts.on('-v', '--verbose', 'Verbose logging') do
|
171
|
+
Hutch::Config.log_level = Logger::DEBUG
|
172
|
+
end
|
173
|
+
|
174
|
+
opts.on('--namespace NAMESPACE', 'Queue namespace') do |namespace|
|
175
|
+
Hutch::Config.namespace = namespace
|
176
|
+
end
|
177
|
+
|
178
|
+
opts.on('-d', '--daemonise', 'Daemonise') do |daemonise|
|
179
|
+
Hutch::Config.daemonise = daemonise
|
180
|
+
end
|
181
|
+
|
182
|
+
opts.on('--pidfile PIDFILE', 'Pidfile') do |pidfile|
|
183
|
+
Hutch::Config.pidfile = pidfile
|
184
|
+
end
|
185
|
+
|
186
|
+
opts.on('--version', 'Print the version and exit') do
|
187
|
+
puts "hutch v#{VERSION}"
|
188
|
+
exit 0
|
189
|
+
end
|
190
|
+
|
191
|
+
opts.on('-h', '--help', 'Show this message and exit') do
|
192
|
+
puts opts
|
193
|
+
exit 0
|
194
|
+
end
|
195
|
+
end.parse!(args)
|
196
|
+
end
|
197
|
+
|
198
|
+
def write_pid
|
199
|
+
pidfile = File.expand_path(Hutch::Config.pidfile)
|
200
|
+
Hutch.logger.info "writing pid in #{pidfile}"
|
201
|
+
File.open(pidfile, 'w') { |f| f.puts ::Process.pid }
|
202
|
+
end
|
203
|
+
|
204
|
+
end
|
205
|
+
end
|
data/lib/hutch/config.rb
ADDED
@@ -0,0 +1,125 @@
|
|
1
|
+
require 'hutch/error_handlers/logger'
|
2
|
+
require 'erb'
|
3
|
+
require 'logger'
|
4
|
+
|
5
|
+
module Hutch
|
6
|
+
class UnknownAttributeError < StandardError; end
|
7
|
+
|
8
|
+
module Config
|
9
|
+
require 'yaml'
|
10
|
+
|
11
|
+
def self.initialize(params={})
|
12
|
+
@config = {
|
13
|
+
mq_host: 'localhost',
|
14
|
+
mq_port: 5672,
|
15
|
+
mq_exchange: 'hutch', # TODO: should this be required?
|
16
|
+
mq_vhost: '/',
|
17
|
+
mq_tls: false,
|
18
|
+
mq_tls_cert: nil,
|
19
|
+
mq_tls_key: nil,
|
20
|
+
mq_tls_ca_certificates: nil,
|
21
|
+
mq_verify_peer: true,
|
22
|
+
mq_username: 'guest',
|
23
|
+
mq_password: 'guest',
|
24
|
+
mq_api_host: 'localhost',
|
25
|
+
mq_api_port: 15672,
|
26
|
+
mq_api_ssl: false,
|
27
|
+
heartbeat: 30,
|
28
|
+
# placeholder, allows specifying connection parameters
|
29
|
+
# as a URI.
|
30
|
+
uri: nil,
|
31
|
+
log_level: Logger::INFO,
|
32
|
+
require_paths: [],
|
33
|
+
autoload_rails: true,
|
34
|
+
error_handlers: [Hutch::ErrorHandlers::Logger.new],
|
35
|
+
tracer: Hutch::Tracers::NullTracer,
|
36
|
+
namespace: nil,
|
37
|
+
daemonise: false,
|
38
|
+
pidfile: nil,
|
39
|
+
channel_prefetch: 0,
|
40
|
+
# enables publisher confirms, leaves it up to the app
|
41
|
+
# how they are tracked
|
42
|
+
publisher_confirms: false,
|
43
|
+
# like `publisher_confirms` above but also
|
44
|
+
# forces waiting for a confirm for every publish
|
45
|
+
force_publisher_confirms: false,
|
46
|
+
# Heroku needs > 10. MK.
|
47
|
+
connection_timeout: 11,
|
48
|
+
read_timeout: 11,
|
49
|
+
write_timeout: 11,
|
50
|
+
enable_http_api_use: true,
|
51
|
+
# Number of seconds that a running consumer is given
|
52
|
+
# to finish its job when gracefully exiting Hutch, before
|
53
|
+
# it's killed.
|
54
|
+
graceful_exit_timeout: 11,
|
55
|
+
client_logger: nil,
|
56
|
+
|
57
|
+
consumer_pool_size: 1,
|
58
|
+
|
59
|
+
serializer: Hutch::Serializers::JSON,
|
60
|
+
}.merge(params)
|
61
|
+
end
|
62
|
+
|
63
|
+
def self.get(attr)
|
64
|
+
check_attr(attr)
|
65
|
+
user_config[attr]
|
66
|
+
end
|
67
|
+
|
68
|
+
def self.set(attr, value)
|
69
|
+
check_attr(attr)
|
70
|
+
user_config[attr] = value
|
71
|
+
end
|
72
|
+
|
73
|
+
class << self
|
74
|
+
alias_method :[], :get
|
75
|
+
alias_method :[]=, :set
|
76
|
+
end
|
77
|
+
|
78
|
+
def self.check_attr(attr)
|
79
|
+
unless user_config.key?(attr)
|
80
|
+
raise UnknownAttributeError, "#{attr} is not a valid config attribute"
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
84
|
+
def self.user_config
|
85
|
+
initialize unless @config
|
86
|
+
@config
|
87
|
+
end
|
88
|
+
|
89
|
+
def self.to_hash
|
90
|
+
self.user_config
|
91
|
+
end
|
92
|
+
|
93
|
+
def self.load_from_file(file)
|
94
|
+
YAML.load(ERB.new(File.read(file)).result).each do |attr, value|
|
95
|
+
Hutch::Config.send("#{attr}=", convert_value(attr, value))
|
96
|
+
end
|
97
|
+
end
|
98
|
+
|
99
|
+
def self.convert_value(attr, value)
|
100
|
+
case attr
|
101
|
+
when "tracer"
|
102
|
+
Kernel.const_get(value)
|
103
|
+
else
|
104
|
+
value
|
105
|
+
end
|
106
|
+
end
|
107
|
+
|
108
|
+
def self.method_missing(method, *args, &block)
|
109
|
+
attr = method.to_s.sub(/=$/, '').to_sym
|
110
|
+
return super unless user_config.key?(attr)
|
111
|
+
|
112
|
+
if method =~ /=$/
|
113
|
+
set(attr, args.first)
|
114
|
+
else
|
115
|
+
get(attr)
|
116
|
+
end
|
117
|
+
end
|
118
|
+
|
119
|
+
private
|
120
|
+
|
121
|
+
def deep_copy(obj)
|
122
|
+
Marshal.load(Marshal.dump(obj))
|
123
|
+
end
|
124
|
+
end
|
125
|
+
end
|
@@ -0,0 +1,75 @@
|
|
1
|
+
require 'set'
|
2
|
+
|
3
|
+
module Hutch
|
4
|
+
# Include this module in a class to register it as a consumer. Consumers
|
5
|
+
# gain a class method called `consume`, which should be used to register
|
6
|
+
# the routing keys a consumer is interested in.
|
7
|
+
module Consumer
|
8
|
+
attr_accessor :broker, :delivery_info
|
9
|
+
|
10
|
+
def self.included(base)
|
11
|
+
base.extend(ClassMethods)
|
12
|
+
Hutch.register_consumer(base)
|
13
|
+
end
|
14
|
+
|
15
|
+
def reject!
|
16
|
+
broker.reject(delivery_info.delivery_tag)
|
17
|
+
end
|
18
|
+
|
19
|
+
def requeue!
|
20
|
+
broker.requeue(delivery_info.delivery_tag)
|
21
|
+
end
|
22
|
+
|
23
|
+
def logger
|
24
|
+
Hutch::Logging.logger
|
25
|
+
end
|
26
|
+
|
27
|
+
module ClassMethods
|
28
|
+
# Add one or more routing keys to the set of routing keys the consumer
|
29
|
+
# wants to subscribe to.
|
30
|
+
def consume(*routing_keys)
|
31
|
+
@routing_keys = self.routing_keys.union(routing_keys)
|
32
|
+
end
|
33
|
+
|
34
|
+
# Explicitly set the queue name
|
35
|
+
def queue_name(name)
|
36
|
+
@queue_name = name
|
37
|
+
end
|
38
|
+
|
39
|
+
# Allow to specify custom arguments that will be passed when creating the queue.
|
40
|
+
def arguments(arguments = {})
|
41
|
+
@arguments = arguments
|
42
|
+
end
|
43
|
+
|
44
|
+
# Set custom serializer class, override global value
|
45
|
+
def serializer(name)
|
46
|
+
@serializer = name
|
47
|
+
end
|
48
|
+
|
49
|
+
# The RabbitMQ queue name for the consumer. This is derived from the
|
50
|
+
# fully-qualified class name. Module separators are replaced with single
|
51
|
+
# colons, camelcased class names are converted to snake case.
|
52
|
+
def get_queue_name
|
53
|
+
return @queue_name unless @queue_name.nil?
|
54
|
+
queue_name = self.name.gsub(/::/, ':')
|
55
|
+
queue_name.gsub!(/([^A-Z:])([A-Z])/) { "#{$1}_#{$2}" }
|
56
|
+
queue_name.downcase
|
57
|
+
end
|
58
|
+
|
59
|
+
# Returns consumer custom arguments.
|
60
|
+
def get_arguments
|
61
|
+
@arguments || {}
|
62
|
+
end
|
63
|
+
|
64
|
+
# Accessor for the consumer's routing key.
|
65
|
+
def routing_keys
|
66
|
+
@routing_keys ||= Set.new
|
67
|
+
end
|
68
|
+
|
69
|
+
def get_serializer
|
70
|
+
@serializer
|
71
|
+
end
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
@@ -0,0 +1,26 @@
|
|
1
|
+
require 'hutch/logging'
|
2
|
+
require 'airbrake'
|
3
|
+
|
4
|
+
module Hutch
|
5
|
+
module ErrorHandlers
|
6
|
+
class Airbrake
|
7
|
+
include Logging
|
8
|
+
|
9
|
+
def handle(message_id, payload, consumer, ex)
|
10
|
+
prefix = "message(#{message_id || '-'}): "
|
11
|
+
logger.error prefix + "Logging event to Airbrake"
|
12
|
+
logger.error prefix + "#{ex.class} - #{ex.message}"
|
13
|
+
::Airbrake.notify_or_ignore(ex, {
|
14
|
+
:error_class => ex.class.name,
|
15
|
+
:error_message => "#{ ex.class.name }: #{ ex.message }",
|
16
|
+
:backtrace => ex.backtrace,
|
17
|
+
:parameters => {
|
18
|
+
:payload => payload,
|
19
|
+
:consumer => consumer,
|
20
|
+
},
|
21
|
+
:cgi_data => ENV.to_hash,
|
22
|
+
})
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
require 'hutch/logging'
|
2
|
+
require 'honeybadger'
|
3
|
+
|
4
|
+
module Hutch
|
5
|
+
module ErrorHandlers
|
6
|
+
class Honeybadger
|
7
|
+
include Logging
|
8
|
+
|
9
|
+
def handle(message_id, payload, consumer, ex)
|
10
|
+
prefix = "message(#{message_id || '-'}): "
|
11
|
+
logger.error prefix + "Logging event to Honeybadger"
|
12
|
+
logger.error prefix + "#{ex.class} - #{ex.message}"
|
13
|
+
::Honeybadger.notify_or_ignore(
|
14
|
+
:error_class => ex.class.name,
|
15
|
+
:error_message => "#{ ex.class.name }: #{ ex.message }",
|
16
|
+
:backtrace => ex.backtrace,
|
17
|
+
:context => {
|
18
|
+
:message_id => message_id,
|
19
|
+
:consumer => consumer
|
20
|
+
},
|
21
|
+
:parameters => {
|
22
|
+
:payload => payload
|
23
|
+
}
|
24
|
+
)
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|