hutch 0.21.0-java → 0.25.0-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/.gitignore +3 -0
- data/.rspec +1 -0
- data/.travis.yml +11 -12
- data/.yardopts +5 -0
- data/CHANGELOG.md +118 -1
- data/Gemfile +15 -4
- data/Guardfile +13 -4
- data/README.md +274 -24
- data/Rakefile +8 -1
- data/hutch.gemspec +6 -7
- data/lib/hutch.rb +11 -8
- data/lib/hutch/adapters/march_hare.rb +1 -1
- data/lib/hutch/broker.rb +113 -110
- data/lib/hutch/cli.rb +42 -11
- data/lib/hutch/config.rb +209 -59
- data/lib/hutch/error_handlers.rb +1 -0
- data/lib/hutch/error_handlers/airbrake.rb +44 -16
- data/lib/hutch/error_handlers/base.rb +15 -0
- data/lib/hutch/error_handlers/honeybadger.rb +33 -18
- data/lib/hutch/error_handlers/logger.rb +12 -6
- data/lib/hutch/error_handlers/opbeat.rb +30 -0
- data/lib/hutch/error_handlers/sentry.rb +14 -6
- data/lib/hutch/logging.rb +5 -5
- data/lib/hutch/publisher.rb +75 -0
- data/lib/hutch/tracers.rb +1 -0
- data/lib/hutch/tracers/opbeat.rb +37 -0
- data/lib/hutch/version.rb +1 -1
- data/lib/hutch/waiter.rb +104 -0
- data/lib/hutch/worker.rb +50 -66
- data/lib/yard-settings/handler.rb +38 -0
- data/lib/yard-settings/yard-settings.rb +2 -0
- data/spec/hutch/broker_spec.rb +162 -77
- data/spec/hutch/cli_spec.rb +16 -3
- data/spec/hutch/config_spec.rb +83 -22
- data/spec/hutch/error_handlers/airbrake_spec.rb +25 -10
- data/spec/hutch/error_handlers/honeybadger_spec.rb +24 -2
- data/spec/hutch/error_handlers/logger_spec.rb +14 -1
- data/spec/hutch/error_handlers/opbeat_spec.rb +37 -0
- data/spec/hutch/error_handlers/sentry_spec.rb +18 -1
- data/spec/hutch/logger_spec.rb +12 -6
- data/spec/hutch/waiter_spec.rb +51 -0
- data/spec/hutch/worker_spec.rb +33 -4
- data/spec/spec_helper.rb +7 -5
- data/spec/tracers/opbeat_spec.rb +44 -0
- data/templates/default/class/html/settings.erb +0 -0
- data/templates/default/class/setup.rb +4 -0
- data/templates/default/fulldoc/html/css/hutch.css +13 -0
- data/templates/default/layout/html/setup.rb +7 -0
- data/templates/default/method_details/html/settings.erb +5 -0
- data/templates/default/method_details/setup.rb +4 -0
- data/templates/default/method_details/text/settings.erb +0 -0
- data/templates/default/module/html/settings.erb +40 -0
- data/templates/default/module/setup.rb +4 -0
- metadata +41 -38
data/lib/hutch/cli.rb
CHANGED
@@ -1,9 +1,9 @@
|
|
1
1
|
require 'optparse'
|
2
2
|
|
3
|
-
require 'hutch/version'
|
4
3
|
require 'hutch/logging'
|
5
|
-
require 'hutch/exceptions'
|
6
4
|
require 'hutch/config'
|
5
|
+
require 'hutch/version'
|
6
|
+
require 'hutch/exceptions'
|
7
7
|
|
8
8
|
module Hutch
|
9
9
|
class CLI
|
@@ -11,9 +11,10 @@ module Hutch
|
|
11
11
|
|
12
12
|
# Run a Hutch worker with the command line interface.
|
13
13
|
def run(argv = ARGV)
|
14
|
+
Hutch::Config.initialize
|
14
15
|
parse_options(argv)
|
15
16
|
|
16
|
-
|
17
|
+
daemonise_process
|
17
18
|
|
18
19
|
write_pid if Hutch::Config.pidfile
|
19
20
|
|
@@ -71,8 +72,8 @@ module Hutch
|
|
71
72
|
end
|
72
73
|
rails_path = File.expand_path(File.join(path, 'config/environment.rb'))
|
73
74
|
if is_rails_app && File.exist?(rails_path)
|
74
|
-
logger.info "found rails project (#{path}), booting app"
|
75
75
|
ENV['RACK_ENV'] ||= ENV['RAILS_ENV'] || 'development'
|
76
|
+
logger.info "found rails project (#{path}), booting app in #{ENV['RACK_ENV']} environment"
|
76
77
|
require rails_path
|
77
78
|
::Rails.application.eager_load!
|
78
79
|
return true
|
@@ -85,10 +86,13 @@ module Hutch
|
|
85
86
|
# gracefully (with a SIGQUIT, SIGTERM or SIGINT).
|
86
87
|
def start_work_loop
|
87
88
|
Hutch.connect
|
88
|
-
@worker = Hutch::Worker.new(Hutch.broker, Hutch.consumers)
|
89
|
+
@worker = Hutch::Worker.new(Hutch.broker, Hutch.consumers, Hutch::Config.setup_procs)
|
89
90
|
@worker.run
|
90
91
|
:success
|
91
92
|
rescue ConnectionError, AuthenticationError, WorkerSetupError => ex
|
93
|
+
Hutch::Config[:error_handlers].each do |backend|
|
94
|
+
backend.handle_setup_exception(ex)
|
95
|
+
end
|
92
96
|
logger.fatal ex.message
|
93
97
|
:error
|
94
98
|
end
|
@@ -109,14 +113,16 @@ module Hutch
|
|
109
113
|
Hutch::Config.mq_tls = tls
|
110
114
|
end
|
111
115
|
|
112
|
-
opts.on('--mq-tls-cert FILE', 'Certificate
|
113
|
-
|
114
|
-
|
116
|
+
opts.on('--mq-tls-cert FILE', 'Certificate for TLS client verification') do |file|
|
117
|
+
abort_without_file(file, 'Certificate file') do
|
118
|
+
Hutch::Config.mq_tls_cert = file
|
119
|
+
end
|
115
120
|
end
|
116
121
|
|
117
122
|
opts.on('--mq-tls-key FILE', 'Private key for TLS client verification') do |file|
|
118
|
-
|
119
|
-
|
123
|
+
abort_without_file(file, 'Private key file') do
|
124
|
+
Hutch::Config.mq_tls_key = file
|
125
|
+
end
|
120
126
|
end
|
121
127
|
|
122
128
|
opts.on('--mq-exchange EXCHANGE',
|
@@ -154,7 +160,7 @@ module Hutch
|
|
154
160
|
begin
|
155
161
|
File.open(file) { |fp| Hutch::Config.load_from_file(fp) }
|
156
162
|
rescue Errno::ENOENT
|
157
|
-
|
163
|
+
abort_with_message("Config file '#{file}' not found")
|
158
164
|
end
|
159
165
|
end
|
160
166
|
|
@@ -186,6 +192,10 @@ module Hutch
|
|
186
192
|
Hutch::Config.pidfile = pidfile
|
187
193
|
end
|
188
194
|
|
195
|
+
opts.on('--only-group GROUP', 'Load only consumers in this group') do |group|
|
196
|
+
Hutch::Config.group = group
|
197
|
+
end
|
198
|
+
|
189
199
|
opts.on('--version', 'Print the version and exit') do
|
190
200
|
puts "hutch v#{VERSION}"
|
191
201
|
exit 0
|
@@ -204,5 +214,26 @@ module Hutch
|
|
204
214
|
File.open(pidfile, 'w') { |f| f.puts ::Process.pid }
|
205
215
|
end
|
206
216
|
|
217
|
+
private
|
218
|
+
|
219
|
+
def daemonise_process
|
220
|
+
return unless Hutch::Config.daemonise
|
221
|
+
if defined?(JRUBY_VERSION)
|
222
|
+
Hutch.logger.warn "JRuby ignores the --daemonise option"
|
223
|
+
return
|
224
|
+
end
|
225
|
+
|
226
|
+
::Process.daemon(true)
|
227
|
+
end
|
228
|
+
|
229
|
+
def abort_without_file(file, file_description, &block)
|
230
|
+
abort_with_message("#{file_description} '#{file}' not found") unless File.exists?(file)
|
231
|
+
|
232
|
+
yield
|
233
|
+
end
|
234
|
+
|
235
|
+
def abort_with_message(message)
|
236
|
+
abort message
|
237
|
+
end
|
207
238
|
end
|
208
239
|
end
|
data/lib/hutch/config.rb
CHANGED
@@ -1,77 +1,232 @@
|
|
1
1
|
require 'hutch/error_handlers/logger'
|
2
|
+
require 'hutch/tracers'
|
3
|
+
require 'hutch/serializers/json'
|
2
4
|
require 'erb'
|
3
5
|
require 'logger'
|
4
6
|
|
5
7
|
module Hutch
|
6
8
|
class UnknownAttributeError < StandardError; end
|
7
9
|
|
10
|
+
# Configuration settings, available everywhere
|
11
|
+
#
|
12
|
+
# There are defaults, which can be overridden by ENV variables prefixed by
|
13
|
+
# <tt>HUTCH_</tt>, and each of these can be overridden using the {.set}
|
14
|
+
# method.
|
15
|
+
#
|
16
|
+
# @example Configuring on the command-line
|
17
|
+
# HUTCH_PUBLISHER_CONFIRMS=false hutch
|
8
18
|
module Config
|
9
19
|
require 'yaml'
|
20
|
+
@string_keys = Set.new
|
21
|
+
@number_keys = Set.new
|
22
|
+
@boolean_keys = Set.new
|
23
|
+
@settings_defaults = {}
|
24
|
+
|
25
|
+
# Define a String user setting
|
26
|
+
# @!visibility private
|
27
|
+
def self.string_setting(name, default_value)
|
28
|
+
@string_keys << name
|
29
|
+
@settings_defaults[name] = default_value
|
30
|
+
end
|
31
|
+
|
32
|
+
# Define a Number user setting
|
33
|
+
# @!visibility private
|
34
|
+
def self.number_setting(name, default_value)
|
35
|
+
@number_keys << name
|
36
|
+
@settings_defaults[name] = default_value
|
37
|
+
end
|
38
|
+
|
39
|
+
# Define a Boolean user setting
|
40
|
+
# @!visibility private
|
41
|
+
def self.boolean_setting(name, default_value)
|
42
|
+
@boolean_keys << name
|
43
|
+
@settings_defaults[name] = default_value
|
44
|
+
end
|
45
|
+
|
46
|
+
# RabbitMQ hostname
|
47
|
+
string_setting :mq_host, '127.0.0.1'
|
48
|
+
|
49
|
+
# RabbitMQ Exchange to use for publishing
|
50
|
+
string_setting :mq_exchange, 'hutch'
|
51
|
+
|
52
|
+
# RabbitMQ vhost to use
|
53
|
+
string_setting :mq_vhost, '/'
|
54
|
+
|
55
|
+
# RabbitMQ username to use.
|
56
|
+
#
|
57
|
+
# As of RabbitMQ 3.3.0, <tt>guest</tt> can only can connect from localhost.
|
58
|
+
string_setting :mq_username, 'guest'
|
59
|
+
|
60
|
+
# RabbitMQ password
|
61
|
+
string_setting :mq_password, 'guest'
|
62
|
+
|
63
|
+
# RabbitMQ URI (takes precedence over MQ username, password, host, port and vhost settings)
|
64
|
+
string_setting :uri, nil
|
65
|
+
|
66
|
+
# RabbitMQ HTTP API hostname
|
67
|
+
string_setting :mq_api_host, '127.0.0.1'
|
68
|
+
|
69
|
+
# RabbitMQ port
|
70
|
+
number_setting :mq_port, 5672
|
71
|
+
|
72
|
+
# RabbitMQ HTTP API port
|
73
|
+
number_setting :mq_api_port, 15672
|
74
|
+
|
75
|
+
# [RabbitMQ heartbeat timeout](http://rabbitmq.com/heartbeats.html)
|
76
|
+
number_setting :heartbeat, 30
|
77
|
+
|
78
|
+
# The <tt>basic.qos</tt> prefetch value to use.
|
79
|
+
#
|
80
|
+
# Default: `0`, no limit. See Bunny and RabbitMQ documentation.
|
81
|
+
number_setting :channel_prefetch, 0
|
82
|
+
|
83
|
+
# Bunny's socket open timeout
|
84
|
+
number_setting :connection_timeout, 11
|
85
|
+
|
86
|
+
# Bunny's socket read timeout
|
87
|
+
number_setting :read_timeout, 11
|
88
|
+
|
89
|
+
# Bunny's socket write timeout
|
90
|
+
number_setting :write_timeout, 11
|
91
|
+
|
92
|
+
# FIXME: DOCUMENT THIS
|
93
|
+
number_setting :graceful_exit_timeout, 11
|
94
|
+
|
95
|
+
# Bunny consumer work pool size
|
96
|
+
number_setting :consumer_pool_size, 1
|
97
|
+
|
98
|
+
# Should TLS be used?
|
99
|
+
boolean_setting :mq_tls, false
|
100
|
+
|
101
|
+
# Should SSL certificate be verified?
|
102
|
+
boolean_setting :mq_verify_peer, true
|
103
|
+
|
104
|
+
# Should SSL be used for the RabbitMQ API?
|
105
|
+
boolean_setting :mq_api_ssl, false
|
106
|
+
|
107
|
+
# Should the current Rails app directory be required?
|
108
|
+
boolean_setting :autoload_rails, true
|
109
|
+
|
110
|
+
# Should the Hutch runner process daemonise?
|
111
|
+
#
|
112
|
+
# The option is ignored on JRuby.
|
113
|
+
boolean_setting :daemonise, false
|
114
|
+
|
115
|
+
# Should RabbitMQ publisher confirms be enabled?
|
116
|
+
#
|
117
|
+
# Leaves it up to the app how they are tracked
|
118
|
+
# (e.g. using Hutch::Broker#confirm_select callback or Hutch::Broker#wait_for_confirms)
|
119
|
+
boolean_setting :publisher_confirms, false
|
120
|
+
|
121
|
+
# Enables publisher confirms, forces Hutch::Broker#wait_for_confirms for
|
122
|
+
# every publish.
|
123
|
+
#
|
124
|
+
# **This is the safest option which also offers the
|
125
|
+
# lowest throughput**.
|
126
|
+
boolean_setting :force_publisher_confirms, false
|
127
|
+
|
128
|
+
# Should the RabbitMQ HTTP API be used?
|
129
|
+
boolean_setting :enable_http_api_use, true
|
130
|
+
|
131
|
+
# Should Bunny's consumer work pool threads abort on exception.
|
132
|
+
#
|
133
|
+
# The option is ignored on JRuby.
|
134
|
+
boolean_setting :consumer_pool_abort_on_exception, false
|
135
|
+
|
136
|
+
# Prefix displayed on the consumers tags.
|
137
|
+
string_setting :consumer_tag_prefix, 'hutch'
|
138
|
+
|
139
|
+
string_setting :group, ''
|
140
|
+
|
141
|
+
# Set of all setting keys
|
142
|
+
ALL_KEYS = @boolean_keys + @number_keys + @string_keys
|
10
143
|
|
11
144
|
def self.initialize(params = {})
|
12
|
-
@config
|
13
|
-
|
14
|
-
|
15
|
-
|
145
|
+
unless @config
|
146
|
+
@config = default_config
|
147
|
+
define_methods
|
148
|
+
@config.merge!(env_based_config)
|
149
|
+
end
|
150
|
+
@config.merge!(params)
|
151
|
+
@config
|
152
|
+
end
|
153
|
+
|
154
|
+
# Default settings
|
155
|
+
#
|
156
|
+
# @return [Hash]
|
157
|
+
def self.default_config
|
158
|
+
@settings_defaults.merge({
|
16
159
|
mq_exchange_options: {},
|
17
|
-
mq_vhost: '/',
|
18
|
-
mq_tls: false,
|
19
160
|
mq_tls_cert: nil,
|
20
161
|
mq_tls_key: nil,
|
21
162
|
mq_tls_ca_certificates: nil,
|
22
|
-
mq_verify_peer: true,
|
23
|
-
mq_username: 'guest',
|
24
|
-
mq_password: 'guest',
|
25
|
-
mq_api_host: 'localhost',
|
26
|
-
mq_api_port: 15672,
|
27
|
-
mq_api_ssl: false,
|
28
|
-
heartbeat: 30,
|
29
|
-
# placeholder, allows specifying connection parameters
|
30
|
-
# as a URI.
|
31
163
|
uri: nil,
|
32
164
|
log_level: Logger::INFO,
|
165
|
+
client_logger: nil,
|
33
166
|
require_paths: [],
|
34
|
-
autoload_rails: true,
|
35
167
|
error_handlers: [Hutch::ErrorHandlers::Logger.new],
|
36
168
|
# note that this is not a list, it is a chain of responsibility
|
37
169
|
# that will fall back to "nack unconditionally"
|
38
170
|
error_acknowledgements: [],
|
171
|
+
setup_procs: [],
|
172
|
+
consumer_groups: {},
|
39
173
|
tracer: Hutch::Tracers::NullTracer,
|
40
174
|
namespace: nil,
|
41
|
-
daemonise: false,
|
42
175
|
pidfile: nil,
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
176
|
+
serializer: Hutch::Serializers::JSON
|
177
|
+
})
|
178
|
+
end
|
179
|
+
|
180
|
+
# Override defaults with ENV variables which begin with <tt>HUTCH_</tt>
|
181
|
+
#
|
182
|
+
# @return [Hash]
|
183
|
+
def self.env_based_config
|
184
|
+
env_keys_configured.each_with_object({}) {|attr, result|
|
185
|
+
value = ENV[key_for(attr)]
|
186
|
+
|
187
|
+
case
|
188
|
+
when is_bool(attr) || value == 'false'
|
189
|
+
result[attr] = to_bool(value)
|
190
|
+
when is_num(attr)
|
191
|
+
result[attr] = value.to_i
|
192
|
+
else
|
193
|
+
result[attr] = value
|
194
|
+
end
|
195
|
+
}
|
196
|
+
end
|
60
197
|
|
61
|
-
|
198
|
+
# @return [Array<Symbol>]
|
199
|
+
def self.env_keys_configured
|
200
|
+
ALL_KEYS.each {|attr| check_attr(attr) }
|
62
201
|
|
63
|
-
|
64
|
-
}.merge(params)
|
202
|
+
ALL_KEYS.select { |attr| ENV.key?(key_for(attr)) }
|
65
203
|
end
|
66
204
|
|
67
205
|
def self.get(attr)
|
68
|
-
check_attr(attr)
|
69
|
-
user_config[attr]
|
206
|
+
check_attr(attr.to_sym)
|
207
|
+
user_config[attr.to_sym]
|
208
|
+
end
|
209
|
+
|
210
|
+
def self.key_for(attr)
|
211
|
+
key = attr.to_s.gsub('.', '__').upcase
|
212
|
+
"HUTCH_#{key}"
|
213
|
+
end
|
214
|
+
|
215
|
+
def self.is_bool(attr)
|
216
|
+
@boolean_keys.include?(attr)
|
217
|
+
end
|
218
|
+
|
219
|
+
def self.to_bool(value)
|
220
|
+
!(value.nil? || value == '' || value =~ /^(false|f|no|n|0)$/i || value == false)
|
221
|
+
end
|
222
|
+
|
223
|
+
def self.is_num(attr)
|
224
|
+
@number_keys.include?(attr)
|
70
225
|
end
|
71
226
|
|
72
227
|
def self.set(attr, value)
|
73
|
-
check_attr(attr)
|
74
|
-
user_config[attr] = value
|
228
|
+
check_attr(attr.to_sym)
|
229
|
+
user_config[attr.to_sym] = value
|
75
230
|
end
|
76
231
|
|
77
232
|
class << self
|
@@ -81,17 +236,16 @@ module Hutch
|
|
81
236
|
|
82
237
|
def self.check_attr(attr)
|
83
238
|
unless user_config.key?(attr)
|
84
|
-
raise UnknownAttributeError, "#{attr} is not a valid config attribute"
|
239
|
+
raise UnknownAttributeError, "#{attr.inspect} is not a valid config attribute"
|
85
240
|
end
|
86
241
|
end
|
87
242
|
|
88
243
|
def self.user_config
|
89
|
-
|
90
|
-
@config
|
244
|
+
@config ||= initialize
|
91
245
|
end
|
92
246
|
|
93
247
|
def self.to_hash
|
94
|
-
|
248
|
+
user_config
|
95
249
|
end
|
96
250
|
|
97
251
|
def self.load_from_file(file)
|
@@ -102,28 +256,24 @@ module Hutch
|
|
102
256
|
|
103
257
|
def self.convert_value(attr, value)
|
104
258
|
case attr
|
105
|
-
when
|
259
|
+
when 'tracer'
|
106
260
|
Kernel.const_get(value)
|
107
261
|
else
|
108
262
|
value
|
109
263
|
end
|
110
264
|
end
|
111
265
|
|
112
|
-
def self.
|
113
|
-
|
114
|
-
|
266
|
+
def self.define_methods
|
267
|
+
@config.keys.each do |key|
|
268
|
+
define_singleton_method(key) do
|
269
|
+
get(key)
|
270
|
+
end
|
115
271
|
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
get(attr)
|
272
|
+
define_singleton_method("#{key}=") do |val|
|
273
|
+
set(key, val)
|
274
|
+
end
|
120
275
|
end
|
121
276
|
end
|
122
|
-
|
123
|
-
private
|
124
|
-
|
125
|
-
def deep_copy(obj)
|
126
|
-
Marshal.load(Marshal.dump(obj))
|
127
|
-
end
|
128
277
|
end
|
129
278
|
end
|
279
|
+
Hutch::Config.initialize
|
data/lib/hutch/error_handlers.rb
CHANGED