hutch 0.21.0 → 0.22.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +2 -0
- data/.yardopts +5 -0
- data/CHANGELOG.md +55 -1
- data/Gemfile +1 -1
- data/README.md +2 -2
- data/Rakefile +8 -1
- data/hutch.gemspec +4 -1
- data/lib/hutch.rb +11 -8
- data/lib/hutch/adapters/march_hare.rb +1 -1
- data/lib/hutch/broker.rb +89 -110
- data/lib/hutch/cli.rb +34 -10
- data/lib/hutch/config.rb +192 -55
- data/lib/hutch/error_handlers/sentry.rb +1 -1
- data/lib/hutch/logging.rb +1 -0
- data/lib/hutch/publisher.rb +75 -0
- data/lib/hutch/version.rb +1 -1
- data/lib/hutch/waiter.rb +41 -0
- data/lib/hutch/worker.rb +8 -60
- data/lib/yard-settings/handler.rb +38 -0
- data/lib/yard-settings/yard-settings.rb +2 -0
- data/spec/hutch/broker_spec.rb +107 -64
- data/spec/hutch/cli_spec.rb +3 -3
- data/spec/hutch/config_spec.rb +60 -22
- data/spec/hutch/error_handlers/sentry_spec.rb +1 -1
- data/spec/hutch/logger_spec.rb +12 -6
- data/spec/hutch/waiter_spec.rb +33 -0
- data/spec/hutch/worker_spec.rb +13 -2
- data/spec/spec_helper.rb +7 -5
- data/templates/default/class/html/settings.erb +0 -0
- data/templates/default/class/setup.rb +4 -0
- data/templates/default/fulldoc/html/css/hutch.css +13 -0
- data/templates/default/layout/html/setup.rb +7 -0
- data/templates/default/method_details/html/settings.erb +5 -0
- data/templates/default/method_details/setup.rb +4 -0
- data/templates/default/method_details/text/settings.erb +0 -0
- data/templates/default/module/html/settings.erb +40 -0
- data/templates/default/module/setup.rb +4 -0
- metadata +63 -5
data/lib/hutch/cli.rb
CHANGED
@@ -1,9 +1,9 @@
|
|
1
1
|
require 'optparse'
|
2
2
|
|
3
|
-
require 'hutch/version'
|
4
3
|
require 'hutch/logging'
|
5
|
-
require 'hutch/exceptions'
|
6
4
|
require 'hutch/config'
|
5
|
+
require 'hutch/version'
|
6
|
+
require 'hutch/exceptions'
|
7
7
|
|
8
8
|
module Hutch
|
9
9
|
class CLI
|
@@ -11,9 +11,10 @@ module Hutch
|
|
11
11
|
|
12
12
|
# Run a Hutch worker with the command line interface.
|
13
13
|
def run(argv = ARGV)
|
14
|
+
Hutch::Config.initialize
|
14
15
|
parse_options(argv)
|
15
16
|
|
16
|
-
|
17
|
+
daemonise_process
|
17
18
|
|
18
19
|
write_pid if Hutch::Config.pidfile
|
19
20
|
|
@@ -85,7 +86,7 @@ module Hutch
|
|
85
86
|
# gracefully (with a SIGQUIT, SIGTERM or SIGINT).
|
86
87
|
def start_work_loop
|
87
88
|
Hutch.connect
|
88
|
-
@worker = Hutch::Worker.new(Hutch.broker, Hutch.consumers)
|
89
|
+
@worker = Hutch::Worker.new(Hutch.broker, Hutch.consumers, Hutch::Config.setup_procs)
|
89
90
|
@worker.run
|
90
91
|
:success
|
91
92
|
rescue ConnectionError, AuthenticationError, WorkerSetupError => ex
|
@@ -109,14 +110,16 @@ module Hutch
|
|
109
110
|
Hutch::Config.mq_tls = tls
|
110
111
|
end
|
111
112
|
|
112
|
-
opts.on('--mq-tls-cert FILE', 'Certificate
|
113
|
-
|
114
|
-
|
113
|
+
opts.on('--mq-tls-cert FILE', 'Certificate for TLS client verification') do |file|
|
114
|
+
abort_without_file(file, 'Certificate file') do
|
115
|
+
Hutch::Config.mq_tls_cert = file
|
116
|
+
end
|
115
117
|
end
|
116
118
|
|
117
119
|
opts.on('--mq-tls-key FILE', 'Private key for TLS client verification') do |file|
|
118
|
-
|
119
|
-
|
120
|
+
abort_without_file(file, 'Private key file') do
|
121
|
+
Hutch::Config.mq_tls_key = file
|
122
|
+
end
|
120
123
|
end
|
121
124
|
|
122
125
|
opts.on('--mq-exchange EXCHANGE',
|
@@ -154,7 +157,7 @@ module Hutch
|
|
154
157
|
begin
|
155
158
|
File.open(file) { |fp| Hutch::Config.load_from_file(fp) }
|
156
159
|
rescue Errno::ENOENT
|
157
|
-
|
160
|
+
abort_with_message("Config file '#{file}' not found")
|
158
161
|
end
|
159
162
|
end
|
160
163
|
|
@@ -204,5 +207,26 @@ module Hutch
|
|
204
207
|
File.open(pidfile, 'w') { |f| f.puts ::Process.pid }
|
205
208
|
end
|
206
209
|
|
210
|
+
private
|
211
|
+
|
212
|
+
def daemonise_process
|
213
|
+
return unless Hutch::Config.daemonise
|
214
|
+
if defined?(JRUBY_VERSION)
|
215
|
+
Hutch.logger.warn "JRuby ignores the --daemonise option"
|
216
|
+
return
|
217
|
+
end
|
218
|
+
|
219
|
+
::Process.daemon(true)
|
220
|
+
end
|
221
|
+
|
222
|
+
def abort_without_file(file, file_description, &block)
|
223
|
+
abort_with_message("#{file_description} '#{file}' not found") unless File.exists?(file)
|
224
|
+
|
225
|
+
yield
|
226
|
+
end
|
227
|
+
|
228
|
+
def abort_with_message(message)
|
229
|
+
abort message
|
230
|
+
end
|
207
231
|
end
|
208
232
|
end
|
data/lib/hutch/config.rb
CHANGED
@@ -1,67 +1,193 @@
|
|
1
1
|
require 'hutch/error_handlers/logger'
|
2
|
+
require 'hutch/tracers'
|
3
|
+
require 'hutch/serializers/json'
|
2
4
|
require 'erb'
|
3
5
|
require 'logger'
|
4
6
|
|
5
7
|
module Hutch
|
6
8
|
class UnknownAttributeError < StandardError; end
|
7
9
|
|
10
|
+
# Configuration settings, available everywhere
|
11
|
+
#
|
12
|
+
# There are defaults, which can be overridden by ENV variables prefixed by
|
13
|
+
# <tt>HUTCH_</tt>, and each of these can be overridden using the {.set}
|
14
|
+
# method.
|
15
|
+
#
|
16
|
+
# @example Configuring on the command-line
|
17
|
+
# HUTCH_PUBLISHER_CONFIRMS=false hutch
|
8
18
|
module Config
|
9
19
|
require 'yaml'
|
20
|
+
@string_keys = Set.new
|
21
|
+
@number_keys = Set.new
|
22
|
+
@boolean_keys = Set.new
|
23
|
+
@settings_defaults = {}
|
24
|
+
|
25
|
+
# Define a String user setting
|
26
|
+
# @!visibility private
|
27
|
+
def self.string_setting(name, default_value)
|
28
|
+
@string_keys << name
|
29
|
+
@settings_defaults[name] = default_value
|
30
|
+
end
|
31
|
+
|
32
|
+
# Define a Number user setting
|
33
|
+
# @!visibility private
|
34
|
+
def self.number_setting(name, default_value)
|
35
|
+
@number_keys << name
|
36
|
+
@settings_defaults[name] = default_value
|
37
|
+
end
|
38
|
+
|
39
|
+
# Define a Boolean user setting
|
40
|
+
# @!visibility private
|
41
|
+
def self.boolean_setting(name, default_value)
|
42
|
+
@boolean_keys << name
|
43
|
+
@settings_defaults[name] = default_value
|
44
|
+
end
|
45
|
+
|
46
|
+
# RabbitMQ hostname
|
47
|
+
string_setting :mq_host, '127.0.0.1'
|
48
|
+
|
49
|
+
# RabbitMQ Exchange to use for publishing
|
50
|
+
string_setting :mq_exchange, 'hutch'
|
51
|
+
|
52
|
+
# RabbitMQ vhost to use
|
53
|
+
string_setting :mq_vhost, '/'
|
54
|
+
|
55
|
+
# RabbitMQ username to use.
|
56
|
+
#
|
57
|
+
# As of RabbitMQ 3.3.0, <tt>guest</tt> can only can connect from localhost.
|
58
|
+
string_setting :mq_username, 'guest'
|
59
|
+
|
60
|
+
# RabbitMQ password
|
61
|
+
string_setting :mq_password, 'guest'
|
62
|
+
|
63
|
+
# RabbitMQ HTTP API hostname
|
64
|
+
string_setting :mq_api_host, '127.0.0.1'
|
65
|
+
|
66
|
+
# RabbitMQ port
|
67
|
+
number_setting :mq_port, 5672
|
68
|
+
|
69
|
+
# RabbitMQ HTTP API port
|
70
|
+
number_setting :mq_api_port, 15672
|
71
|
+
|
72
|
+
# [RabbitMQ heartbeat timeout](http://rabbitmq.com/heartbeats.html)
|
73
|
+
number_setting :heartbeat, 30
|
74
|
+
|
75
|
+
# The <tt>basic.qos</tt> prefetch value to use.
|
76
|
+
#
|
77
|
+
# Default: `0`, no limit. See Bunny and RabbitMQ documentation.
|
78
|
+
number_setting :channel_prefetch, 0
|
79
|
+
|
80
|
+
# Bunny's socket open timeout
|
81
|
+
number_setting :connection_timeout, 11
|
82
|
+
|
83
|
+
# Bunny's socket read timeout
|
84
|
+
number_setting :read_timeout, 11
|
85
|
+
|
86
|
+
# Bunny's socket write timeout
|
87
|
+
number_setting :write_timeout, 11
|
88
|
+
|
89
|
+
# FIXME: DOCUMENT THIS
|
90
|
+
number_setting :graceful_exit_timeout, 11
|
91
|
+
|
92
|
+
# Bunny consumer work pool size
|
93
|
+
number_setting :consumer_pool_size, 1
|
94
|
+
|
95
|
+
# Should TLS be used?
|
96
|
+
boolean_setting :mq_tls, false
|
97
|
+
|
98
|
+
# Should SSL certificate be verified?
|
99
|
+
boolean_setting :mq_verify_peer, true
|
100
|
+
|
101
|
+
# Should SSL be used for the RabbitMQ API?
|
102
|
+
boolean_setting :mq_api_ssl, false
|
103
|
+
|
104
|
+
# Should the current Rails app directory be required?
|
105
|
+
boolean_setting :autoload_rails, true
|
106
|
+
|
107
|
+
# Should the Hutch runner process daemonise?
|
108
|
+
#
|
109
|
+
# The option is ignored on JRuby.
|
110
|
+
boolean_setting :daemonise, false
|
111
|
+
|
112
|
+
# Should RabbitMQ publisher confirms be enabled?
|
113
|
+
#
|
114
|
+
# Leaves it up to the app how they are tracked
|
115
|
+
# (e.g. using Hutch::Broker#confirm_select callback or Hutch::Broker#wait_for_confirms)
|
116
|
+
boolean_setting :publisher_confirms, false
|
117
|
+
|
118
|
+
# Enables publisher confirms, forces Hutch::Broker#wait_for_confirms for
|
119
|
+
# every publish.
|
120
|
+
#
|
121
|
+
# **This is the safest option which also offers the
|
122
|
+
# lowest throughput**.
|
123
|
+
boolean_setting :force_publisher_confirms, false
|
124
|
+
|
125
|
+
# Should the RabbitMQ HTTP API be used?
|
126
|
+
boolean_setting :enable_http_api_use, true
|
127
|
+
|
128
|
+
# Should Bunny's consumer work pool threads abort on exception.
|
129
|
+
#
|
130
|
+
# The option is ignored on JRuby.
|
131
|
+
boolean_setting :consumer_pool_abort_on_exception, false
|
132
|
+
|
133
|
+
# Set of all setting keys
|
134
|
+
ALL_KEYS = @boolean_keys + @number_keys + @string_keys
|
10
135
|
|
11
136
|
def self.initialize(params = {})
|
12
|
-
@config =
|
13
|
-
|
14
|
-
|
15
|
-
|
137
|
+
@config = default_config
|
138
|
+
@config.merge!(env_based_config).merge!(params)
|
139
|
+
define_methods
|
140
|
+
@config
|
141
|
+
end
|
142
|
+
|
143
|
+
# Default settings
|
144
|
+
#
|
145
|
+
# @return [Hash]
|
146
|
+
def self.default_config
|
147
|
+
@settings_defaults.merge({
|
16
148
|
mq_exchange_options: {},
|
17
|
-
mq_vhost: '/',
|
18
|
-
mq_tls: false,
|
19
149
|
mq_tls_cert: nil,
|
20
150
|
mq_tls_key: nil,
|
21
151
|
mq_tls_ca_certificates: nil,
|
22
|
-
mq_verify_peer: true,
|
23
|
-
mq_username: 'guest',
|
24
|
-
mq_password: 'guest',
|
25
|
-
mq_api_host: 'localhost',
|
26
|
-
mq_api_port: 15672,
|
27
|
-
mq_api_ssl: false,
|
28
|
-
heartbeat: 30,
|
29
|
-
# placeholder, allows specifying connection parameters
|
30
|
-
# as a URI.
|
31
152
|
uri: nil,
|
32
153
|
log_level: Logger::INFO,
|
154
|
+
client_logger: nil,
|
33
155
|
require_paths: [],
|
34
|
-
autoload_rails: true,
|
35
156
|
error_handlers: [Hutch::ErrorHandlers::Logger.new],
|
36
157
|
# note that this is not a list, it is a chain of responsibility
|
37
158
|
# that will fall back to "nack unconditionally"
|
38
159
|
error_acknowledgements: [],
|
160
|
+
setup_procs: [],
|
39
161
|
tracer: Hutch::Tracers::NullTracer,
|
40
162
|
namespace: nil,
|
41
|
-
daemonise: false,
|
42
163
|
pidfile: nil,
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
164
|
+
serializer: Hutch::Serializers::JSON
|
165
|
+
})
|
166
|
+
end
|
167
|
+
|
168
|
+
# Override defaults with ENV variables which begin with <tt>HUTCH_</tt>
|
169
|
+
#
|
170
|
+
# @return [Hash]
|
171
|
+
def self.env_based_config
|
172
|
+
env_keys_configured.each_with_object({}) {|attr, result|
|
173
|
+
value = ENV[key_for(attr)]
|
174
|
+
|
175
|
+
case
|
176
|
+
when is_bool(attr) || value == 'false'
|
177
|
+
result[attr] = to_bool(value)
|
178
|
+
when is_num(attr)
|
179
|
+
result[attr] = value.to_i
|
180
|
+
else
|
181
|
+
result[attr] = value
|
182
|
+
end
|
183
|
+
}
|
184
|
+
end
|
60
185
|
|
61
|
-
|
186
|
+
# @return [Array<Symbol>]
|
187
|
+
def self.env_keys_configured
|
188
|
+
ALL_KEYS.each {|attr| check_attr(attr) }
|
62
189
|
|
63
|
-
|
64
|
-
}.merge(params)
|
190
|
+
ALL_KEYS.select { |attr| ENV.key?(key_for(attr)) }
|
65
191
|
end
|
66
192
|
|
67
193
|
def self.get(attr)
|
@@ -69,6 +195,23 @@ module Hutch
|
|
69
195
|
user_config[attr]
|
70
196
|
end
|
71
197
|
|
198
|
+
def self.key_for(attr)
|
199
|
+
key = attr.to_s.gsub('.', '__').upcase
|
200
|
+
"HUTCH_#{key}"
|
201
|
+
end
|
202
|
+
|
203
|
+
def self.is_bool(attr)
|
204
|
+
@boolean_keys.include?(attr)
|
205
|
+
end
|
206
|
+
|
207
|
+
def self.to_bool(value)
|
208
|
+
!(value.nil? || value == '' || value =~ /^(false|f|no|n|0)$/i || value == false)
|
209
|
+
end
|
210
|
+
|
211
|
+
def self.is_num(attr)
|
212
|
+
@number_keys.include?(attr)
|
213
|
+
end
|
214
|
+
|
72
215
|
def self.set(attr, value)
|
73
216
|
check_attr(attr)
|
74
217
|
user_config[attr] = value
|
@@ -81,17 +224,16 @@ module Hutch
|
|
81
224
|
|
82
225
|
def self.check_attr(attr)
|
83
226
|
unless user_config.key?(attr)
|
84
|
-
raise UnknownAttributeError, "#{attr} is not a valid config attribute"
|
227
|
+
raise UnknownAttributeError, "#{attr.inspect} is not a valid config attribute"
|
85
228
|
end
|
86
229
|
end
|
87
230
|
|
88
231
|
def self.user_config
|
89
|
-
|
90
|
-
@config
|
232
|
+
@config ||= initialize
|
91
233
|
end
|
92
234
|
|
93
235
|
def self.to_hash
|
94
|
-
|
236
|
+
user_config
|
95
237
|
end
|
96
238
|
|
97
239
|
def self.load_from_file(file)
|
@@ -102,28 +244,23 @@ module Hutch
|
|
102
244
|
|
103
245
|
def self.convert_value(attr, value)
|
104
246
|
case attr
|
105
|
-
when
|
247
|
+
when 'tracer'
|
106
248
|
Kernel.const_get(value)
|
107
249
|
else
|
108
250
|
value
|
109
251
|
end
|
110
252
|
end
|
111
253
|
|
112
|
-
def self.
|
113
|
-
|
114
|
-
|
254
|
+
def self.define_methods
|
255
|
+
@config.keys.each do |key|
|
256
|
+
define_singleton_method(key) do
|
257
|
+
get(key)
|
258
|
+
end
|
115
259
|
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
get(attr)
|
260
|
+
define_singleton_method("#{key}=") do |val|
|
261
|
+
set(key, val)
|
262
|
+
end
|
120
263
|
end
|
121
264
|
end
|
122
|
-
|
123
|
-
private
|
124
|
-
|
125
|
-
def deep_copy(obj)
|
126
|
-
Marshal.load(Marshal.dump(obj))
|
127
|
-
end
|
128
265
|
end
|
129
266
|
end
|
@@ -16,7 +16,7 @@ module Hutch
|
|
16
16
|
prefix = "message(#{message_id || '-'}): "
|
17
17
|
logger.error prefix + "Logging event to Sentry"
|
18
18
|
logger.error prefix + "#{ex.class} - #{ex.message}"
|
19
|
-
Raven.capture_exception(ex)
|
19
|
+
Raven.capture_exception(ex, extra: { payload: payload })
|
20
20
|
end
|
21
21
|
end
|
22
22
|
end
|
data/lib/hutch/logging.rb
CHANGED
@@ -0,0 +1,75 @@
|
|
1
|
+
require 'securerandom'
|
2
|
+
require 'hutch/logging'
|
3
|
+
require 'hutch/exceptions'
|
4
|
+
|
5
|
+
module Hutch
|
6
|
+
class Publisher
|
7
|
+
include Logging
|
8
|
+
attr_reader :connection, :channel, :exchange, :config
|
9
|
+
|
10
|
+
def initialize(connection, channel, exchange, config = Hutch::Config)
|
11
|
+
@connection = connection
|
12
|
+
@channel = channel
|
13
|
+
@exchange = exchange
|
14
|
+
@config = config
|
15
|
+
end
|
16
|
+
|
17
|
+
def publish(routing_key, message, properties = {}, options = {})
|
18
|
+
ensure_connection!(routing_key, message)
|
19
|
+
|
20
|
+
serializer = options[:serializer] || config[:serializer]
|
21
|
+
|
22
|
+
non_overridable_properties = {
|
23
|
+
routing_key: routing_key,
|
24
|
+
timestamp: connection.current_timestamp,
|
25
|
+
content_type: serializer.content_type,
|
26
|
+
}
|
27
|
+
properties[:message_id] ||= generate_id
|
28
|
+
|
29
|
+
payload = serializer.encode(message)
|
30
|
+
|
31
|
+
log_publication(serializer, payload, routing_key)
|
32
|
+
|
33
|
+
response = exchange.publish(payload, {persistent: true}.
|
34
|
+
merge(properties).
|
35
|
+
merge(global_properties).
|
36
|
+
merge(non_overridable_properties))
|
37
|
+
|
38
|
+
channel.wait_for_confirms if config[:force_publisher_confirms]
|
39
|
+
response
|
40
|
+
end
|
41
|
+
|
42
|
+
private
|
43
|
+
|
44
|
+
def log_publication(serializer, payload, routing_key)
|
45
|
+
logger.info {
|
46
|
+
spec =
|
47
|
+
if serializer.binary?
|
48
|
+
"#{payload.bytesize} bytes message"
|
49
|
+
else
|
50
|
+
"message '#{payload}'"
|
51
|
+
end
|
52
|
+
"publishing #{spec} to #{routing_key}"
|
53
|
+
}
|
54
|
+
end
|
55
|
+
|
56
|
+
def raise_publish_error(reason, routing_key, message)
|
57
|
+
msg = "unable to publish - #{reason}. Message: #{JSON.dump(message)}, Routing key: #{routing_key}."
|
58
|
+
logger.error(msg)
|
59
|
+
raise PublishError, msg
|
60
|
+
end
|
61
|
+
|
62
|
+
def ensure_connection!(routing_key, message)
|
63
|
+
raise_publish_error('no connection to broker', routing_key, message) unless connection
|
64
|
+
raise_publish_error('connection is closed', routing_key, message) unless connection.open?
|
65
|
+
end
|
66
|
+
|
67
|
+
def generate_id
|
68
|
+
SecureRandom.uuid
|
69
|
+
end
|
70
|
+
|
71
|
+
def global_properties
|
72
|
+
Hutch.global_properties.respond_to?(:call) ? Hutch.global_properties.call : Hutch.global_properties
|
73
|
+
end
|
74
|
+
end
|
75
|
+
end
|