hutch 0.21.0 → 0.22.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,9 +1,9 @@
1
1
  require 'optparse'
2
2
 
3
- require 'hutch/version'
4
3
  require 'hutch/logging'
5
- require 'hutch/exceptions'
6
4
  require 'hutch/config'
5
+ require 'hutch/version'
6
+ require 'hutch/exceptions'
7
7
 
8
8
  module Hutch
9
9
  class CLI
@@ -11,9 +11,10 @@ module Hutch
11
11
 
12
12
  # Run a Hutch worker with the command line interface.
13
13
  def run(argv = ARGV)
14
+ Hutch::Config.initialize
14
15
  parse_options(argv)
15
16
 
16
- ::Process.daemon(true) if Hutch::Config.daemonise
17
+ daemonise_process
17
18
 
18
19
  write_pid if Hutch::Config.pidfile
19
20
 
@@ -85,7 +86,7 @@ module Hutch
85
86
  # gracefully (with a SIGQUIT, SIGTERM or SIGINT).
86
87
  def start_work_loop
87
88
  Hutch.connect
88
- @worker = Hutch::Worker.new(Hutch.broker, Hutch.consumers)
89
+ @worker = Hutch::Worker.new(Hutch.broker, Hutch.consumers, Hutch::Config.setup_procs)
89
90
  @worker.run
90
91
  :success
91
92
  rescue ConnectionError, AuthenticationError, WorkerSetupError => ex
@@ -109,14 +110,16 @@ module Hutch
109
110
  Hutch::Config.mq_tls = tls
110
111
  end
111
112
 
112
- opts.on('--mq-tls-cert FILE', 'Certificate for TLS client verification') do |file|
113
- abort "Certificate file '#{file}' not found" unless File.exists?(file)
114
- Hutch::Config.mq_tls_cert = file
113
+ opts.on('--mq-tls-cert FILE', 'Certificate for TLS client verification') do |file|
114
+ abort_without_file(file, 'Certificate file') do
115
+ Hutch::Config.mq_tls_cert = file
116
+ end
115
117
  end
116
118
 
117
119
  opts.on('--mq-tls-key FILE', 'Private key for TLS client verification') do |file|
118
- abort "Private key file '#{file}' not found" unless File.exists?(file)
119
- Hutch::Config.mq_tls_key = file
120
+ abort_without_file(file, 'Private key file') do
121
+ Hutch::Config.mq_tls_key = file
122
+ end
120
123
  end
121
124
 
122
125
  opts.on('--mq-exchange EXCHANGE',
@@ -154,7 +157,7 @@ module Hutch
154
157
  begin
155
158
  File.open(file) { |fp| Hutch::Config.load_from_file(fp) }
156
159
  rescue Errno::ENOENT
157
- abort "Config file '#{file}' not found"
160
+ abort_with_message("Config file '#{file}' not found")
158
161
  end
159
162
  end
160
163
 
@@ -204,5 +207,26 @@ module Hutch
204
207
  File.open(pidfile, 'w') { |f| f.puts ::Process.pid }
205
208
  end
206
209
 
210
+ private
211
+
212
+ def daemonise_process
213
+ return unless Hutch::Config.daemonise
214
+ if defined?(JRUBY_VERSION)
215
+ Hutch.logger.warn "JRuby ignores the --daemonise option"
216
+ return
217
+ end
218
+
219
+ ::Process.daemon(true)
220
+ end
221
+
222
+ def abort_without_file(file, file_description, &block)
223
+ abort_with_message("#{file_description} '#{file}' not found") unless File.exists?(file)
224
+
225
+ yield
226
+ end
227
+
228
+ def abort_with_message(message)
229
+ abort message
230
+ end
207
231
  end
208
232
  end
@@ -1,67 +1,193 @@
1
1
  require 'hutch/error_handlers/logger'
2
+ require 'hutch/tracers'
3
+ require 'hutch/serializers/json'
2
4
  require 'erb'
3
5
  require 'logger'
4
6
 
5
7
  module Hutch
6
8
  class UnknownAttributeError < StandardError; end
7
9
 
10
+ # Configuration settings, available everywhere
11
+ #
12
+ # There are defaults, which can be overridden by ENV variables prefixed by
13
+ # <tt>HUTCH_</tt>, and each of these can be overridden using the {.set}
14
+ # method.
15
+ #
16
+ # @example Configuring on the command-line
17
+ # HUTCH_PUBLISHER_CONFIRMS=false hutch
8
18
  module Config
9
19
  require 'yaml'
20
+ @string_keys = Set.new
21
+ @number_keys = Set.new
22
+ @boolean_keys = Set.new
23
+ @settings_defaults = {}
24
+
25
+ # Define a String user setting
26
+ # @!visibility private
27
+ def self.string_setting(name, default_value)
28
+ @string_keys << name
29
+ @settings_defaults[name] = default_value
30
+ end
31
+
32
+ # Define a Number user setting
33
+ # @!visibility private
34
+ def self.number_setting(name, default_value)
35
+ @number_keys << name
36
+ @settings_defaults[name] = default_value
37
+ end
38
+
39
+ # Define a Boolean user setting
40
+ # @!visibility private
41
+ def self.boolean_setting(name, default_value)
42
+ @boolean_keys << name
43
+ @settings_defaults[name] = default_value
44
+ end
45
+
46
+ # RabbitMQ hostname
47
+ string_setting :mq_host, '127.0.0.1'
48
+
49
+ # RabbitMQ Exchange to use for publishing
50
+ string_setting :mq_exchange, 'hutch'
51
+
52
+ # RabbitMQ vhost to use
53
+ string_setting :mq_vhost, '/'
54
+
55
+ # RabbitMQ username to use.
56
+ #
57
+ # As of RabbitMQ 3.3.0, <tt>guest</tt> can only can connect from localhost.
58
+ string_setting :mq_username, 'guest'
59
+
60
+ # RabbitMQ password
61
+ string_setting :mq_password, 'guest'
62
+
63
+ # RabbitMQ HTTP API hostname
64
+ string_setting :mq_api_host, '127.0.0.1'
65
+
66
+ # RabbitMQ port
67
+ number_setting :mq_port, 5672
68
+
69
+ # RabbitMQ HTTP API port
70
+ number_setting :mq_api_port, 15672
71
+
72
+ # [RabbitMQ heartbeat timeout](http://rabbitmq.com/heartbeats.html)
73
+ number_setting :heartbeat, 30
74
+
75
+ # The <tt>basic.qos</tt> prefetch value to use.
76
+ #
77
+ # Default: `0`, no limit. See Bunny and RabbitMQ documentation.
78
+ number_setting :channel_prefetch, 0
79
+
80
+ # Bunny's socket open timeout
81
+ number_setting :connection_timeout, 11
82
+
83
+ # Bunny's socket read timeout
84
+ number_setting :read_timeout, 11
85
+
86
+ # Bunny's socket write timeout
87
+ number_setting :write_timeout, 11
88
+
89
+ # FIXME: DOCUMENT THIS
90
+ number_setting :graceful_exit_timeout, 11
91
+
92
+ # Bunny consumer work pool size
93
+ number_setting :consumer_pool_size, 1
94
+
95
+ # Should TLS be used?
96
+ boolean_setting :mq_tls, false
97
+
98
+ # Should SSL certificate be verified?
99
+ boolean_setting :mq_verify_peer, true
100
+
101
+ # Should SSL be used for the RabbitMQ API?
102
+ boolean_setting :mq_api_ssl, false
103
+
104
+ # Should the current Rails app directory be required?
105
+ boolean_setting :autoload_rails, true
106
+
107
+ # Should the Hutch runner process daemonise?
108
+ #
109
+ # The option is ignored on JRuby.
110
+ boolean_setting :daemonise, false
111
+
112
+ # Should RabbitMQ publisher confirms be enabled?
113
+ #
114
+ # Leaves it up to the app how they are tracked
115
+ # (e.g. using Hutch::Broker#confirm_select callback or Hutch::Broker#wait_for_confirms)
116
+ boolean_setting :publisher_confirms, false
117
+
118
+ # Enables publisher confirms, forces Hutch::Broker#wait_for_confirms for
119
+ # every publish.
120
+ #
121
+ # **This is the safest option which also offers the
122
+ # lowest throughput**.
123
+ boolean_setting :force_publisher_confirms, false
124
+
125
+ # Should the RabbitMQ HTTP API be used?
126
+ boolean_setting :enable_http_api_use, true
127
+
128
+ # Should Bunny's consumer work pool threads abort on exception.
129
+ #
130
+ # The option is ignored on JRuby.
131
+ boolean_setting :consumer_pool_abort_on_exception, false
132
+
133
+ # Set of all setting keys
134
+ ALL_KEYS = @boolean_keys + @number_keys + @string_keys
10
135
 
11
136
  def self.initialize(params = {})
12
- @config = {
13
- mq_host: 'localhost',
14
- mq_port: 5672,
15
- mq_exchange: 'hutch', # TODO: should this be required?
137
+ @config = default_config
138
+ @config.merge!(env_based_config).merge!(params)
139
+ define_methods
140
+ @config
141
+ end
142
+
143
+ # Default settings
144
+ #
145
+ # @return [Hash]
146
+ def self.default_config
147
+ @settings_defaults.merge({
16
148
  mq_exchange_options: {},
17
- mq_vhost: '/',
18
- mq_tls: false,
19
149
  mq_tls_cert: nil,
20
150
  mq_tls_key: nil,
21
151
  mq_tls_ca_certificates: nil,
22
- mq_verify_peer: true,
23
- mq_username: 'guest',
24
- mq_password: 'guest',
25
- mq_api_host: 'localhost',
26
- mq_api_port: 15672,
27
- mq_api_ssl: false,
28
- heartbeat: 30,
29
- # placeholder, allows specifying connection parameters
30
- # as a URI.
31
152
  uri: nil,
32
153
  log_level: Logger::INFO,
154
+ client_logger: nil,
33
155
  require_paths: [],
34
- autoload_rails: true,
35
156
  error_handlers: [Hutch::ErrorHandlers::Logger.new],
36
157
  # note that this is not a list, it is a chain of responsibility
37
158
  # that will fall back to "nack unconditionally"
38
159
  error_acknowledgements: [],
160
+ setup_procs: [],
39
161
  tracer: Hutch::Tracers::NullTracer,
40
162
  namespace: nil,
41
- daemonise: false,
42
163
  pidfile: nil,
43
- channel_prefetch: 0,
44
- # enables publisher confirms, leaves it up to the app
45
- # how they are tracked
46
- publisher_confirms: false,
47
- # like `publisher_confirms` above but also
48
- # forces waiting for a confirm for every publish
49
- force_publisher_confirms: false,
50
- # Heroku needs > 10. MK.
51
- connection_timeout: 11,
52
- read_timeout: 11,
53
- write_timeout: 11,
54
- enable_http_api_use: true,
55
- # Number of seconds that a running consumer is given
56
- # to finish its job when gracefully exiting Hutch, before
57
- # it's killed.
58
- graceful_exit_timeout: 11,
59
- client_logger: nil,
164
+ serializer: Hutch::Serializers::JSON
165
+ })
166
+ end
167
+
168
+ # Override defaults with ENV variables which begin with <tt>HUTCH_</tt>
169
+ #
170
+ # @return [Hash]
171
+ def self.env_based_config
172
+ env_keys_configured.each_with_object({}) {|attr, result|
173
+ value = ENV[key_for(attr)]
174
+
175
+ case
176
+ when is_bool(attr) || value == 'false'
177
+ result[attr] = to_bool(value)
178
+ when is_num(attr)
179
+ result[attr] = value.to_i
180
+ else
181
+ result[attr] = value
182
+ end
183
+ }
184
+ end
60
185
 
61
- consumer_pool_size: 1,
186
+ # @return [Array<Symbol>]
187
+ def self.env_keys_configured
188
+ ALL_KEYS.each {|attr| check_attr(attr) }
62
189
 
63
- serializer: Hutch::Serializers::JSON,
64
- }.merge(params)
190
+ ALL_KEYS.select { |attr| ENV.key?(key_for(attr)) }
65
191
  end
66
192
 
67
193
  def self.get(attr)
@@ -69,6 +195,23 @@ module Hutch
69
195
  user_config[attr]
70
196
  end
71
197
 
198
+ def self.key_for(attr)
199
+ key = attr.to_s.gsub('.', '__').upcase
200
+ "HUTCH_#{key}"
201
+ end
202
+
203
+ def self.is_bool(attr)
204
+ @boolean_keys.include?(attr)
205
+ end
206
+
207
+ def self.to_bool(value)
208
+ !(value.nil? || value == '' || value =~ /^(false|f|no|n|0)$/i || value == false)
209
+ end
210
+
211
+ def self.is_num(attr)
212
+ @number_keys.include?(attr)
213
+ end
214
+
72
215
  def self.set(attr, value)
73
216
  check_attr(attr)
74
217
  user_config[attr] = value
@@ -81,17 +224,16 @@ module Hutch
81
224
 
82
225
  def self.check_attr(attr)
83
226
  unless user_config.key?(attr)
84
- raise UnknownAttributeError, "#{attr} is not a valid config attribute"
227
+ raise UnknownAttributeError, "#{attr.inspect} is not a valid config attribute"
85
228
  end
86
229
  end
87
230
 
88
231
  def self.user_config
89
- initialize unless @config
90
- @config
232
+ @config ||= initialize
91
233
  end
92
234
 
93
235
  def self.to_hash
94
- self.user_config
236
+ user_config
95
237
  end
96
238
 
97
239
  def self.load_from_file(file)
@@ -102,28 +244,23 @@ module Hutch
102
244
 
103
245
  def self.convert_value(attr, value)
104
246
  case attr
105
- when "tracer"
247
+ when 'tracer'
106
248
  Kernel.const_get(value)
107
249
  else
108
250
  value
109
251
  end
110
252
  end
111
253
 
112
- def self.method_missing(method, *args, &block)
113
- attr = method.to_s.sub(/=$/, '').to_sym
114
- return super unless user_config.key?(attr)
254
+ def self.define_methods
255
+ @config.keys.each do |key|
256
+ define_singleton_method(key) do
257
+ get(key)
258
+ end
115
259
 
116
- if method =~ /=$/
117
- set(attr, args.first)
118
- else
119
- get(attr)
260
+ define_singleton_method("#{key}=") do |val|
261
+ set(key, val)
262
+ end
120
263
  end
121
264
  end
122
-
123
- private
124
-
125
- def deep_copy(obj)
126
- Marshal.load(Marshal.dump(obj))
127
- end
128
265
  end
129
266
  end
@@ -16,7 +16,7 @@ module Hutch
16
16
  prefix = "message(#{message_id || '-'}): "
17
17
  logger.error prefix + "Logging event to Sentry"
18
18
  logger.error prefix + "#{ex.class} - #{ex.message}"
19
- Raven.capture_exception(ex)
19
+ Raven.capture_exception(ex, extra: { payload: payload })
20
20
  end
21
21
  end
22
22
  end
@@ -11,6 +11,7 @@ module Hutch
11
11
 
12
12
  def self.setup_logger(target = $stdout)
13
13
  require 'hutch/config'
14
+ Hutch::Config.initialize
14
15
  @logger = Logger.new(target)
15
16
  @logger.level = Hutch::Config.log_level
16
17
  @logger.formatter = HutchFormatter.new
@@ -0,0 +1,75 @@
1
+ require 'securerandom'
2
+ require 'hutch/logging'
3
+ require 'hutch/exceptions'
4
+
5
+ module Hutch
6
+ class Publisher
7
+ include Logging
8
+ attr_reader :connection, :channel, :exchange, :config
9
+
10
+ def initialize(connection, channel, exchange, config = Hutch::Config)
11
+ @connection = connection
12
+ @channel = channel
13
+ @exchange = exchange
14
+ @config = config
15
+ end
16
+
17
+ def publish(routing_key, message, properties = {}, options = {})
18
+ ensure_connection!(routing_key, message)
19
+
20
+ serializer = options[:serializer] || config[:serializer]
21
+
22
+ non_overridable_properties = {
23
+ routing_key: routing_key,
24
+ timestamp: connection.current_timestamp,
25
+ content_type: serializer.content_type,
26
+ }
27
+ properties[:message_id] ||= generate_id
28
+
29
+ payload = serializer.encode(message)
30
+
31
+ log_publication(serializer, payload, routing_key)
32
+
33
+ response = exchange.publish(payload, {persistent: true}.
34
+ merge(properties).
35
+ merge(global_properties).
36
+ merge(non_overridable_properties))
37
+
38
+ channel.wait_for_confirms if config[:force_publisher_confirms]
39
+ response
40
+ end
41
+
42
+ private
43
+
44
+ def log_publication(serializer, payload, routing_key)
45
+ logger.info {
46
+ spec =
47
+ if serializer.binary?
48
+ "#{payload.bytesize} bytes message"
49
+ else
50
+ "message '#{payload}'"
51
+ end
52
+ "publishing #{spec} to #{routing_key}"
53
+ }
54
+ end
55
+
56
+ def raise_publish_error(reason, routing_key, message)
57
+ msg = "unable to publish - #{reason}. Message: #{JSON.dump(message)}, Routing key: #{routing_key}."
58
+ logger.error(msg)
59
+ raise PublishError, msg
60
+ end
61
+
62
+ def ensure_connection!(routing_key, message)
63
+ raise_publish_error('no connection to broker', routing_key, message) unless connection
64
+ raise_publish_error('connection is closed', routing_key, message) unless connection.open?
65
+ end
66
+
67
+ def generate_id
68
+ SecureRandom.uuid
69
+ end
70
+
71
+ def global_properties
72
+ Hutch.global_properties.respond_to?(:call) ? Hutch.global_properties.call : Hutch.global_properties
73
+ end
74
+ end
75
+ end