hutch 0.21.0-java → 0.25.0-java

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. checksums.yaml +5 -5
  2. data/.gitignore +3 -0
  3. data/.rspec +1 -0
  4. data/.travis.yml +11 -12
  5. data/.yardopts +5 -0
  6. data/CHANGELOG.md +118 -1
  7. data/Gemfile +15 -4
  8. data/Guardfile +13 -4
  9. data/README.md +274 -24
  10. data/Rakefile +8 -1
  11. data/hutch.gemspec +6 -7
  12. data/lib/hutch.rb +11 -8
  13. data/lib/hutch/adapters/march_hare.rb +1 -1
  14. data/lib/hutch/broker.rb +113 -110
  15. data/lib/hutch/cli.rb +42 -11
  16. data/lib/hutch/config.rb +209 -59
  17. data/lib/hutch/error_handlers.rb +1 -0
  18. data/lib/hutch/error_handlers/airbrake.rb +44 -16
  19. data/lib/hutch/error_handlers/base.rb +15 -0
  20. data/lib/hutch/error_handlers/honeybadger.rb +33 -18
  21. data/lib/hutch/error_handlers/logger.rb +12 -6
  22. data/lib/hutch/error_handlers/opbeat.rb +30 -0
  23. data/lib/hutch/error_handlers/sentry.rb +14 -6
  24. data/lib/hutch/logging.rb +5 -5
  25. data/lib/hutch/publisher.rb +75 -0
  26. data/lib/hutch/tracers.rb +1 -0
  27. data/lib/hutch/tracers/opbeat.rb +37 -0
  28. data/lib/hutch/version.rb +1 -1
  29. data/lib/hutch/waiter.rb +104 -0
  30. data/lib/hutch/worker.rb +50 -66
  31. data/lib/yard-settings/handler.rb +38 -0
  32. data/lib/yard-settings/yard-settings.rb +2 -0
  33. data/spec/hutch/broker_spec.rb +162 -77
  34. data/spec/hutch/cli_spec.rb +16 -3
  35. data/spec/hutch/config_spec.rb +83 -22
  36. data/spec/hutch/error_handlers/airbrake_spec.rb +25 -10
  37. data/spec/hutch/error_handlers/honeybadger_spec.rb +24 -2
  38. data/spec/hutch/error_handlers/logger_spec.rb +14 -1
  39. data/spec/hutch/error_handlers/opbeat_spec.rb +37 -0
  40. data/spec/hutch/error_handlers/sentry_spec.rb +18 -1
  41. data/spec/hutch/logger_spec.rb +12 -6
  42. data/spec/hutch/waiter_spec.rb +51 -0
  43. data/spec/hutch/worker_spec.rb +33 -4
  44. data/spec/spec_helper.rb +7 -5
  45. data/spec/tracers/opbeat_spec.rb +44 -0
  46. data/templates/default/class/html/settings.erb +0 -0
  47. data/templates/default/class/setup.rb +4 -0
  48. data/templates/default/fulldoc/html/css/hutch.css +13 -0
  49. data/templates/default/layout/html/setup.rb +7 -0
  50. data/templates/default/method_details/html/settings.erb +5 -0
  51. data/templates/default/method_details/setup.rb +4 -0
  52. data/templates/default/method_details/text/settings.erb +0 -0
  53. data/templates/default/module/html/settings.erb +40 -0
  54. data/templates/default/module/setup.rb +4 -0
  55. metadata +41 -38
@@ -1,9 +1,9 @@
1
1
  require 'optparse'
2
2
 
3
- require 'hutch/version'
4
3
  require 'hutch/logging'
5
- require 'hutch/exceptions'
6
4
  require 'hutch/config'
5
+ require 'hutch/version'
6
+ require 'hutch/exceptions'
7
7
 
8
8
  module Hutch
9
9
  class CLI
@@ -11,9 +11,10 @@ module Hutch
11
11
 
12
12
  # Run a Hutch worker with the command line interface.
13
13
  def run(argv = ARGV)
14
+ Hutch::Config.initialize
14
15
  parse_options(argv)
15
16
 
16
- ::Process.daemon(true) if Hutch::Config.daemonise
17
+ daemonise_process
17
18
 
18
19
  write_pid if Hutch::Config.pidfile
19
20
 
@@ -71,8 +72,8 @@ module Hutch
71
72
  end
72
73
  rails_path = File.expand_path(File.join(path, 'config/environment.rb'))
73
74
  if is_rails_app && File.exist?(rails_path)
74
- logger.info "found rails project (#{path}), booting app"
75
75
  ENV['RACK_ENV'] ||= ENV['RAILS_ENV'] || 'development'
76
+ logger.info "found rails project (#{path}), booting app in #{ENV['RACK_ENV']} environment"
76
77
  require rails_path
77
78
  ::Rails.application.eager_load!
78
79
  return true
@@ -85,10 +86,13 @@ module Hutch
85
86
  # gracefully (with a SIGQUIT, SIGTERM or SIGINT).
86
87
  def start_work_loop
87
88
  Hutch.connect
88
- @worker = Hutch::Worker.new(Hutch.broker, Hutch.consumers)
89
+ @worker = Hutch::Worker.new(Hutch.broker, Hutch.consumers, Hutch::Config.setup_procs)
89
90
  @worker.run
90
91
  :success
91
92
  rescue ConnectionError, AuthenticationError, WorkerSetupError => ex
93
+ Hutch::Config[:error_handlers].each do |backend|
94
+ backend.handle_setup_exception(ex)
95
+ end
92
96
  logger.fatal ex.message
93
97
  :error
94
98
  end
@@ -109,14 +113,16 @@ module Hutch
109
113
  Hutch::Config.mq_tls = tls
110
114
  end
111
115
 
112
- opts.on('--mq-tls-cert FILE', 'Certificate for TLS client verification') do |file|
113
- abort "Certificate file '#{file}' not found" unless File.exists?(file)
114
- Hutch::Config.mq_tls_cert = file
116
+ opts.on('--mq-tls-cert FILE', 'Certificate for TLS client verification') do |file|
117
+ abort_without_file(file, 'Certificate file') do
118
+ Hutch::Config.mq_tls_cert = file
119
+ end
115
120
  end
116
121
 
117
122
  opts.on('--mq-tls-key FILE', 'Private key for TLS client verification') do |file|
118
- abort "Private key file '#{file}' not found" unless File.exists?(file)
119
- Hutch::Config.mq_tls_key = file
123
+ abort_without_file(file, 'Private key file') do
124
+ Hutch::Config.mq_tls_key = file
125
+ end
120
126
  end
121
127
 
122
128
  opts.on('--mq-exchange EXCHANGE',
@@ -154,7 +160,7 @@ module Hutch
154
160
  begin
155
161
  File.open(file) { |fp| Hutch::Config.load_from_file(fp) }
156
162
  rescue Errno::ENOENT
157
- abort "Config file '#{file}' not found"
163
+ abort_with_message("Config file '#{file}' not found")
158
164
  end
159
165
  end
160
166
 
@@ -186,6 +192,10 @@ module Hutch
186
192
  Hutch::Config.pidfile = pidfile
187
193
  end
188
194
 
195
+ opts.on('--only-group GROUP', 'Load only consumers in this group') do |group|
196
+ Hutch::Config.group = group
197
+ end
198
+
189
199
  opts.on('--version', 'Print the version and exit') do
190
200
  puts "hutch v#{VERSION}"
191
201
  exit 0
@@ -204,5 +214,26 @@ module Hutch
204
214
  File.open(pidfile, 'w') { |f| f.puts ::Process.pid }
205
215
  end
206
216
 
217
+ private
218
+
219
+ def daemonise_process
220
+ return unless Hutch::Config.daemonise
221
+ if defined?(JRUBY_VERSION)
222
+ Hutch.logger.warn "JRuby ignores the --daemonise option"
223
+ return
224
+ end
225
+
226
+ ::Process.daemon(true)
227
+ end
228
+
229
+ def abort_without_file(file, file_description, &block)
230
+ abort_with_message("#{file_description} '#{file}' not found") unless File.exists?(file)
231
+
232
+ yield
233
+ end
234
+
235
+ def abort_with_message(message)
236
+ abort message
237
+ end
207
238
  end
208
239
  end
@@ -1,77 +1,232 @@
1
1
  require 'hutch/error_handlers/logger'
2
+ require 'hutch/tracers'
3
+ require 'hutch/serializers/json'
2
4
  require 'erb'
3
5
  require 'logger'
4
6
 
5
7
  module Hutch
6
8
  class UnknownAttributeError < StandardError; end
7
9
 
10
+ # Configuration settings, available everywhere
11
+ #
12
+ # There are defaults, which can be overridden by ENV variables prefixed by
13
+ # <tt>HUTCH_</tt>, and each of these can be overridden using the {.set}
14
+ # method.
15
+ #
16
+ # @example Configuring on the command-line
17
+ # HUTCH_PUBLISHER_CONFIRMS=false hutch
8
18
  module Config
9
19
  require 'yaml'
20
+ @string_keys = Set.new
21
+ @number_keys = Set.new
22
+ @boolean_keys = Set.new
23
+ @settings_defaults = {}
24
+
25
+ # Define a String user setting
26
+ # @!visibility private
27
+ def self.string_setting(name, default_value)
28
+ @string_keys << name
29
+ @settings_defaults[name] = default_value
30
+ end
31
+
32
+ # Define a Number user setting
33
+ # @!visibility private
34
+ def self.number_setting(name, default_value)
35
+ @number_keys << name
36
+ @settings_defaults[name] = default_value
37
+ end
38
+
39
+ # Define a Boolean user setting
40
+ # @!visibility private
41
+ def self.boolean_setting(name, default_value)
42
+ @boolean_keys << name
43
+ @settings_defaults[name] = default_value
44
+ end
45
+
46
+ # RabbitMQ hostname
47
+ string_setting :mq_host, '127.0.0.1'
48
+
49
+ # RabbitMQ Exchange to use for publishing
50
+ string_setting :mq_exchange, 'hutch'
51
+
52
+ # RabbitMQ vhost to use
53
+ string_setting :mq_vhost, '/'
54
+
55
+ # RabbitMQ username to use.
56
+ #
57
+ # As of RabbitMQ 3.3.0, <tt>guest</tt> can only can connect from localhost.
58
+ string_setting :mq_username, 'guest'
59
+
60
+ # RabbitMQ password
61
+ string_setting :mq_password, 'guest'
62
+
63
+ # RabbitMQ URI (takes precedence over MQ username, password, host, port and vhost settings)
64
+ string_setting :uri, nil
65
+
66
+ # RabbitMQ HTTP API hostname
67
+ string_setting :mq_api_host, '127.0.0.1'
68
+
69
+ # RabbitMQ port
70
+ number_setting :mq_port, 5672
71
+
72
+ # RabbitMQ HTTP API port
73
+ number_setting :mq_api_port, 15672
74
+
75
+ # [RabbitMQ heartbeat timeout](http://rabbitmq.com/heartbeats.html)
76
+ number_setting :heartbeat, 30
77
+
78
+ # The <tt>basic.qos</tt> prefetch value to use.
79
+ #
80
+ # Default: `0`, no limit. See Bunny and RabbitMQ documentation.
81
+ number_setting :channel_prefetch, 0
82
+
83
+ # Bunny's socket open timeout
84
+ number_setting :connection_timeout, 11
85
+
86
+ # Bunny's socket read timeout
87
+ number_setting :read_timeout, 11
88
+
89
+ # Bunny's socket write timeout
90
+ number_setting :write_timeout, 11
91
+
92
+ # FIXME: DOCUMENT THIS
93
+ number_setting :graceful_exit_timeout, 11
94
+
95
+ # Bunny consumer work pool size
96
+ number_setting :consumer_pool_size, 1
97
+
98
+ # Should TLS be used?
99
+ boolean_setting :mq_tls, false
100
+
101
+ # Should SSL certificate be verified?
102
+ boolean_setting :mq_verify_peer, true
103
+
104
+ # Should SSL be used for the RabbitMQ API?
105
+ boolean_setting :mq_api_ssl, false
106
+
107
+ # Should the current Rails app directory be required?
108
+ boolean_setting :autoload_rails, true
109
+
110
+ # Should the Hutch runner process daemonise?
111
+ #
112
+ # The option is ignored on JRuby.
113
+ boolean_setting :daemonise, false
114
+
115
+ # Should RabbitMQ publisher confirms be enabled?
116
+ #
117
+ # Leaves it up to the app how they are tracked
118
+ # (e.g. using Hutch::Broker#confirm_select callback or Hutch::Broker#wait_for_confirms)
119
+ boolean_setting :publisher_confirms, false
120
+
121
+ # Enables publisher confirms, forces Hutch::Broker#wait_for_confirms for
122
+ # every publish.
123
+ #
124
+ # **This is the safest option which also offers the
125
+ # lowest throughput**.
126
+ boolean_setting :force_publisher_confirms, false
127
+
128
+ # Should the RabbitMQ HTTP API be used?
129
+ boolean_setting :enable_http_api_use, true
130
+
131
+ # Should Bunny's consumer work pool threads abort on exception.
132
+ #
133
+ # The option is ignored on JRuby.
134
+ boolean_setting :consumer_pool_abort_on_exception, false
135
+
136
+ # Prefix displayed on the consumers tags.
137
+ string_setting :consumer_tag_prefix, 'hutch'
138
+
139
+ string_setting :group, ''
140
+
141
+ # Set of all setting keys
142
+ ALL_KEYS = @boolean_keys + @number_keys + @string_keys
10
143
 
11
144
  def self.initialize(params = {})
12
- @config = {
13
- mq_host: 'localhost',
14
- mq_port: 5672,
15
- mq_exchange: 'hutch', # TODO: should this be required?
145
+ unless @config
146
+ @config = default_config
147
+ define_methods
148
+ @config.merge!(env_based_config)
149
+ end
150
+ @config.merge!(params)
151
+ @config
152
+ end
153
+
154
+ # Default settings
155
+ #
156
+ # @return [Hash]
157
+ def self.default_config
158
+ @settings_defaults.merge({
16
159
  mq_exchange_options: {},
17
- mq_vhost: '/',
18
- mq_tls: false,
19
160
  mq_tls_cert: nil,
20
161
  mq_tls_key: nil,
21
162
  mq_tls_ca_certificates: nil,
22
- mq_verify_peer: true,
23
- mq_username: 'guest',
24
- mq_password: 'guest',
25
- mq_api_host: 'localhost',
26
- mq_api_port: 15672,
27
- mq_api_ssl: false,
28
- heartbeat: 30,
29
- # placeholder, allows specifying connection parameters
30
- # as a URI.
31
163
  uri: nil,
32
164
  log_level: Logger::INFO,
165
+ client_logger: nil,
33
166
  require_paths: [],
34
- autoload_rails: true,
35
167
  error_handlers: [Hutch::ErrorHandlers::Logger.new],
36
168
  # note that this is not a list, it is a chain of responsibility
37
169
  # that will fall back to "nack unconditionally"
38
170
  error_acknowledgements: [],
171
+ setup_procs: [],
172
+ consumer_groups: {},
39
173
  tracer: Hutch::Tracers::NullTracer,
40
174
  namespace: nil,
41
- daemonise: false,
42
175
  pidfile: nil,
43
- channel_prefetch: 0,
44
- # enables publisher confirms, leaves it up to the app
45
- # how they are tracked
46
- publisher_confirms: false,
47
- # like `publisher_confirms` above but also
48
- # forces waiting for a confirm for every publish
49
- force_publisher_confirms: false,
50
- # Heroku needs > 10. MK.
51
- connection_timeout: 11,
52
- read_timeout: 11,
53
- write_timeout: 11,
54
- enable_http_api_use: true,
55
- # Number of seconds that a running consumer is given
56
- # to finish its job when gracefully exiting Hutch, before
57
- # it's killed.
58
- graceful_exit_timeout: 11,
59
- client_logger: nil,
176
+ serializer: Hutch::Serializers::JSON
177
+ })
178
+ end
179
+
180
+ # Override defaults with ENV variables which begin with <tt>HUTCH_</tt>
181
+ #
182
+ # @return [Hash]
183
+ def self.env_based_config
184
+ env_keys_configured.each_with_object({}) {|attr, result|
185
+ value = ENV[key_for(attr)]
186
+
187
+ case
188
+ when is_bool(attr) || value == 'false'
189
+ result[attr] = to_bool(value)
190
+ when is_num(attr)
191
+ result[attr] = value.to_i
192
+ else
193
+ result[attr] = value
194
+ end
195
+ }
196
+ end
60
197
 
61
- consumer_pool_size: 1,
198
+ # @return [Array<Symbol>]
199
+ def self.env_keys_configured
200
+ ALL_KEYS.each {|attr| check_attr(attr) }
62
201
 
63
- serializer: Hutch::Serializers::JSON,
64
- }.merge(params)
202
+ ALL_KEYS.select { |attr| ENV.key?(key_for(attr)) }
65
203
  end
66
204
 
67
205
  def self.get(attr)
68
- check_attr(attr)
69
- user_config[attr]
206
+ check_attr(attr.to_sym)
207
+ user_config[attr.to_sym]
208
+ end
209
+
210
+ def self.key_for(attr)
211
+ key = attr.to_s.gsub('.', '__').upcase
212
+ "HUTCH_#{key}"
213
+ end
214
+
215
+ def self.is_bool(attr)
216
+ @boolean_keys.include?(attr)
217
+ end
218
+
219
+ def self.to_bool(value)
220
+ !(value.nil? || value == '' || value =~ /^(false|f|no|n|0)$/i || value == false)
221
+ end
222
+
223
+ def self.is_num(attr)
224
+ @number_keys.include?(attr)
70
225
  end
71
226
 
72
227
  def self.set(attr, value)
73
- check_attr(attr)
74
- user_config[attr] = value
228
+ check_attr(attr.to_sym)
229
+ user_config[attr.to_sym] = value
75
230
  end
76
231
 
77
232
  class << self
@@ -81,17 +236,16 @@ module Hutch
81
236
 
82
237
  def self.check_attr(attr)
83
238
  unless user_config.key?(attr)
84
- raise UnknownAttributeError, "#{attr} is not a valid config attribute"
239
+ raise UnknownAttributeError, "#{attr.inspect} is not a valid config attribute"
85
240
  end
86
241
  end
87
242
 
88
243
  def self.user_config
89
- initialize unless @config
90
- @config
244
+ @config ||= initialize
91
245
  end
92
246
 
93
247
  def self.to_hash
94
- self.user_config
248
+ user_config
95
249
  end
96
250
 
97
251
  def self.load_from_file(file)
@@ -102,28 +256,24 @@ module Hutch
102
256
 
103
257
  def self.convert_value(attr, value)
104
258
  case attr
105
- when "tracer"
259
+ when 'tracer'
106
260
  Kernel.const_get(value)
107
261
  else
108
262
  value
109
263
  end
110
264
  end
111
265
 
112
- def self.method_missing(method, *args, &block)
113
- attr = method.to_s.sub(/=$/, '').to_sym
114
- return super unless user_config.key?(attr)
266
+ def self.define_methods
267
+ @config.keys.each do |key|
268
+ define_singleton_method(key) do
269
+ get(key)
270
+ end
115
271
 
116
- if method =~ /=$/
117
- set(attr, args.first)
118
- else
119
- get(attr)
272
+ define_singleton_method("#{key}=") do |val|
273
+ set(key, val)
274
+ end
120
275
  end
121
276
  end
122
-
123
- private
124
-
125
- def deep_copy(obj)
126
- Marshal.load(Marshal.dump(obj))
127
- end
128
277
  end
129
278
  end
279
+ Hutch::Config.initialize
@@ -4,5 +4,6 @@ module Hutch
4
4
  autoload :Sentry, 'hutch/error_handlers/sentry'
5
5
  autoload :Honeybadger, 'hutch/error_handlers/honeybadger'
6
6
  autoload :Airbrake, 'hutch/error_handlers/airbrake'
7
+ autoload :Opbeat, 'hutch/error_handlers/opbeat'
7
8
  end
8
9
  end